Auto-update README.md with cargo run --release -- --write-results-table
This commit is contained in:
parent
ef860fa73b
commit
8337c61ea2
3 changed files with 144 additions and 40 deletions
34
README.md
34
README.md
|
@ -55,25 +55,23 @@ Some examples:
|
||||||
- [A cheating strategy](src/strategies/cheating.rs), using `Rc<RefCell<_>>`
|
- [A cheating strategy](src/strategies/cheating.rs), using `Rc<RefCell<_>>`
|
||||||
- [The information strategy](src/strategies/information.rs)!
|
- [The information strategy](src/strategies/information.rs)!
|
||||||
|
|
||||||
## Results
|
## Results (auto-generated)
|
||||||
|
|
||||||
On seeds 0-9999, we have these average scores and win rates:
|
|
||||||
|
|
||||||
| | 2p | 3p | 4p | 5p |
|
|
||||||
|-------|---------|---------|---------|---------|
|
|
||||||
|cheat | 24.8600 | 24.9781 | 24.9715 | 24.9570 |
|
|
||||||
| | 90.52 % | 98.12 % | 97.74 % | 96.57 % |
|
|
||||||
|info | 22.3386 | 24.7322 | 24.8921 | 24.8996 |
|
|
||||||
| | 09.86 % | 80.75 % | 91.58 % | 92.11 % |
|
|
||||||
|
|
||||||
|
|
||||||
To reproduce:
|
To reproduce:
|
||||||
```
|
```
|
||||||
n=10000 # number of rounds to simulate
|
time cargo run --release -- --results-table
|
||||||
t=4 # number of threads
|
|
||||||
for strategy in info cheat; do
|
|
||||||
for p in $(seq 2 5); do
|
|
||||||
time cargo run --release -- -n $n -s 0 -t $t -p $p -g $strategy;
|
|
||||||
done
|
|
||||||
done
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
To update this file:
|
||||||
|
```
|
||||||
|
time cargo run --release -- --write-results-table
|
||||||
|
```
|
||||||
|
|
||||||
|
On the first 20000 seeds, we have these average scores and win rates:
|
||||||
|
|
||||||
|
| | 2p | 3p | 4p | 5p |
|
||||||
|
|---------|---------|---------|---------|---------|
|
||||||
|
| cheat | 24.8594 | 24.9785 | 24.9720 | 24.9557 |
|
||||||
|
| | 90.59 % | 98.17 % | 97.76 % | 96.42 % |
|
||||||
|
| info | 22.3249 | 24.7278 | 24.8919 | 24.8961 |
|
||||||
|
| | 09.81 % | 80.54 % | 91.67 % | 91.90 % |
|
||||||
|
|
94
src/main.rs
94
src/main.rs
|
@ -65,6 +65,10 @@ fn main() {
|
||||||
"STRATEGY");
|
"STRATEGY");
|
||||||
opts.optflag("h", "help",
|
opts.optflag("h", "help",
|
||||||
"Print this help menu");
|
"Print this help menu");
|
||||||
|
opts.optflag("", "results-table",
|
||||||
|
"Print a table of results for each strategy");
|
||||||
|
opts.optflag("", "write-results-table",
|
||||||
|
"Update the results table in README.md");
|
||||||
let matches = match opts.parse(&args[1..]) {
|
let matches = match opts.parse(&args[1..]) {
|
||||||
Ok(m) => { m }
|
Ok(m) => { m }
|
||||||
Err(f) => {
|
Err(f) => {
|
||||||
|
@ -78,6 +82,12 @@ fn main() {
|
||||||
if !matches.free.is_empty() {
|
if !matches.free.is_empty() {
|
||||||
return print_usage(&program, opts);
|
return print_usage(&program, opts);
|
||||||
}
|
}
|
||||||
|
if matches.opt_present("write-results-table") {
|
||||||
|
return write_results_table();
|
||||||
|
}
|
||||||
|
if matches.opt_present("results-table") {
|
||||||
|
return print!("{}", get_results_table());
|
||||||
|
}
|
||||||
|
|
||||||
let log_level_str : &str = &matches.opt_str("l").unwrap_or("info".to_string());
|
let log_level_str : &str = &matches.opt_str("l").unwrap_or("info".to_string());
|
||||||
let log_level = match log_level_str {
|
let log_level = match log_level_str {
|
||||||
|
@ -97,15 +107,18 @@ fn main() {
|
||||||
Box::new(SimpleLogger)
|
Box::new(SimpleLogger)
|
||||||
}).unwrap();
|
}).unwrap();
|
||||||
|
|
||||||
let n = u32::from_str(&matches.opt_str("n").unwrap_or("1".to_string())).unwrap();
|
let n_trials = u32::from_str(&matches.opt_str("n").unwrap_or("1".to_string())).unwrap();
|
||||||
|
|
||||||
let seed = matches.opt_str("s").map(|seed_str| { u32::from_str(&seed_str).unwrap() });
|
let seed = matches.opt_str("s").map(|seed_str| { u32::from_str(&seed_str).unwrap() });
|
||||||
|
|
||||||
let progress_info = matches.opt_str("o").map(|freq_str| { u32::from_str(&freq_str).unwrap() });
|
let progress_info = matches.opt_str("o").map(|freq_str| { u32::from_str(&freq_str).unwrap() });
|
||||||
|
|
||||||
let n_threads = u32::from_str(&matches.opt_str("t").unwrap_or("1".to_string())).unwrap();
|
let n_threads = u32::from_str(&matches.opt_str("t").unwrap_or("1".to_string())).unwrap();
|
||||||
|
|
||||||
let n_players = u32::from_str(&matches.opt_str("p").unwrap_or("4".to_string())).unwrap();
|
let n_players = u32::from_str(&matches.opt_str("p").unwrap_or("4".to_string())).unwrap();
|
||||||
|
let strategy_str : &str = &matches.opt_str("g").unwrap_or("cheat".to_string());
|
||||||
|
|
||||||
|
sim_games(n_players, strategy_str, seed, n_trials, n_threads, progress_info).info();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sim_games(n_players: u32, strategy_str: &str, seed: Option<u32>, n_trials: u32, n_threads: u32, progress_info: Option<u32>)
|
||||||
|
-> simulator::SimResult {
|
||||||
let hand_size = match n_players {
|
let hand_size = match n_players {
|
||||||
2 => 5,
|
2 => 5,
|
||||||
3 => 5,
|
3 => 5,
|
||||||
|
@ -123,7 +136,6 @@ fn main() {
|
||||||
allow_empty_hints: false,
|
allow_empty_hints: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
let strategy_str : &str = &matches.opt_str("g").unwrap_or("cheat".to_string());
|
|
||||||
let strategy_config : Box<strategy::GameStrategyConfig + Sync> = match strategy_str {
|
let strategy_config : Box<strategy::GameStrategyConfig + Sync> = match strategy_str {
|
||||||
"random" => {
|
"random" => {
|
||||||
Box::new(strategies::examples::RandomStrategyConfig {
|
Box::new(strategies::examples::RandomStrategyConfig {
|
||||||
|
@ -140,9 +152,75 @@ fn main() {
|
||||||
as Box<strategy::GameStrategyConfig + Sync>
|
as Box<strategy::GameStrategyConfig + Sync>
|
||||||
},
|
},
|
||||||
_ => {
|
_ => {
|
||||||
print_usage(&program, opts);
|
|
||||||
panic!("Unexpected strategy argument {}", strategy_str);
|
panic!("Unexpected strategy argument {}", strategy_str);
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
simulator::simulate(&game_opts, strategy_config, seed, n, n_threads, progress_info);
|
simulator::simulate(&game_opts, strategy_config, seed, n_trials, n_threads, progress_info)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_results_table() -> String {
|
||||||
|
let strategies = ["cheat", "info"];
|
||||||
|
let player_nums = (2..=5).collect::<Vec<_>>();
|
||||||
|
let seed = 0;
|
||||||
|
let n_trials = 1;
|
||||||
|
let n_threads = 8;
|
||||||
|
|
||||||
|
let intro = format!("On the first {} seeds, we have these average scores and win rates:\n\n", n_trials);
|
||||||
|
let format_name = |x| format!(" {:7} ", x);
|
||||||
|
let format_players = |x| format!(" {}p ", x);
|
||||||
|
let format_percent = |x| format!(" {:05.2} % ", x);
|
||||||
|
let format_score = |x| format!(" {:07.4} ", x);
|
||||||
|
let space = String::from(" ");
|
||||||
|
let dashes = String::from("---------");
|
||||||
|
type TwoLines = (String, String);
|
||||||
|
fn make_twolines(player_nums: &Vec<u32>, head: TwoLines, make_block: &dyn Fn(u32) -> TwoLines) -> TwoLines {
|
||||||
|
let mut blocks = player_nums.iter().cloned().map(make_block).collect::<Vec<_>>();
|
||||||
|
blocks.insert(0, head);
|
||||||
|
fn combine(items: Vec<String>) -> String {
|
||||||
|
items.iter().fold(String::from("|"), |init, next| { init + next + "|" })
|
||||||
|
}
|
||||||
|
let (a, b): (Vec<_>, Vec<_>) = blocks.into_iter().unzip();
|
||||||
|
(combine(a), combine(b))
|
||||||
|
}
|
||||||
|
fn concat_twolines(body: Vec<TwoLines>) -> String {
|
||||||
|
body.into_iter().fold(String::default(), |output, (a, b)| (output + &a + "\n" + &b + "\n"))
|
||||||
|
}
|
||||||
|
let header = make_twolines(&player_nums, (space.clone(), dashes.clone()), &|n_players| (format_players(n_players), dashes.clone()));
|
||||||
|
let mut body = strategies.iter().map(|strategy| {
|
||||||
|
make_twolines(&player_nums, (format_name(strategy), space.clone()), &|n_players| {
|
||||||
|
let simresult = sim_games(n_players, strategy, Some(seed), n_trials, n_threads, None);
|
||||||
|
(format_score(simresult.average_score()), format_percent(simresult.percent_perfect()))
|
||||||
|
})
|
||||||
|
}).collect::<Vec<_>>();
|
||||||
|
body.insert(0, header);
|
||||||
|
intro + &concat_twolines(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_results_table() {
|
||||||
|
let separator = r#"
|
||||||
|
## Results (auto-generated)
|
||||||
|
|
||||||
|
To reproduce:
|
||||||
|
```
|
||||||
|
time cargo run --release -- --results-table
|
||||||
|
```
|
||||||
|
|
||||||
|
To update this file:
|
||||||
|
```
|
||||||
|
time cargo run --release -- --write-results-table
|
||||||
|
```
|
||||||
|
|
||||||
|
"#;
|
||||||
|
let readme = "README.md";
|
||||||
|
let readme_contents = std::fs::read_to_string(readme).unwrap();
|
||||||
|
let readme_init = {
|
||||||
|
let parts = readme_contents.splitn(2, separator).collect::<Vec<_>>();
|
||||||
|
if parts.len() != 2 {
|
||||||
|
panic!("{} has been modified in the Results section!", readme);
|
||||||
|
}
|
||||||
|
parts[0]
|
||||||
|
};
|
||||||
|
let table = get_results_table();
|
||||||
|
let new_readme_contents = String::from(readme_init) + separator + &table;
|
||||||
|
std::fs::write(readme, new_readme_contents).unwrap();
|
||||||
}
|
}
|
||||||
|
|
|
@ -66,7 +66,7 @@ pub fn simulate_once(
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct Histogram {
|
pub struct Histogram {
|
||||||
pub hist: FnvHashMap<Score, u32>,
|
pub hist: FnvHashMap<Score, u32>,
|
||||||
pub sum: Score,
|
pub sum: Score,
|
||||||
pub total_count: u32,
|
pub total_count: u32,
|
||||||
|
@ -123,7 +123,8 @@ pub fn simulate<T: ?Sized>(
|
||||||
n_trials: u32,
|
n_trials: u32,
|
||||||
n_threads: u32,
|
n_threads: u32,
|
||||||
progress_info: Option<u32>,
|
progress_info: Option<u32>,
|
||||||
) where T: GameStrategyConfig + Sync {
|
) -> SimResult
|
||||||
|
where T: GameStrategyConfig + Sync {
|
||||||
|
|
||||||
let first_seed = first_seed_opt.unwrap_or_else(|| rand::thread_rng().next_u32());
|
let first_seed = first_seed_opt.unwrap_or_else(|| rand::thread_rng().next_u32());
|
||||||
|
|
||||||
|
@ -156,7 +157,7 @@ pub fn simulate<T: ?Sized>(
|
||||||
let score = game.score();
|
let score = game.score();
|
||||||
lives_histogram.insert(game.board.lives_remaining);
|
lives_histogram.insert(game.board.lives_remaining);
|
||||||
score_histogram.insert(score);
|
score_histogram.insert(score);
|
||||||
if score != PERFECT_SCORE { non_perfect_seeds.push((score, seed)); }
|
if score != PERFECT_SCORE { non_perfect_seeds.push(seed); }
|
||||||
}
|
}
|
||||||
if progress_info.is_some() {
|
if progress_info.is_some() {
|
||||||
info!("Thread {} done", i);
|
info!("Thread {} done", i);
|
||||||
|
@ -165,7 +166,7 @@ pub fn simulate<T: ?Sized>(
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut non_perfect_seeds : Vec<(Score,u32)> = Vec::new();
|
let mut non_perfect_seeds : Vec<u32> = Vec::new();
|
||||||
let mut score_histogram = Histogram::new();
|
let mut score_histogram = Histogram::new();
|
||||||
let mut lives_histogram = Histogram::new();
|
let mut lives_histogram = Histogram::new();
|
||||||
for join_handle in join_handles {
|
for join_handle in join_handles {
|
||||||
|
@ -175,17 +176,44 @@ pub fn simulate<T: ?Sized>(
|
||||||
lives_histogram.merge(thread_lives_histogram);
|
lives_histogram.merge(thread_lives_histogram);
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("Score histogram:\n{}", score_histogram);
|
|
||||||
|
|
||||||
non_perfect_seeds.sort();
|
non_perfect_seeds.sort();
|
||||||
// info!("Seeds with non-perfect score: {:?}", non_perfect_seeds);
|
SimResult {
|
||||||
if non_perfect_seeds.len() > 0 {
|
scores: score_histogram,
|
||||||
info!("Example seed with non-perfect score: {}",
|
lives: lives_histogram,
|
||||||
non_perfect_seeds.get(0).unwrap().1);
|
non_perfect_seed: non_perfect_seeds.get(0).cloned(),
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("Percentage perfect: {:?}%", score_histogram.percentage_with(&PERFECT_SCORE) * 100.0);
|
|
||||||
info!("Average score: {:?}", score_histogram.average());
|
|
||||||
info!("Average lives: {:?}", lives_histogram.average());
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct SimResult {
|
||||||
|
pub scores: Histogram,
|
||||||
|
pub lives: Histogram,
|
||||||
|
pub non_perfect_seed: Option<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SimResult {
|
||||||
|
pub fn percent_perfect(&self) -> f32 {
|
||||||
|
self.scores.percentage_with(&PERFECT_SCORE) * 100.0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn average_score(&self) -> f32 {
|
||||||
|
self.scores.average()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn average_lives(&self) -> f32 {
|
||||||
|
self.lives.average()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn info(&self) {
|
||||||
|
info!("Score histogram:\n{}", self.scores);
|
||||||
|
|
||||||
|
// info!("Seeds with non-perfect score: {:?}", non_perfect_seeds);
|
||||||
|
if let Some(seed) = self.non_perfect_seed {
|
||||||
|
info!("Example seed with non-perfect score: {}", seed);
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Percentage perfect: {:?}%", self.percent_perfect());
|
||||||
|
info!("Average score: {:?}", self.average_score());
|
||||||
|
info!("Average lives: {:?}", self.average_lives());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue