From 8fd230da1cb191001c2b5715bc6967281c84af26 Mon Sep 17 00:00:00 2001 From: Felix Bauckholt Date: Sat, 9 Mar 2019 14:41:18 +0100 Subject: [PATCH 1/2] Report standard errors in README.md --- README.md | 12 ++++++------ src/main.rs | 22 ++++++++++++++-------- src/simulator.rs | 21 +++++++++++++++++++++ 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index c1e8a05..551cbef 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,9 @@ time cargo run --release -- --write-results-table On the first 20000 seeds, we have these average scores and win rates: -| | 2p | 3p | 4p | 5p | -|---------|---------|---------|---------|---------| -| cheat | 24.8594 | 24.9785 | 24.9720 | 24.9557 | -| | 90.59 % | 98.17 % | 97.76 % | 96.42 % | -| info | 22.3249 | 24.7278 | 24.8919 | 24.8961 | -| | 09.81 % | 80.54 % | 91.67 % | 91.90 % | +| | 2p (stderr) | 3p (stderr) | 4p (stderr) | 5p (stderr) | +|---------|------------------|------------------|------------------|------------------| +| cheat | 24.8594 (0.0036) | 24.9785 (0.0012) | 24.9720 (0.0014) | 24.9557 (0.0018) | +| | 90.59 % (0.21 %) | 98.17 % (0.09 %) | 97.76 % (0.10 %) | 96.42 % (0.13 %) | +| info | 22.3249 (0.0128) | 24.7278 (0.0046) | 24.8919 (0.0029) | 24.8961 (0.0027) | +| | 09.81 % (0.21 %) | 80.54 % (0.28 %) | 91.67 % (0.20 %) | 91.90 % (0.19 %) | diff --git a/src/main.rs b/src/main.rs index c322ba9..e044557 100644 --- a/src/main.rs +++ b/src/main.rs @@ -166,12 +166,13 @@ fn get_results_table() -> String { let n_threads = 8; let intro = format!("On the first {} seeds, we have these average scores and win rates:\n\n", n_trials); - let format_name = |x| format!(" {:7} ", x); - let format_players = |x| format!(" {}p ", x); - let format_percent = |x| format!(" {:05.2} % ", x); - let format_score = |x| format!(" {:07.4} ", x); - let space = String::from(" "); - let dashes = String::from("---------"); + let format_name = |x| format!(" {:7} ", x); + let format_players = |x| format!(" {}p (stderr) ", x); + let format_percent = |x, stderr| format!(" {:05.2} % ({:.2} %) ", x, stderr); + let format_score = |x, stderr| format!(" {:07.4} ({:.4}) ", x, stderr); + let space = String::from(" "); + let dashes = String::from("---------"); + let dashes_long = String::from("------------------"); type TwoLines = (String, String); fn make_twolines(player_nums: &Vec, head: TwoLines, make_block: &dyn Fn(u32) -> TwoLines) -> TwoLines { let mut blocks = player_nums.iter().cloned().map(make_block).collect::>(); @@ -185,11 +186,16 @@ fn get_results_table() -> String { fn concat_twolines(body: Vec) -> String { body.into_iter().fold(String::default(), |output, (a, b)| (output + &a + "\n" + &b + "\n")) } - let header = make_twolines(&player_nums, (space.clone(), dashes.clone()), &|n_players| (format_players(n_players), dashes.clone())); + let header = make_twolines(&player_nums, + (space.clone(), dashes.clone()), + &|n_players| (format_players(n_players), dashes_long.clone())); let mut body = strategies.iter().map(|strategy| { make_twolines(&player_nums, (format_name(strategy), space.clone()), &|n_players| { let simresult = sim_games(n_players, strategy, Some(seed), n_trials, n_threads, None); - (format_score(simresult.average_score()), format_percent(simresult.percent_perfect())) + ( + format_score(simresult.average_score(), simresult.score_stderr()), + format_percent(simresult.percent_perfect(), simresult.percent_perfect_stderr()) + ) }) }).collect::>(); body.insert(0, header); diff --git a/src/simulator.rs b/src/simulator.rs index 831521f..cf96183 100644 --- a/src/simulator.rs +++ b/src/simulator.rs @@ -97,6 +97,17 @@ impl Histogram { pub fn average(&self) -> f32 { (self.sum as f32) / (self.total_count as f32) } + pub fn stdev_of_average(&self) -> f32 { + let average = self.average(); + let mut var_sum = 0.0; + for (&val, &count) in self.hist.iter() { + var_sum += (val as f32 - average).powi(2) * count as f32; + } + // Divide by (self.total_count - 1) estimate the variance of the distribution, + // then divide by self.total_count estimate the variance of the sample average, + // then take the sqrt to get the stdev. + (var_sum / (((self.total_count - 1) * self.total_count) as f32)).sqrt() + } pub fn merge(&mut self, other: Histogram) { for (val, count) in other.hist.into_iter() { self.insert_many(val, count); @@ -196,10 +207,20 @@ impl SimResult { self.scores.percentage_with(&PERFECT_SCORE) * 100.0 } + pub fn percent_perfect_stderr(&self) -> f32 { + let pp = self.percent_perfect() / 100.0; + let stdev = (pp*(1.0 - pp) / ((self.scores.total_count - 1) as f32)).sqrt(); + stdev * 100.0 + } + pub fn average_score(&self) -> f32 { self.scores.average() } + pub fn score_stderr(&self) -> f32 { + self.scores.stdev_of_average() + } + pub fn average_lives(&self) -> f32 { self.lives.average() } From c791adcb5a078e7eaddaea0ad68b80c5ef676365 Mon Sep 17 00:00:00 2001 From: Jeff Wu Date: Sun, 10 Mar 2019 22:26:24 -0700 Subject: [PATCH 2/2] make readme slightly nicer --- README.md | 12 ++++++------ src/main.rs | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 551cbef..e8f75a6 100644 --- a/README.md +++ b/README.md @@ -67,11 +67,11 @@ To update this file: time cargo run --release -- --write-results-table ``` -On the first 20000 seeds, we have these average scores and win rates: +On the first 20000 seeds, we have these scores and win rates (average ± standard error): -| | 2p (stderr) | 3p (stderr) | 4p (stderr) | 5p (stderr) | +| | 2p | 3p | 4p | 5p | |---------|------------------|------------------|------------------|------------------| -| cheat | 24.8594 (0.0036) | 24.9785 (0.0012) | 24.9720 (0.0014) | 24.9557 (0.0018) | -| | 90.59 % (0.21 %) | 98.17 % (0.09 %) | 97.76 % (0.10 %) | 96.42 % (0.13 %) | -| info | 22.3249 (0.0128) | 24.7278 (0.0046) | 24.8919 (0.0029) | 24.8961 (0.0027) | -| | 09.81 % (0.21 %) | 80.54 % (0.28 %) | 91.67 % (0.20 %) | 91.90 % (0.19 %) | +| cheat | 24.8594 ± 0.0036 | 24.9785 ± 0.0012 | 24.9720 ± 0.0014 | 24.9557 ± 0.0018 | +| | 90.59 ± 0.21 % | 98.17 ± 0.09 % | 97.76 ± 0.10 % | 96.42 ± 0.13 % | +| info | 22.3249 ± 0.0128 | 24.7278 ± 0.0046 | 24.8919 ± 0.0029 | 24.8961 ± 0.0027 | +| | 09.81 ± 0.21 % | 80.54 ± 0.28 % | 91.67 ± 0.20 % | 91.90 ± 0.19 % | diff --git a/src/main.rs b/src/main.rs index e044557..fbaef54 100644 --- a/src/main.rs +++ b/src/main.rs @@ -165,11 +165,11 @@ fn get_results_table() -> String { let n_trials = 20000; let n_threads = 8; - let intro = format!("On the first {} seeds, we have these average scores and win rates:\n\n", n_trials); + let intro = format!("On the first {} seeds, we have these scores and win rates (average ± standard error):\n\n", n_trials); let format_name = |x| format!(" {:7} ", x); - let format_players = |x| format!(" {}p (stderr) ", x); - let format_percent = |x, stderr| format!(" {:05.2} % ({:.2} %) ", x, stderr); - let format_score = |x, stderr| format!(" {:07.4} ({:.4}) ", x, stderr); + let format_players = |x| format!(" {}p ", x); + let format_percent = |x, stderr| format!(" {:05.2} ± {:.2} % ", x, stderr); + let format_score = |x, stderr| format!(" {:07.4} ± {:.4} ", x, stderr); let space = String::from(" "); let dashes = String::from("---------"); let dashes_long = String::from("------------------");