Report standard errors in README.md

README.md

@@ -69,9 +69,9 @@ time cargo run --release -- --write-results-table
 
 On the first 20000 seeds, we have these average scores and win rates:
 
-|         |   2p    |   3p    |   4p    |   5p    |
+|         |     2p  (stderr) |     3p  (stderr) |     4p  (stderr) |     5p  (stderr) |
-|---------|---------|---------|---------|---------|
+|---------|------------------|------------------|------------------|------------------|
-| cheat   | 24.8594 | 24.9785 | 24.9720 | 24.9557 |
+| cheat   | 24.8594 (0.0036) | 24.9785 (0.0012) | 24.9720 (0.0014) | 24.9557 (0.0018) |
-|         | 90.59 % | 98.17 % | 97.76 % | 96.42 % |
+|         | 90.59 % (0.21 %) | 98.17 % (0.09 %) | 97.76 % (0.10 %) | 96.42 % (0.13 %) |
-| info    | 22.3249 | 24.7278 | 24.8919 | 24.8961 |
+| info    | 22.3249 (0.0128) | 24.7278 (0.0046) | 24.8919 (0.0029) | 24.8961 (0.0027) |
-|         | 09.81 % | 80.54 % | 91.67 % | 91.90 % |
+|         | 09.81 % (0.21 %) | 80.54 % (0.28 %) | 91.67 % (0.20 %) | 91.90 % (0.19 %) |

src/main.rs

@@ -166,12 +166,13 @@ fn get_results_table() -> String {
     let n_threads = 8;
 
     let intro = format!("On the first {} seeds, we have these average scores and win rates:\n\n", n_trials);
-    let format_name    = |x|  format!(" {:7} ",      x);
+    let format_name    = |x|         format!(" {:7} ",      x);
-    let format_players = |x|  format!("   {}p    ",  x);
+    let format_players = |x|         format!("     {}p  (stderr) ",  x);
-    let format_percent = |x|  format!(" {:05.2} % ", x);
+    let format_percent = |x, stderr| format!(" {:05.2} % ({:.2} %) ", x, stderr);
-    let format_score   = |x|  format!(" {:07.4} ",   x);
+    let format_score   = |x, stderr| format!(" {:07.4} ({:.4}) ", x, stderr);
-    let space          = String::from("         ");
+    let space          =        String::from("         ");
-    let dashes         = String::from("---------");
+    let dashes         =        String::from("---------");
+    let dashes_long    =        String::from("------------------");
     type TwoLines = (String, String);
     fn make_twolines(player_nums: &Vec<u32>, head: TwoLines, make_block: &dyn Fn(u32) -> TwoLines) -> TwoLines {
         let mut blocks = player_nums.iter().cloned().map(make_block).collect::<Vec<_>>();
@@ -185,11 +186,16 @@ fn get_results_table() -> String {
     fn concat_twolines(body: Vec<TwoLines>) -> String {
         body.into_iter().fold(String::default(), |output, (a, b)| (output + &a + "\n" + &b + "\n"))
     }
-    let header = make_twolines(&player_nums, (space.clone(), dashes.clone()), &|n_players| (format_players(n_players), dashes.clone()));
+    let header = make_twolines(&player_nums,
+                               (space.clone(), dashes.clone()),
+                               &|n_players| (format_players(n_players), dashes_long.clone()));
     let mut body = strategies.iter().map(|strategy| {
         make_twolines(&player_nums, (format_name(strategy), space.clone()), &|n_players| {
             let simresult = sim_games(n_players, strategy, Some(seed), n_trials, n_threads, None);
-            (format_score(simresult.average_score()), format_percent(simresult.percent_perfect()))
+            (
+                format_score(simresult.average_score(), simresult.score_stderr()),
+                format_percent(simresult.percent_perfect(), simresult.percent_perfect_stderr())
+            )
         })
     }).collect::<Vec<_>>();
     body.insert(0, header);

src/simulator.rs

@@ -97,6 +97,17 @@ impl Histogram {
     pub fn average(&self) -> f32 {
         (self.sum as f32) / (self.total_count as f32)
     }
+    pub fn stdev_of_average(&self) -> f32 {
+        let average = self.average();
+        let mut var_sum = 0.0;
+        for (&val, &count) in self.hist.iter() {
+            var_sum += (val as f32 - average).powi(2) * count as f32;
+        }
+        // Divide by (self.total_count - 1) estimate the variance of the distribution,
+        // then divide by self.total_count estimate the variance of the sample average,
+        // then take the sqrt to get the stdev.
+        (var_sum / (((self.total_count - 1) * self.total_count) as f32)).sqrt()
+    }
     pub fn merge(&mut self, other: Histogram) {
         for (val, count) in other.hist.into_iter() {
             self.insert_many(val, count);
@@ -196,10 +207,20 @@ impl SimResult {
         self.scores.percentage_with(&PERFECT_SCORE) * 100.0
     }
 
+    pub fn percent_perfect_stderr(&self) -> f32 {
+        let pp = self.percent_perfect() / 100.0;
+        let stdev = (pp*(1.0 - pp) / ((self.scores.total_count - 1) as f32)).sqrt();
+        stdev * 100.0
+    }
+
     pub fn average_score(&self) -> f32 {
         self.scores.average()
     }
 
+    pub fn score_stderr(&self) -> f32 {
+        self.scores.stdev_of_average()
+    }
+
     pub fn average_lives(&self) -> f32 {
         self.lives.average()
     }