Merge pull request #9 from felixbauckholt/determinism

Determinism

Cargo.lock

@@ -3,6 +3,11 @@ name = "crossbeam"
 version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "fnv"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "getopts"
 version = "0.2.14"
@@ -34,6 +39,7 @@ name = "rust_hanabi"
 version = "0.1.0"
 dependencies = [
  "crossbeam 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -41,6 +47,7 @@ dependencies = [
 
 [metadata]
 "checksum crossbeam 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "348228ce9f93d20ffc30c18e575f82fa41b9c8bf064806c65d41eba4771595a0"
+"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
 "checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685"
 "checksum libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "4870ef6725dde13394134e587e4ab4eca13cb92e916209a31c851b49131d3c75"
 "checksum log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "038b5d13189a14e5b6ac384fdb7c691a45ef0885f6d2dddbf422e6c3506b8234"

Cargo.toml

@@ -7,4 +7,5 @@ authors = ["Jeff Wu <wuthefwasthat@gmail.com>"]
 rand = "*"
 log = "*"
 getopts = "*"
+fnv = "*"
 crossbeam = "0.2.5"

README.md

@@ -55,25 +55,23 @@ Some examples:
 - [A cheating strategy](src/strategies/cheating.rs), using `Rc<RefCell<_>>`
 - [The information strategy](src/strategies/information.rs)!
 
-## Results
+## Results (auto-generated)
-
-On seeds 0-9999, we have these average scores and win rates:
-
-|       |   2p    |   3p    |   4p    |   5p    |
-|-------|---------|---------|---------|---------|
-|cheat  | 24.8600 | 24.9781 | 24.9715 | 24.9570 |
-|       | 90.52 % | 98.12 % | 97.74 % | 96.57 % |
-|info   | 22.3386 | 24.7322 | 24.8921 | 24.8996 |
-|       | 09.86 % | 80.75 % | 91.58 % | 92.11 % |
-
 
 To reproduce:
 ```
-n=10000   # number of rounds to simulate
+time cargo run --release -- --results-table
-t=4       # number of threads
-for strategy in info cheat; do
-  for p in $(seq 2 5); do
-    time cargo run --release -- -n $n -s 0 -t $t -p $p -g $strategy;
-  done
-done
 ```
+
+To update this file:
+```
+time cargo run --release -- --write-results-table
+```
+
+On the first 20000 seeds, we have these average scores and win rates:
+
+|         |   2p    |   3p    |   4p    |   5p    |
+|---------|---------|---------|---------|---------|
+| cheat   | 24.8594 | 24.9785 | 24.9720 | 24.9557 |
+|         | 90.59 % | 98.17 % | 97.76 % | 96.42 % |
+| info    | 22.3249 | 24.7278 | 24.8919 | 24.8961 |
+|         | 09.81 % | 80.54 % | 91.67 % | 91.90 % |

src/game.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use fnv::FnvHashMap;
 use std::fmt;
 use std::ops::Range;
 
@@ -46,11 +46,11 @@ impl fmt::Debug for Card {
 
 #[derive(Debug,Clone)]
 pub struct CardCounts {
-    counts: HashMap<Card, u32>,
+    counts: FnvHashMap<Card, u32>,
 }
 impl CardCounts {
     pub fn new() -> CardCounts {
-        let mut counts = HashMap::new();
+        let mut counts = FnvHashMap::default();
         for &color in COLORS.iter() {
             for &value in VALUES.iter() {
                 counts.insert(Card::new(color, value), 0);
@@ -248,7 +248,7 @@ pub struct BoardState {
     pub deck_size: u32,
     pub total_cards: u32,
     pub discard: Discard,
-    pub fireworks: HashMap<Color, Firework>,
+    pub fireworks: FnvHashMap<Color, Firework>,
 
     pub num_players: u32,
 
@@ -271,7 +271,7 @@ impl BoardState {
     pub fn new(opts: &GameOptions, deck_size: u32) -> BoardState {
         let fireworks = COLORS.iter().map(|&color| {
             (color, Firework::new(color))
-        }).collect::<HashMap<_, _>>();
+        }).collect::<FnvHashMap<_, _>>();
 
         BoardState {
             deck_size: deck_size,
@@ -479,7 +479,7 @@ pub struct BorrowedGameView<'a> {
     pub player: Player,
     pub hand_size: usize,
     // the cards of the other players, as well as the information they have
-    pub other_hands: HashMap<Player, &'a Cards>,
+    pub other_hands: FnvHashMap<Player, &'a Cards>,
     // board state
     pub board: &'a BoardState,
 }
@@ -506,7 +506,7 @@ pub struct OwnedGameView {
     pub player: Player,
     pub hand_size: usize,
     // the cards of the other players, as well as the information they have
-    pub other_hands: HashMap<Player, Cards>,
+    pub other_hands: FnvHashMap<Player, Cards>,
     // board state
     pub board: BoardState,
 }
@@ -515,7 +515,7 @@ impl OwnedGameView {
         let other_hands = borrowed_view.other_hands.iter()
             .map(|(&other_player, &player_state)| {
                 (other_player, player_state.clone())
-            }).collect::<HashMap<_, _>>();
+            }).collect::<FnvHashMap<_, _>>();
 
         OwnedGameView {
             player: borrowed_view.player.clone(),
@@ -544,7 +544,7 @@ impl GameView for OwnedGameView {
 // complete game state (known to nobody!)
 #[derive(Debug)]
 pub struct GameState {
-    pub hands: HashMap<Player, Cards>,
+    pub hands: FnvHashMap<Player, Cards>,
     pub board: BoardState,
     pub deck: Cards,
 }
@@ -582,7 +582,7 @@ impl GameState {
                     deck.pop().unwrap()
                 }).collect::<Vec<_>>();
                 (player, hand)
-            }).collect::<HashMap<_, _>>();
+            }).collect::<FnvHashMap<_, _>>();
 
         GameState {
             hands: hands,
@@ -605,7 +605,7 @@ impl GameState {
 
     // get the game state view of a particular player
     pub fn get_view(&self, player: Player) -> BorrowedGameView {
-        let mut other_hands = HashMap::new();
+        let mut other_hands = FnvHashMap::default();
         for (&other_player, hand) in &self.hands {
             if player != other_player {
                 other_hands.insert(other_player, hand);

src/main.rs

@@ -3,6 +3,7 @@ extern crate getopts;
 extern crate log;
 extern crate rand;
 extern crate crossbeam;
+extern crate fnv;
 
 mod helpers;
 mod game;
@@ -64,6 +65,10 @@ fn main() {
                 "STRATEGY");
     opts.optflag("h", "help",
                  "Print this help menu");
+    opts.optflag("", "results-table",
+                 "Print a table of results for each strategy");
+    opts.optflag("", "write-results-table",
+                 "Update the results table in README.md");
     let matches = match opts.parse(&args[1..]) {
         Ok(m) => { m }
         Err(f) => {
@@ -77,6 +82,12 @@ fn main() {
     if !matches.free.is_empty() {
         return print_usage(&program, opts);
     }
+    if matches.opt_present("write-results-table") {
+        return write_results_table();
+    }
+    if matches.opt_present("results-table") {
+        return print!("{}", get_results_table());
+    }
 
     let log_level_str : &str = &matches.opt_str("l").unwrap_or("info".to_string());
     let log_level = match log_level_str {
@@ -96,15 +107,18 @@ fn main() {
         Box::new(SimpleLogger)
     }).unwrap();
 
-    let n = u32::from_str(&matches.opt_str("n").unwrap_or("1".to_string())).unwrap();
+    let n_trials = u32::from_str(&matches.opt_str("n").unwrap_or("1".to_string())).unwrap();
-
     let seed = matches.opt_str("s").map(|seed_str| { u32::from_str(&seed_str).unwrap() });
-
     let progress_info = matches.opt_str("o").map(|freq_str| { u32::from_str(&freq_str).unwrap() });
-
     let n_threads = u32::from_str(&matches.opt_str("t").unwrap_or("1".to_string())).unwrap();
-
     let n_players = u32::from_str(&matches.opt_str("p").unwrap_or("4".to_string())).unwrap();
+    let strategy_str : &str = &matches.opt_str("g").unwrap_or("cheat".to_string());
+
+    sim_games(n_players, strategy_str, seed, n_trials, n_threads, progress_info).info();
+}
+
+fn sim_games(n_players: u32, strategy_str: &str, seed: Option<u32>, n_trials: u32, n_threads: u32, progress_info: Option<u32>)
+    -> simulator::SimResult {
     let hand_size = match n_players {
         2 => 5,
         3 => 5,
@@ -122,7 +136,6 @@ fn main() {
         allow_empty_hints: false,
     };
 
-    let strategy_str : &str = &matches.opt_str("g").unwrap_or("cheat".to_string());
     let strategy_config : Box<strategy::GameStrategyConfig + Sync> = match strategy_str {
         "random" => {
             Box::new(strategies::examples::RandomStrategyConfig {
@@ -139,9 +152,75 @@ fn main() {
                 as Box<strategy::GameStrategyConfig + Sync>
         },
         _ => {
-            print_usage(&program, opts);
             panic!("Unexpected strategy argument {}", strategy_str);
         },
     };
-    simulator::simulate(&game_opts, strategy_config, seed, n, n_threads, progress_info);
+    simulator::simulate(&game_opts, strategy_config, seed, n_trials, n_threads, progress_info)
+}
+
+fn get_results_table() -> String {
+    let strategies = ["cheat", "info"];
+    let player_nums = (2..=5).collect::<Vec<_>>();
+    let seed = 0;
+    let n_trials = 1;
+    let n_threads = 8;
+
+    let intro = format!("On the first {} seeds, we have these average scores and win rates:\n\n", n_trials);
+    let format_name    = |x|  format!(" {:7} ",      x);
+    let format_players = |x|  format!("   {}p    ",  x);
+    let format_percent = |x|  format!(" {:05.2} % ", x);
+    let format_score   = |x|  format!(" {:07.4} ",   x);
+    let space          = String::from("         ");
+    let dashes         = String::from("---------");
+    type TwoLines = (String, String);
+    fn make_twolines(player_nums: &Vec<u32>, head: TwoLines, make_block: &dyn Fn(u32) -> TwoLines) -> TwoLines {
+        let mut blocks = player_nums.iter().cloned().map(make_block).collect::<Vec<_>>();
+        blocks.insert(0, head);
+        fn combine(items: Vec<String>) -> String {
+            items.iter().fold(String::from("|"), |init, next| { init + next + "|" })
+        }
+        let (a, b): (Vec<_>, Vec<_>) = blocks.into_iter().unzip();
+        (combine(a), combine(b))
+    }
+    fn concat_twolines(body: Vec<TwoLines>) -> String {
+        body.into_iter().fold(String::default(), |output, (a, b)| (output + &a + "\n" + &b + "\n"))
+    }
+    let header = make_twolines(&player_nums, (space.clone(), dashes.clone()), &|n_players| (format_players(n_players), dashes.clone()));
+    let mut body = strategies.iter().map(|strategy| {
+        make_twolines(&player_nums, (format_name(strategy), space.clone()), &|n_players| {
+            let simresult = sim_games(n_players, strategy, Some(seed), n_trials, n_threads, None);
+            (format_score(simresult.average_score()), format_percent(simresult.percent_perfect()))
+        })
+    }).collect::<Vec<_>>();
+    body.insert(0, header);
+    intro + &concat_twolines(body)
+}
+
+fn write_results_table() {
+    let separator = r#"
+## Results (auto-generated)
+
+To reproduce:
+```
+time cargo run --release -- --results-table
+```
+
+To update this file:
+```
+time cargo run --release -- --write-results-table
+```
+
+"#;
+    let readme = "README.md";
+    let readme_contents = std::fs::read_to_string(readme).unwrap();
+    let readme_init = {
+        let parts = readme_contents.splitn(2, separator).collect::<Vec<_>>();
+        if parts.len() != 2 {
+            panic!("{} has been modified in the Results section!", readme);
+        }
+        parts[0]
+    };
+    let table = get_results_table();
+    let new_readme_contents = String::from(readme_init) + separator + &table;
+    std::fs::write(readme, new_readme_contents).unwrap();
 }

src/simulator.rs

@@ -1,5 +1,5 @@
 use rand::{self, Rng, SeedableRng};
-use std::collections::HashMap;
+use fnv::FnvHashMap;
 use std::fmt;
 use crossbeam;
 
@@ -25,17 +25,15 @@ fn new_deck(seed: u32) -> Cards {
 pub fn simulate_once(
         opts: &GameOptions,
         game_strategy: Box<GameStrategy>,
-        seed_opt: Option<u32>,
+        seed: u32,
     ) -> GameState {
-
-    let seed = seed_opt.unwrap_or(rand::thread_rng().next_u32());
     let deck = new_deck(seed);
 
     let mut game = GameState::new(opts, deck);
 
     let mut strategies = game.get_players().map(|player| {
         (player, game_strategy.initialize(player, &game.get_view(player)))
-    }).collect::<HashMap<Player, Box<PlayerStrategy>>>();
+    }).collect::<FnvHashMap<Player, Box<PlayerStrategy>>>();
 
     while !game.is_over() {
         let player = game.board.player;
@@ -68,15 +66,15 @@ pub fn simulate_once(
 }
 
 #[derive(Debug)]
-struct Histogram {
+pub struct Histogram {
-    pub hist: HashMap<Score, u32>,
+    pub hist: FnvHashMap<Score, u32>,
     pub sum: Score,
     pub total_count: u32,
 }
 impl Histogram {
     pub fn new() -> Histogram {
         Histogram {
-            hist: HashMap::new(),
+            hist: FnvHashMap::default(),
             sum: 0,
             total_count: 0,
         }
@@ -125,9 +123,10 @@ pub fn simulate<T: ?Sized>(
         n_trials: u32,
         n_threads: u32,
         progress_info: Option<u32>,
-    ) where T: GameStrategyConfig + Sync {
+    ) -> SimResult
+    where T: GameStrategyConfig + Sync {
 
-    let first_seed = first_seed_opt.unwrap_or(rand::thread_rng().next_u32());
+    let first_seed = first_seed_opt.unwrap_or_else(|| rand::thread_rng().next_u32());
 
     let strat_config_ref = &strat_config;
     crossbeam::scope(|scope| {
@@ -154,11 +153,11 @@ pub fn simulate<T: ?Sized>(
                             );
                         }
                     }
-                    let game = simulate_once(&opts, strat_config_ref.initialize(&opts), Some(seed));
+                    let game = simulate_once(&opts, strat_config_ref.initialize(&opts), seed);
                     let score = game.score();
                     lives_histogram.insert(game.board.lives_remaining);
                     score_histogram.insert(score);
-                    if score != PERFECT_SCORE { non_perfect_seeds.push((score, seed)); }
+                    if score != PERFECT_SCORE { non_perfect_seeds.push(seed); }
                 }
                 if progress_info.is_some() {
                     info!("Thread {} done", i);
@@ -167,7 +166,7 @@ pub fn simulate<T: ?Sized>(
             }));
         }
 
-        let mut non_perfect_seeds : Vec<(Score,u32)> = Vec::new();
+        let mut non_perfect_seeds : Vec<u32> = Vec::new();
         let mut score_histogram = Histogram::new();
         let mut lives_histogram = Histogram::new();
         for join_handle in join_handles {
@@ -177,17 +176,44 @@ pub fn simulate<T: ?Sized>(
             lives_histogram.merge(thread_lives_histogram);
         }
 
-        info!("Score histogram:\n{}", score_histogram);
-
         non_perfect_seeds.sort();
-        // info!("Seeds with non-perfect score: {:?}", non_perfect_seeds);
+        SimResult {
-        if non_perfect_seeds.len() > 0 {
+            scores: score_histogram,
-            info!("Example seed with non-perfect score: {}",
+            lives: lives_histogram,
-                  non_perfect_seeds.get(0).unwrap().1);
+            non_perfect_seed: non_perfect_seeds.get(0).cloned(),
         }
-
-        info!("Percentage perfect: {:?}%", score_histogram.percentage_with(&PERFECT_SCORE) * 100.0);
-        info!("Average score: {:?}", score_histogram.average());
-        info!("Average lives: {:?}", lives_histogram.average());
     })
 }
+
+pub struct SimResult {
+    pub scores: Histogram,
+    pub lives: Histogram,
+    pub non_perfect_seed: Option<u32>,
+}
+
+impl SimResult {
+    pub fn percent_perfect(&self) -> f32 {
+        self.scores.percentage_with(&PERFECT_SCORE) * 100.0
+    }
+
+    pub fn average_score(&self) -> f32 {
+        self.scores.average()
+    }
+
+    pub fn average_lives(&self) -> f32 {
+        self.lives.average()
+    }
+
+    pub fn info(&self) {
+        info!("Score histogram:\n{}", self.scores);
+
+        // info!("Seeds with non-perfect score: {:?}", non_perfect_seeds);
+        if let Some(seed) = self.non_perfect_seed {
+            info!("Example seed with non-perfect score: {}", seed);
+        }
+
+        info!("Percentage perfect: {:?}%", self.percent_perfect());
+        info!("Average score: {:?}", self.average_score());
+        info!("Average lives: {:?}", self.average_lives());
+    }
+}

src/strategies/cheating.rs

@@ -1,6 +1,6 @@
 use std::rc::Rc;
 use std::cell::{RefCell};
-use std::collections::{HashMap, HashSet};
+use fnv::{FnvHashMap, FnvHashSet};
 
 use strategy::*;
 use game::*;
@@ -31,13 +31,13 @@ impl GameStrategyConfig for CheatingStrategyConfig {
 }
 
 pub struct CheatingStrategy {
-    player_hands_cheat: Rc<RefCell<HashMap<Player, Cards>>>,
+    player_hands_cheat: Rc<RefCell<FnvHashMap<Player, Cards>>>,
 }
 
 impl CheatingStrategy {
     pub fn new() -> CheatingStrategy {
         CheatingStrategy {
-            player_hands_cheat: Rc::new(RefCell::new(HashMap::new())),
+            player_hands_cheat: Rc::new(RefCell::new(FnvHashMap::default())),
         }
     }
 }
@@ -56,7 +56,7 @@ impl GameStrategy for CheatingStrategy {
 }
 
 pub struct CheatingPlayerStrategy {
-    player_hands_cheat: Rc<RefCell<HashMap<Player, Cards>>>,
+    player_hands_cheat: Rc<RefCell<FnvHashMap<Player, Cards>>>,
     me: Player,
 }
 impl CheatingPlayerStrategy {
@@ -120,7 +120,7 @@ impl CheatingPlayerStrategy {
     }
 
     fn find_useless_card(&self, view: &BorrowedGameView, hand: &Cards) -> Option<usize> {
-        let mut set: HashSet<Card> = HashSet::new();
+        let mut set: FnvHashSet<Card> = FnvHashSet::default();
 
         for (i, card) in hand.iter().enumerate() {
             if view.board.is_dead(card) {

src/strategies/information.rs

@@ -1,4 +1,4 @@
-use std::collections::{HashMap, HashSet};
+use fnv::{FnvHashMap, FnvHashSet};
 use std::cmp::Ordering;
 
 use strategy::*;
@@ -185,14 +185,14 @@ impl Question for AdditiveComboQuestion {
 struct CardPossibilityPartition {
     index: usize,
     n_partitions: u32,
-    partition: HashMap<Card, u32>,
+    partition: FnvHashMap<Card, u32>,
 }
 impl CardPossibilityPartition {
     fn new(
         index: usize, max_n_partitions: u32, card_table: &CardPossibilityTable, view: &OwnedGameView
     ) -> CardPossibilityPartition {
         let mut cur_block = 0;
-        let mut partition = HashMap::new();
+        let mut partition = FnvHashMap::default();
         let mut n_partitions = 0;
 
         let has_dead = card_table.probability_is_dead(&view.board) != 0.0;
@@ -288,7 +288,7 @@ impl GameStrategy for InformationStrategy {
             view.board.get_players().map(|player| {
                 let hand_info = HandInfo::new(view.board.hand_size);
                 (player, hand_info)
-            }).collect::<HashMap<_,_>>();
+            }).collect::<FnvHashMap<_,_>>();
 
         Box::new(InformationPlayerStrategy {
             me: player,
@@ -301,7 +301,7 @@ impl GameStrategy for InformationStrategy {
 
 pub struct InformationPlayerStrategy {
     me: Player,
-    public_info: HashMap<Player, HandInfo<CardPossibilityTable>>,
+    public_info: FnvHashMap<Player, HandInfo<CardPossibilityTable>>,
     public_counts: CardCounts, // what any newly drawn card should be
     last_view: OwnedGameView, // the view on the previous turn
 }
@@ -550,8 +550,8 @@ impl InformationPlayerStrategy {
     }
 
     fn find_useless_cards(&self, view: &OwnedGameView, hand: &HandInfo<CardPossibilityTable>) -> Vec<usize> {
-        let mut useless: HashSet<usize> = HashSet::new();
+        let mut useless: FnvHashSet<usize> = FnvHashSet::default();
-        let mut seen: HashMap<Card, usize> = HashMap::new();
+        let mut seen: FnvHashMap<Card, usize> = FnvHashMap::default();
 
         for (i, card_table) in hand.iter().enumerate() {
             if card_table.probability_is_dead(view.get_board()) == 1.0 {
@@ -792,7 +792,7 @@ impl InformationPlayerStrategy {
         (0 .. n - 1).into_iter().map(|i| { (player + 1 + i) % n }).collect()
     }
 
-    fn get_best_hint_of_options(&self, hint_player: Player, hint_option_set: HashSet<Hinted>) -> Hinted {
+    fn get_best_hint_of_options(&self, hint_player: Player, hint_option_set: FnvHashSet<Hinted>) -> Hinted {
         let view = &self.last_view;
 
         // using hint goodness barely helps
@@ -864,7 +864,7 @@ impl InformationPlayerStrategy {
                 2 => {
                     // NOTE: this doesn't do that much better than just hinting
                     // the first thing that doesn't match the hint_card
-                    let mut hint_option_set = HashSet::new();
+                    let mut hint_option_set = FnvHashSet::default();
                     for card in hand {
                         if card.color != hint_card.color {
                             hint_option_set.insert(Hinted::Color(card.color));
@@ -889,7 +889,7 @@ impl InformationPlayerStrategy {
                 }
                 2 => {
                     // Any value hint for a card other than the first
-                    let mut hint_option_set = HashSet::new();
+                    let mut hint_option_set = FnvHashSet::default();
                     for card in hand {
                         if card.value != hint_card.value {
                             hint_option_set.insert(Hinted::Value(card.value));
@@ -899,7 +899,7 @@ impl InformationPlayerStrategy {
                 }
                 3 => {
                     // Any color hint for a card other than the first
-                    let mut hint_option_set = HashSet::new();
+                    let mut hint_option_set = FnvHashSet::default();
                     for card in hand {
                         if card.color != hint_card.color {
                             hint_option_set.insert(Hinted::Color(card.color));