Merge branch 'main' of https://git.abstractnonsen.se/Hanabi/hanabi-league into minimize-loss

results/min_loss_rating_list.json

@@ -0,0 +1,468 @@
+{
+  "players": [
+    {
+      "id": 1,
+      "rating": 1656.9933584592068,
+      "username": "asaelr",
+      "stdev": 176.95337811507542
+    },
+    {
+      "id": 6,
+      "rating": 1547.1931967851367,
+      "username": "ElenaDhynho",
+      "stdev": 42.422674257282964
+    },
+    {
+      "id": 7,
+      "rating": 1596.6512180445707,
+      "username": "Eliclax",
+      "stdev": 25.554046049882707
+    },
+    {
+      "id": 8,
+      "rating": 1687.0065708056572,
+      "username": "Fafrd",
+      "stdev": 26.138849069401964
+    },
+    {
+      "id": 9,
+      "rating": 1495.04192425988,
+      "username": "Feich59",
+      "stdev": 73.58522619789503
+    },
+    {
+      "id": 10,
+      "rating": 1613.9053316660961,
+      "username": "gw12346",
+      "stdev": 15.446846483972713
+    },
+    {
+      "id": 11,
+      "rating": 1534.2115497751606,
+      "username": "hakha",
+      "stdev": 17.89293430394373
+    },
+    {
+      "id": 12,
+      "rating": 1702.8665184294564,
+      "username": "Helana",
+      "stdev": 17.959192573261596
+    },
+    {
+      "id": 13,
+      "rating": 1592.3685480643976,
+      "username": "hnter",
+      "stdev": 109.75405196408813
+    },
+    {
+      "id": 14,
+      "rating": 1684.5329659244817,
+      "username": "inseres",
+      "stdev": 5.193191918103997
+    },
+    {
+      "id": 15,
+      "rating": 1678.1068142163554,
+      "username": "Jay",
+      "stdev": 11.220557444400496
+    },
+    {
+      "id": 16,
+      "rating": 1657.7422053946832,
+      "username": "kimbifille",
+      "stdev": 71.62732692961684
+    },
+    {
+      "id": 17,
+      "rating": 1571.2263147756535,
+      "username": "Kowalski1337",
+      "stdev": 59.88063309820496
+    },
+    {
+      "id": 18,
+      "rating": 1673.652278716787,
+      "username": "macanek",
+      "stdev": 37.248874749072684
+    },
+    {
+      "id": 19,
+      "rating": 1734.2840478574078,
+      "username": "MarkusKahlsen",
+      "stdev": 137.263308306954
+    },
+    {
+      "id": 20,
+      "rating": 1599.1013632247043,
+      "username": "Neema",
+      "stdev": 29.99279133170806
+    },
+    {
+      "id": 21,
+      "rating": 1535.5621323285413,
+      "username": "newduke",
+      "stdev": 28.69029514671429
+    },
+    {
+      "id": 22,
+      "rating": 1439.1271150938064,
+      "username": "NishaNoire",
+      "stdev": 135.00876312408192
+    },
+    {
+      "id": 25,
+      "rating": 1589.9273430803019,
+      "username": "omegaxis",
+      "stdev": 45.681434487254656
+    },
+    {
+      "id": 27,
+      "rating": 1719.856499810081,
+      "username": "pianoblook",
+      "stdev": 43.91423347247681
+    },
+    {
+      "id": 28,
+      "rating": 1645.6070307223388,
+      "username": "posij118",
+      "stdev": 69.56052128100919
+    },
+    {
+      "id": 29,
+      "rating": 1700.7312163612453,
+      "username": "purplejoe",
+      "stdev": 4.333849603201881
+    },
+    {
+      "id": 31,
+      "rating": 1754.8673876868077,
+      "username": "rahsosprout",
+      "stdev": 24.20149808460386
+    },
+    {
+      "id": 32,
+      "rating": 1749.8186567295743,
+      "username": "Ramanujan",
+      "stdev": 35.186989633353924
+    },
+    {
+      "id": 33,
+      "rating": 1505.8114349633183,
+      "username": "ricardodd",
+      "stdev": 63.063499555331056
+    },
+    {
+      "id": 34,
+      "rating": 1424.7454795953922,
+      "username": "RIMBarisax",
+      "stdev": 54.62925500990815
+    },
+    {
+      "id": 35,
+      "rating": 1709.7754274336994,
+      "username": "rz",
+      "stdev": 34.75350346761236
+    },
+    {
+      "id": 36,
+      "rating": 1645.505505262562,
+      "username": "Sagnik Saha",
+      "stdev": 82.56415392788715
+    },
+    {
+      "id": 37,
+      "rating": 1607.8618746654392,
+      "username": "sodiumdebt",
+      "stdev": 74.74531010470325
+    },
+    {
+      "id": 38,
+      "rating": 1799.3860289683867,
+      "username": "spring",
+      "stdev": 6.935110645012115
+    },
+    {
+      "id": 39,
+      "rating": 1547.0924841832866,
+      "username": "StKildaFan",
+      "stdev": 22.21102514134186
+    },
+    {
+      "id": 40,
+      "rating": 1676.6787745059055,
+      "username": "str8tsknacker",
+      "stdev": 50.74354553034183
+    },
+    {
+      "id": 41,
+      "rating": 1731.4770087590437,
+      "username": "Sturm",
+      "stdev": 57.142346148910896
+    },
+    {
+      "id": 42,
+      "rating": 1501.164246026532,
+      "username": "TimeHoodie",
+      "stdev": 35.10667883309847
+    },
+    {
+      "id": 43,
+      "rating": 1655.211023199,
+      "username": "vEnhance",
+      "stdev": 42.389156840017165
+    },
+    {
+      "id": 44,
+      "rating": 1533.6010216854027,
+      "username": "vermling",
+      "stdev": 90.93922443450435
+    },
+    {
+      "id": 45,
+      "rating": 1586.770150886957,
+      "username": "wateroffire",
+      "stdev": 3.9655080603422466
+    },
+    {
+      "id": 46,
+      "rating": 1614.507394752425,
+      "username": "WillFlame",
+      "stdev": 9.831543835666807
+    },
+    {
+      "id": 47,
+      "rating": 1713.6463481565615,
+      "username": "Yagami Black",
+      "stdev": 25.99563483333066
+    },
+    {
+      "id": 48,
+      "rating": 1457.1982216219058,
+      "username": "youisme",
+      "stdev": 81.81998675205648
+    },
+    {
+      "id": 51,
+      "rating": 1437.5089899938134,
+      "username": "Libster",
+      "stdev": 8.378590826005142
+    },
+    {
+      "id": 52,
+      "rating": 1672.8512878685628,
+      "username": "maxeymo",
+      "stdev": 36.0712267697332
+    },
+    {
+      "id": 53,
+      "rating": 1642.5304416034514,
+      "username": "joano580",
+      "stdev": 100.76861593939701
+    },
+    {
+      "id": 54,
+      "rating": 1550.2628143305851,
+      "username": "ReaverSe",
+      "stdev": 17.2818922023245
+    },
+    {
+      "id": 55,
+      "rating": 1642.9660148054686,
+      "username": "benzloeb",
+      "stdev": 3.4106951597543786
+    },
+    {
+      "id": 56,
+      "rating": 1598.4171994249805,
+      "username": "percolate",
+      "stdev": 30.720436445911385
+    },
+    {
+      "id": 58,
+      "rating": 1464.6446630379342,
+      "username": "Random Guy JCI",
+      "stdev": 12.310038039511529
+    },
+    {
+      "id": 59,
+      "rating": 1606.062637102755,
+      "username": "aara",
+      "stdev": 21.876906585784877
+    },
+    {
+      "id": 60,
+      "rating": 1601.4626434355132,
+      "username": "amattias",
+      "stdev": 33.079466261234664
+    },
+    {
+      "id": 64,
+      "rating": 1601.5458400329035,
+      "username": "wtfitsnotbutter",
+      "stdev": 53.56517099030838
+    },
+    {
+      "id": 65,
+      "rating": 1480.6158683098095,
+      "username": "Alix_Eisenhardt",
+      "stdev": 50.30711678640308
+    },
+    {
+      "id": 67,
+      "rating": 1617.5730419198144,
+      "username": "Vivarus",
+      "stdev": 9.56814210631875
+    },
+    {
+      "id": 69,
+      "rating": 1543.1725821109499,
+      "username": "FrozenStella",
+      "stdev": 47.349220108955066
+    },
+    {
+      "id": 70,
+      "rating": 1589.17602492783,
+      "username": "GameConqueror",
+      "stdev": 34.77764553932631
+    },
+    {
+      "id": 71,
+      "rating": 1631.0886330291135,
+      "username": "seungapark",
+      "stdev": 36.35644480232967
+    },
+    {
+      "id": 72,
+      "rating": 1544.058017455111,
+      "username": "sinuni_hung",
+      "stdev": 8.068740309638756
+    },
+    {
+      "id": 73,
+      "rating": 1652.5253246400134,
+      "username": "TheDaniMan",
+      "stdev": 43.89447022365573
+    },
+    {
+      "id": 79,
+      "rating": 1487.3404436278843,
+      "username": "Kaznad",
+      "stdev": 15.830803035745209
+    },
+    {
+      "id": 80,
+      "rating": 1392.331302957674,
+      "username": "Kernel",
+      "stdev": 42.02247099621528
+    },
+    {
+      "id": 81,
+      "rating": 1428.4800076632162,
+      "username": "Le Codex",
+      "stdev": 43.789622164916196
+    },
+    {
+      "id": 82,
+      "rating": 1521.363531664333,
+      "username": "Nipalup",
+      "stdev": 104.0370045655497
+    }
+  ],
+  "variants": [
+    {
+      "id": 0,
+      "rating": 1359.6612370979074,
+      "num_players": 3,
+      "name": "No Variant",
+      "num_suits": 5,
+      "stdev": 37.51830058593259
+    },
+    {
+      "id": 1,
+      "rating": 1409.8285544138594,
+      "num_players": 3,
+      "name": "6 Suits",
+      "num_suits": 6,
+      "stdev": 56.644590533491034
+    },
+    {
+      "id": 51,
+      "rating": 1719.7607226091918,
+      "num_players": 3,
+      "name": "Clue Starved (6 Suits)",
+      "num_suits": 6,
+      "stdev": 8.202101579431847
+    },
+    {
+      "id": 52,
+      "rating": 1662.868864963005,
+      "num_players": 3,
+      "name": "Clue Starved (5 Suits)",
+      "num_suits": 5,
+      "stdev": 8.674234234615462
+    },
+    {
+      "id": 0,
+      "rating": 1439.4451347766544,
+      "num_players": 4,
+      "name": "No Variant",
+      "num_suits": 5,
+      "stdev": 2.016825515539241
+    },
+    {
+      "id": 1,
+      "rating": 1434.2720502574575,
+      "num_players": 4,
+      "name": "6 Suits",
+      "num_suits": 6,
+      "stdev": 29.836778320728673
+    },
+    {
+      "id": 51,
+      "rating": 1779.845972756269,
+      "num_players": 4,
+      "name": "Clue Starved (6 Suits)",
+      "num_suits": 6,
+      "stdev": 47.696178458669856
+    },
+    {
+      "id": 52,
+      "rating": 1701.8431047063805,
+      "num_players": 4,
+      "name": "Clue Starved (5 Suits)",
+      "num_suits": 5,
+      "stdev": 42.02213931310847
+    },
+    {
+      "id": 0,
+      "rating": 1426.7937800667978,
+      "num_players": 5,
+      "name": "No Variant",
+      "num_suits": 5,
+      "stdev": 54.921511042199526
+    },
+    {
+      "id": 1,
+      "rating": 1495.2514337712873,
+      "num_players": 5,
+      "name": "6 Suits",
+      "num_suits": 6,
+      "stdev": 43.16247744670021
+    },
+    {
+      "id": 51,
+      "rating": 1898.843058275536,
+      "num_players": 5,
+      "name": null,
+      "num_suits": 6,
+      "stdev": 22.62832823639932
+    },
+    {
+      "id": 52,
+      "rating": 1916.5106100141977,
+      "num_players": 5,
+      "name": "Clue Starved (5 Suits)",
+      "num_suits": 5,
+      "stdev": 37.61403719598357
+    }
+  ]
+}

src/minimize_loss.py

@@ -0,0 +1,406 @@
+from typing import Iterable, List, Dict
+
+from config import config_manager
+from constants import UNWINNABLE_SEED_FRACTION
+
+from dataclasses import dataclass
+
+import json
+import datetime
+
+import numpy as np
+from numpy.typing import NDArray
+from scipy.optimize import minimize, Bounds
+
+MAX_RATING_DIFF = 1200
+RATING_PRIOR = 1600
+# This allows us to do integer optimization on more fine-grained grid.
+LCM_FACTOR = 60
+NUM_RANDOM_INITS = 50
+NUM_CV_ITERS = 50
+
+VARIANT_DEFAULT_RATINGS = np.array([1400, 1400, 1800, 1800]).reshape((1, 4))
+VARIANT_NUM_PLAYERS_MODIFIERS = np.array([0, 0, 100]).reshape((3, 1))
+
+# As of season 1, this is [1400 1400 1800 1800 1400 1400 1800 1800 1500 1500 1900 1900].
+VARIANT_RATING_PRIORS = np.ravel(
+    VARIANT_DEFAULT_RATINGS + VARIANT_NUM_PLAYERS_MODIFIERS
+)
+
+
+@dataclass
+class GlobalInfo:
+    rated_ids: Iterable[int]
+    variant_ids: Iterable[int]
+    player_counts: Iterable[int]
+    user_names: Iterable[str]
+    game_counts: Iterable[int]
+    rated_id_indices_in_rating_list: Dict[int, int]
+
+    @property
+    def rating_list_length(self):
+        return len(self.rated_ids) + len(self.variant_ids) * len(self.player_counts)
+
+
+@dataclass
+class GameRow:
+    game_id: int
+    num_players: int
+    users: List[str]
+    user_ids: List[int]
+    user_rating_changes: List[float]
+    user_ratings_after: List[float]
+    variant_id: int
+    variant_name: str
+    num_suits: id
+    rating_type: int
+    seed: str
+    score: int
+    num_turns: int
+    datetime_finished: datetime.datetime
+    league_id: int
+    num_bdrs: int
+    num_crits_lost: int
+    game_outcomes: List[str]
+    variant_rating_change: float
+    variant_rating_after: float
+
+
+def calculate_loss(
+    rating_list: Iterable[int],
+    game_list: Iterable[GameRow],
+    global_info: GlobalInfo,
+    p_win_lookup_table: NDArray[np.float32],
+    player_rating_priors: Iterable[int],
+    player_prior_weight: float,
+    variant_rating_priors: Iterable[int],
+    variant_prior_weight: float,
+):
+    loss = np.float32(0)
+
+    for game in game_list:
+        team_ratings = [
+            rating_list[global_info.rated_id_indices_in_rating_list[user_id]]
+            for user_id in game["user_ids"]
+        ]
+        team_rating = sum(team_ratings) / len(team_ratings)
+
+        variant_rating = rating_list[
+            calculate_variant_index_in_rating_list(
+                len(game["users"]), game["variant_id"], global_info
+            )
+        ]
+
+        rating_diff = int(np.round(LCM_FACTOR * (team_rating - variant_rating)))
+        p_win = calculate_p_win(rating_diff, p_win_lookup_table)
+
+        game_won = game["game_outcomes"].count("Win") > 0
+        loss += calculate_loss_for_one_game(p_win, game_won)
+
+    # Add two dummy single-player games for each player with prior_weight * square-root of number of games weight - one won and lost
+    for player_rating_prior, rated_id, game_count in zip(
+        player_rating_priors, global_info.rated_ids, global_info.game_counts
+    ):
+        team_rating = rating_list[global_info.rated_id_indices_in_rating_list[rated_id]]
+        variant_rating = player_rating_prior
+
+        rating_diff = int(np.round(LCM_FACTOR * (team_rating - variant_rating)))
+        p_win = calculate_p_win(rating_diff, p_win_lookup_table)
+
+        loss += (
+            np.sqrt(game_count)
+            * player_prior_weight
+            * calculate_loss_for_one_game(p_win, True)
+        )
+        loss += (
+            np.sqrt(game_count)
+            * player_prior_weight
+            * calculate_loss_for_one_game(p_win, False)
+        )
+
+    # Add two dummy single-player games for each player with prior_weight * square-root of number of games weight - one won and lost
+    for variant_rating, variant_rating_prior in zip(
+        rating_list[len(global_info.rated_ids) :],
+        variant_rating_priors,
+    ):
+        team_rating = variant_rating_prior
+
+        rating_diff = int(np.round(LCM_FACTOR * (team_rating - variant_rating)))
+        p_win = calculate_p_win(rating_diff, p_win_lookup_table)
+
+        loss += (
+            np.sqrt(game_count)
+            * variant_prior_weight
+            * calculate_loss_for_one_game(p_win, True)
+        )
+        loss += (
+            np.sqrt(game_count)
+            * variant_prior_weight
+            * calculate_loss_for_one_game(p_win, False)
+        )
+
+    return loss
+
+
+def calculate_loss_gradient(rating_list, *args):
+    y = calculate_loss(rating_list, *args)
+    eye = np.eye(len(rating_list), dtype=int)
+
+    gradient = [
+        calculate_loss(rating_list + eye[i], *args) - y for i in range(len(rating_list))
+    ]
+
+    return gradient
+
+
+def calculate_minloss_ratings(
+    game_list: List[GameRow],
+    global_info: GlobalInfo,
+    p_win_lookup_table: NDArray[np.float32],
+    player_rating_priors: int | Iterable[int] = RATING_PRIOR,
+    player_random_init_stdev: int = 0,
+    player_prior_weight=0.5,
+    variant_rating_priors: int | Iterable[int] = RATING_PRIOR,
+    variant_prior_weight=0.5,
+    variant_random_init_stdev: int = 0,
+):
+    player_rating_priors = np.broadcast_to(
+        player_rating_priors, len(global_info.rated_ids)
+    )
+    variant_rating_priors = np.broadcast_to(
+        variant_rating_priors,
+        len(global_info.variant_ids) * len(global_info.player_counts),
+    )
+
+    prior_rating_list = np.concatenate(
+        (
+            player_rating_priors
+            + np.clip(
+                np.random.normal(
+                    0, player_random_init_stdev, len(player_rating_priors)
+                ),
+                -3 * player_random_init_stdev,
+                3 * player_random_init_stdev,
+            ),
+            variant_rating_priors
+            + np.clip(
+                np.random.normal(
+                    0, variant_random_init_stdev, len(variant_rating_priors)
+                ),
+                -3 * variant_random_init_stdev,
+                3 * variant_random_init_stdev,
+            ),
+        ),
+        dtype=np.float32,
+    )
+
+    rating_list = minimize(
+        calculate_loss,
+        prior_rating_list,
+        (
+            game_list,
+            global_info,
+            p_win_lookup_table,
+            player_rating_priors,
+            player_prior_weight,
+            variant_rating_priors,
+            variant_prior_weight,
+        ),
+        bounds=Bounds(RATING_PRIOR - MAX_RATING_DIFF, RATING_PRIOR + MAX_RATING_DIFF),
+        jac=calculate_loss_gradient,
+        tol=0.0001,
+    )
+
+    return rating_list
+
+
+def calculate_p_win_lookup_table(rating_diff_indices: Iterable[int]):
+    lookup_table = np.full(2 * MAX_RATING_DIFF * LCM_FACTOR + 1, 0.5, dtype=np.float32)
+
+    for rating_diff_index in rating_diff_indices:
+        p_win = calculate_p_win(rating_diff_index, None)
+        lookup_table[rating_diff_index] = p_win
+
+    return lookup_table
+
+
+def calculate_p_win(
+    rating_diff_index: int, lookup_table: NDArray[np.float32] | None = None
+):
+    if lookup_table is None:
+        return (1 - UNWINNABLE_SEED_FRACTION) * (
+            1 / (1 + 10 ** (-(rating_diff_index / LCM_FACTOR) / 400))
+        )
+    else:
+        return lookup_table[rating_diff_index]
+
+
+# Cross-entropy loss
+def calculate_loss_for_one_game(p_win: np.float32, game_won: bool):
+    if game_won:
+        loss = -np.log(p_win)
+    else:
+        loss = -np.log(1 - p_win)
+
+    return loss
+
+
+def calculate_variant_index_in_rating_list(
+    player_count: int, variant_id: int, global_info: GlobalInfo
+):
+    return (
+        len(global_info.rated_ids)
+        + global_info.player_counts.index(player_count) * len(global_info.variant_ids)
+        + global_info.variant_ids.index(variant_id)
+    )
+
+
+def calculate_player_count_and_variant_id_from_variant_index(
+    index: int, global_info: GlobalInfo
+):
+    num_players = global_info.player_counts[index // len(global_info.variant_ids)]
+    variant_id = global_info.variant_ids[index % len(global_info.variant_ids)]
+
+    return num_players, variant_id
+
+
+def write_data(random_init_rating_lists, cv_rating_lists):
+    with open("../results/min_loss_rating_list.json", "w") as f:
+        rating_list = np.average(random_init_rating_lists, axis=0)
+        rating_stdevs = np.std(cv_rating_lists, axis=0, ddof=1)
+
+        players = []
+        for rated_id, user_name, rating, rating_stdev in zip(
+            global_info.rated_ids, global_info.user_names, rating_list, rating_stdevs
+        ):
+            players.append(
+                {
+                    "id": rated_id,
+                    "rating": rating,
+                    "username": user_name,
+                    "stdev": rating_stdev,
+                }
+            )
+
+        variants = []
+
+        for i, (rating, rating_stdev) in enumerate(
+            zip(
+                rating_list[len(global_info.rated_ids) :],
+                rating_stdevs[len(global_info.rated_ids) :],
+            )
+        ):
+            (
+                num_players,
+                variant_id,
+            ) = calculate_player_count_and_variant_id_from_variant_index(i, global_info)
+            variant_name = None
+
+            for game in game_list:
+                if (
+                    len(game["users"]) == num_players
+                    and game["variant_id"] == variant_id
+                ):
+                    variant_name = game["variant_name"]
+                    num_suits = game["num_suits"]
+
+            variants.append(
+                {
+                    "id": variant_id,
+                    "rating": rating,
+                    "num_players": num_players,
+                    "name": variant_name,
+                    "num_suits": num_suits,
+                    "stdev": rating_stdev,
+                }
+            )
+
+        s = json.dumps({"players": players, "variants": variants})
+        f.write(s)
+
+
+def read_data():
+    with open("../data/games.json") as game_data:
+        game_list = json.loads(game_data.read())
+
+    p_win_lookup_table = calculate_p_win_lookup_table(
+        np.arange(-MAX_RATING_DIFF * LCM_FACTOR, MAX_RATING_DIFF * LCM_FACTOR + 1)
+    )
+
+    config = config_manager.get_config()
+
+    rated_ids_dupes = [ID for game in game_list for ID in game["user_ids"]]
+    rated_ids = list(set(rated_ids_dupes))
+
+    variant_ids_dupes = [game["variant_id"] for game in game_list]
+    variant_ids = list(set(variant_ids_dupes))
+
+    nums_players = [game["num_players"] for game in game_list]
+
+    player_counts = list(range(config.min_player_count, config.max_player_count + 1))
+    game_counts = np.concatenate(
+        (
+            [rated_ids_dupes.count(ID) for ID in rated_ids],
+            np.zeros(len(variant_ids) * len(player_counts)),
+        ),
+    )
+
+    user_names = list(np.full((len(rated_ids),), ""))
+    for game in game_list:
+        for user_id, user_name in zip(game["user_ids"], game["users"]):
+            user_names[rated_ids.index(user_id)] = user_name
+
+    rated_id_indices_in_rating_list = {
+        id: index for index, id in dict(enumerate(rated_ids)).items()
+    }
+
+    global_info = GlobalInfo(
+        rated_ids,
+        variant_ids,
+        player_counts,
+        user_names,
+        game_counts,
+        rated_id_indices_in_rating_list,
+    )
+
+    for variant_id, num_players in zip(variant_ids_dupes, nums_players):
+        game_counts[
+            calculate_variant_index_in_rating_list(num_players, variant_id, global_info)
+        ] += 1
+    global_info.game_counts = game_counts
+
+    return game_list, global_info, p_win_lookup_table
+
+
+if __name__ == "__main__":
+    game_list, global_info, p_win_lookup_table = read_data()
+
+    random_init_rating_lists = []
+    cv_rating_lists = []
+
+    for _ in range(NUM_RANDOM_INITS):
+        random_init_rating_lists.append(
+            calculate_minloss_ratings(
+                game_list,
+                global_info,
+                p_win_lookup_table,
+                variant_rating_priors=VARIANT_RATING_PRIORS,
+                player_random_init_stdev=200,
+                variant_random_init_stdev=200,
+            ).x
+        )
+
+    for _ in range(NUM_CV_ITERS):
+        cv_game_list = np.random.choice(game_list, len(game_list), replace=True)
+        cv_rating_lists.append(
+            calculate_minloss_ratings(
+                cv_game_list,
+                global_info,
+                p_win_lookup_table,
+                variant_rating_priors=VARIANT_RATING_PRIORS,
+                player_random_init_stdev=200,
+                variant_random_init_stdev=200,
+            ).x
+        )
+
+    write_data(random_init_rating_lists, cv_rating_lists)