Compare commits

...

7 commits

3 changed files with 26008 additions and 0 deletions

25134
data/games.json Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,468 @@
{
"players": [
{
"id": 1,
"rating": 1656.9933584592068,
"username": "asaelr",
"stdev": 176.95337811507542
},
{
"id": 6,
"rating": 1547.1931967851367,
"username": "ElenaDhynho",
"stdev": 42.422674257282964
},
{
"id": 7,
"rating": 1596.6512180445707,
"username": "Eliclax",
"stdev": 25.554046049882707
},
{
"id": 8,
"rating": 1687.0065708056572,
"username": "Fafrd",
"stdev": 26.138849069401964
},
{
"id": 9,
"rating": 1495.04192425988,
"username": "Feich59",
"stdev": 73.58522619789503
},
{
"id": 10,
"rating": 1613.9053316660961,
"username": "gw12346",
"stdev": 15.446846483972713
},
{
"id": 11,
"rating": 1534.2115497751606,
"username": "hakha",
"stdev": 17.89293430394373
},
{
"id": 12,
"rating": 1702.8665184294564,
"username": "Helana",
"stdev": 17.959192573261596
},
{
"id": 13,
"rating": 1592.3685480643976,
"username": "hnter",
"stdev": 109.75405196408813
},
{
"id": 14,
"rating": 1684.5329659244817,
"username": "inseres",
"stdev": 5.193191918103997
},
{
"id": 15,
"rating": 1678.1068142163554,
"username": "Jay",
"stdev": 11.220557444400496
},
{
"id": 16,
"rating": 1657.7422053946832,
"username": "kimbifille",
"stdev": 71.62732692961684
},
{
"id": 17,
"rating": 1571.2263147756535,
"username": "Kowalski1337",
"stdev": 59.88063309820496
},
{
"id": 18,
"rating": 1673.652278716787,
"username": "macanek",
"stdev": 37.248874749072684
},
{
"id": 19,
"rating": 1734.2840478574078,
"username": "MarkusKahlsen",
"stdev": 137.263308306954
},
{
"id": 20,
"rating": 1599.1013632247043,
"username": "Neema",
"stdev": 29.99279133170806
},
{
"id": 21,
"rating": 1535.5621323285413,
"username": "newduke",
"stdev": 28.69029514671429
},
{
"id": 22,
"rating": 1439.1271150938064,
"username": "NishaNoire",
"stdev": 135.00876312408192
},
{
"id": 25,
"rating": 1589.9273430803019,
"username": "omegaxis",
"stdev": 45.681434487254656
},
{
"id": 27,
"rating": 1719.856499810081,
"username": "pianoblook",
"stdev": 43.91423347247681
},
{
"id": 28,
"rating": 1645.6070307223388,
"username": "posij118",
"stdev": 69.56052128100919
},
{
"id": 29,
"rating": 1700.7312163612453,
"username": "purplejoe",
"stdev": 4.333849603201881
},
{
"id": 31,
"rating": 1754.8673876868077,
"username": "rahsosprout",
"stdev": 24.20149808460386
},
{
"id": 32,
"rating": 1749.8186567295743,
"username": "Ramanujan",
"stdev": 35.186989633353924
},
{
"id": 33,
"rating": 1505.8114349633183,
"username": "ricardodd",
"stdev": 63.063499555331056
},
{
"id": 34,
"rating": 1424.7454795953922,
"username": "RIMBarisax",
"stdev": 54.62925500990815
},
{
"id": 35,
"rating": 1709.7754274336994,
"username": "rz",
"stdev": 34.75350346761236
},
{
"id": 36,
"rating": 1645.505505262562,
"username": "Sagnik Saha",
"stdev": 82.56415392788715
},
{
"id": 37,
"rating": 1607.8618746654392,
"username": "sodiumdebt",
"stdev": 74.74531010470325
},
{
"id": 38,
"rating": 1799.3860289683867,
"username": "spring",
"stdev": 6.935110645012115
},
{
"id": 39,
"rating": 1547.0924841832866,
"username": "StKildaFan",
"stdev": 22.21102514134186
},
{
"id": 40,
"rating": 1676.6787745059055,
"username": "str8tsknacker",
"stdev": 50.74354553034183
},
{
"id": 41,
"rating": 1731.4770087590437,
"username": "Sturm",
"stdev": 57.142346148910896
},
{
"id": 42,
"rating": 1501.164246026532,
"username": "TimeHoodie",
"stdev": 35.10667883309847
},
{
"id": 43,
"rating": 1655.211023199,
"username": "vEnhance",
"stdev": 42.389156840017165
},
{
"id": 44,
"rating": 1533.6010216854027,
"username": "vermling",
"stdev": 90.93922443450435
},
{
"id": 45,
"rating": 1586.770150886957,
"username": "wateroffire",
"stdev": 3.9655080603422466
},
{
"id": 46,
"rating": 1614.507394752425,
"username": "WillFlame",
"stdev": 9.831543835666807
},
{
"id": 47,
"rating": 1713.6463481565615,
"username": "Yagami Black",
"stdev": 25.99563483333066
},
{
"id": 48,
"rating": 1457.1982216219058,
"username": "youisme",
"stdev": 81.81998675205648
},
{
"id": 51,
"rating": 1437.5089899938134,
"username": "Libster",
"stdev": 8.378590826005142
},
{
"id": 52,
"rating": 1672.8512878685628,
"username": "maxeymo",
"stdev": 36.0712267697332
},
{
"id": 53,
"rating": 1642.5304416034514,
"username": "joano580",
"stdev": 100.76861593939701
},
{
"id": 54,
"rating": 1550.2628143305851,
"username": "ReaverSe",
"stdev": 17.2818922023245
},
{
"id": 55,
"rating": 1642.9660148054686,
"username": "benzloeb",
"stdev": 3.4106951597543786
},
{
"id": 56,
"rating": 1598.4171994249805,
"username": "percolate",
"stdev": 30.720436445911385
},
{
"id": 58,
"rating": 1464.6446630379342,
"username": "Random Guy JCI",
"stdev": 12.310038039511529
},
{
"id": 59,
"rating": 1606.062637102755,
"username": "aara",
"stdev": 21.876906585784877
},
{
"id": 60,
"rating": 1601.4626434355132,
"username": "amattias",
"stdev": 33.079466261234664
},
{
"id": 64,
"rating": 1601.5458400329035,
"username": "wtfitsnotbutter",
"stdev": 53.56517099030838
},
{
"id": 65,
"rating": 1480.6158683098095,
"username": "Alix_Eisenhardt",
"stdev": 50.30711678640308
},
{
"id": 67,
"rating": 1617.5730419198144,
"username": "Vivarus",
"stdev": 9.56814210631875
},
{
"id": 69,
"rating": 1543.1725821109499,
"username": "FrozenStella",
"stdev": 47.349220108955066
},
{
"id": 70,
"rating": 1589.17602492783,
"username": "GameConqueror",
"stdev": 34.77764553932631
},
{
"id": 71,
"rating": 1631.0886330291135,
"username": "seungapark",
"stdev": 36.35644480232967
},
{
"id": 72,
"rating": 1544.058017455111,
"username": "sinuni_hung",
"stdev": 8.068740309638756
},
{
"id": 73,
"rating": 1652.5253246400134,
"username": "TheDaniMan",
"stdev": 43.89447022365573
},
{
"id": 79,
"rating": 1487.3404436278843,
"username": "Kaznad",
"stdev": 15.830803035745209
},
{
"id": 80,
"rating": 1392.331302957674,
"username": "Kernel",
"stdev": 42.02247099621528
},
{
"id": 81,
"rating": 1428.4800076632162,
"username": "Le Codex",
"stdev": 43.789622164916196
},
{
"id": 82,
"rating": 1521.363531664333,
"username": "Nipalup",
"stdev": 104.0370045655497
}
],
"variants": [
{
"id": 0,
"rating": 1359.6612370979074,
"num_players": 3,
"name": "No Variant",
"num_suits": 5,
"stdev": 37.51830058593259
},
{
"id": 1,
"rating": 1409.8285544138594,
"num_players": 3,
"name": "6 Suits",
"num_suits": 6,
"stdev": 56.644590533491034
},
{
"id": 51,
"rating": 1719.7607226091918,
"num_players": 3,
"name": "Clue Starved (6 Suits)",
"num_suits": 6,
"stdev": 8.202101579431847
},
{
"id": 52,
"rating": 1662.868864963005,
"num_players": 3,
"name": "Clue Starved (5 Suits)",
"num_suits": 5,
"stdev": 8.674234234615462
},
{
"id": 0,
"rating": 1439.4451347766544,
"num_players": 4,
"name": "No Variant",
"num_suits": 5,
"stdev": 2.016825515539241
},
{
"id": 1,
"rating": 1434.2720502574575,
"num_players": 4,
"name": "6 Suits",
"num_suits": 6,
"stdev": 29.836778320728673
},
{
"id": 51,
"rating": 1779.845972756269,
"num_players": 4,
"name": "Clue Starved (6 Suits)",
"num_suits": 6,
"stdev": 47.696178458669856
},
{
"id": 52,
"rating": 1701.8431047063805,
"num_players": 4,
"name": "Clue Starved (5 Suits)",
"num_suits": 5,
"stdev": 42.02213931310847
},
{
"id": 0,
"rating": 1426.7937800667978,
"num_players": 5,
"name": "No Variant",
"num_suits": 5,
"stdev": 54.921511042199526
},
{
"id": 1,
"rating": 1495.2514337712873,
"num_players": 5,
"name": "6 Suits",
"num_suits": 6,
"stdev": 43.16247744670021
},
{
"id": 51,
"rating": 1898.843058275536,
"num_players": 5,
"name": null,
"num_suits": 6,
"stdev": 22.62832823639932
},
{
"id": 52,
"rating": 1916.5106100141977,
"num_players": 5,
"name": "Clue Starved (5 Suits)",
"num_suits": 5,
"stdev": 37.61403719598357
}
]
}

406
src/minimize_loss.py Normal file
View file

@ -0,0 +1,406 @@
from typing import Iterable, List, Dict
from config import config_manager
from constants import UNWINNABLE_SEED_FRACTION
from dataclasses import dataclass
import json
import datetime
import numpy as np
from numpy.typing import NDArray
from scipy.optimize import minimize, Bounds
MAX_RATING_DIFF = 1200
RATING_PRIOR = 1600
# This allows us to do integer optimization on more fine-grained grid.
LCM_FACTOR = 60
NUM_RANDOM_INITS = 50
NUM_CV_ITERS = 50
VARIANT_DEFAULT_RATINGS = np.array([1400, 1400, 1800, 1800]).reshape((1, 4))
VARIANT_NUM_PLAYERS_MODIFIERS = np.array([0, 0, 100]).reshape((3, 1))
# As of season 1, this is [1400 1400 1800 1800 1400 1400 1800 1800 1500 1500 1900 1900].
VARIANT_RATING_PRIORS = np.ravel(
VARIANT_DEFAULT_RATINGS + VARIANT_NUM_PLAYERS_MODIFIERS
)
@dataclass
class GlobalInfo:
rated_ids: Iterable[int]
variant_ids: Iterable[int]
player_counts: Iterable[int]
user_names: Iterable[str]
game_counts: Iterable[int]
rated_id_indices_in_rating_list: Dict[int, int]
@property
def rating_list_length(self):
return len(self.rated_ids) + len(self.variant_ids) * len(self.player_counts)
@dataclass
class GameRow:
game_id: int
num_players: int
users: List[str]
user_ids: List[int]
user_rating_changes: List[float]
user_ratings_after: List[float]
variant_id: int
variant_name: str
num_suits: id
rating_type: int
seed: str
score: int
num_turns: int
datetime_finished: datetime.datetime
league_id: int
num_bdrs: int
num_crits_lost: int
game_outcomes: List[str]
variant_rating_change: float
variant_rating_after: float
def calculate_loss(
rating_list: Iterable[int],
game_list: Iterable[GameRow],
global_info: GlobalInfo,
p_win_lookup_table: NDArray[np.float32],
player_rating_priors: Iterable[int],
player_prior_weight: float,
variant_rating_priors: Iterable[int],
variant_prior_weight: float,
):
loss = np.float32(0)
for game in game_list:
team_ratings = [
rating_list[global_info.rated_id_indices_in_rating_list[user_id]]
for user_id in game["user_ids"]
]
team_rating = sum(team_ratings) / len(team_ratings)
variant_rating = rating_list[
calculate_variant_index_in_rating_list(
len(game["users"]), game["variant_id"], global_info
)
]
rating_diff = int(np.round(LCM_FACTOR * (team_rating - variant_rating)))
p_win = calculate_p_win(rating_diff, p_win_lookup_table)
game_won = game["game_outcomes"].count("Win") > 0
loss += calculate_loss_for_one_game(p_win, game_won)
# Add two dummy single-player games for each player with prior_weight * square-root of number of games weight - one won and lost
for player_rating_prior, rated_id, game_count in zip(
player_rating_priors, global_info.rated_ids, global_info.game_counts
):
team_rating = rating_list[global_info.rated_id_indices_in_rating_list[rated_id]]
variant_rating = player_rating_prior
rating_diff = int(np.round(LCM_FACTOR * (team_rating - variant_rating)))
p_win = calculate_p_win(rating_diff, p_win_lookup_table)
loss += (
np.sqrt(game_count)
* player_prior_weight
* calculate_loss_for_one_game(p_win, True)
)
loss += (
np.sqrt(game_count)
* player_prior_weight
* calculate_loss_for_one_game(p_win, False)
)
# Add two dummy single-player games for each player with prior_weight * square-root of number of games weight - one won and lost
for variant_rating, variant_rating_prior in zip(
rating_list[len(global_info.rated_ids) :],
variant_rating_priors,
):
team_rating = variant_rating_prior
rating_diff = int(np.round(LCM_FACTOR * (team_rating - variant_rating)))
p_win = calculate_p_win(rating_diff, p_win_lookup_table)
loss += (
np.sqrt(game_count)
* variant_prior_weight
* calculate_loss_for_one_game(p_win, True)
)
loss += (
np.sqrt(game_count)
* variant_prior_weight
* calculate_loss_for_one_game(p_win, False)
)
return loss
def calculate_loss_gradient(rating_list, *args):
y = calculate_loss(rating_list, *args)
eye = np.eye(len(rating_list), dtype=int)
gradient = [
calculate_loss(rating_list + eye[i], *args) - y for i in range(len(rating_list))
]
return gradient
def calculate_minloss_ratings(
game_list: List[GameRow],
global_info: GlobalInfo,
p_win_lookup_table: NDArray[np.float32],
player_rating_priors: int | Iterable[int] = RATING_PRIOR,
player_random_init_stdev: int = 0,
player_prior_weight=0.5,
variant_rating_priors: int | Iterable[int] = RATING_PRIOR,
variant_prior_weight=0.5,
variant_random_init_stdev: int = 0,
):
player_rating_priors = np.broadcast_to(
player_rating_priors, len(global_info.rated_ids)
)
variant_rating_priors = np.broadcast_to(
variant_rating_priors,
len(global_info.variant_ids) * len(global_info.player_counts),
)
prior_rating_list = np.concatenate(
(
player_rating_priors
+ np.clip(
np.random.normal(
0, player_random_init_stdev, len(player_rating_priors)
),
-3 * player_random_init_stdev,
3 * player_random_init_stdev,
),
variant_rating_priors
+ np.clip(
np.random.normal(
0, variant_random_init_stdev, len(variant_rating_priors)
),
-3 * variant_random_init_stdev,
3 * variant_random_init_stdev,
),
),
dtype=np.float32,
)
rating_list = minimize(
calculate_loss,
prior_rating_list,
(
game_list,
global_info,
p_win_lookup_table,
player_rating_priors,
player_prior_weight,
variant_rating_priors,
variant_prior_weight,
),
bounds=Bounds(RATING_PRIOR - MAX_RATING_DIFF, RATING_PRIOR + MAX_RATING_DIFF),
jac=calculate_loss_gradient,
tol=0.0001,
)
return rating_list
def calculate_p_win_lookup_table(rating_diff_indices: Iterable[int]):
lookup_table = np.full(2 * MAX_RATING_DIFF * LCM_FACTOR + 1, 0.5, dtype=np.float32)
for rating_diff_index in rating_diff_indices:
p_win = calculate_p_win(rating_diff_index, None)
lookup_table[rating_diff_index] = p_win
return lookup_table
def calculate_p_win(
rating_diff_index: int, lookup_table: NDArray[np.float32] | None = None
):
if lookup_table is None:
return (1 - UNWINNABLE_SEED_FRACTION) * (
1 / (1 + 10 ** (-(rating_diff_index / LCM_FACTOR) / 400))
)
else:
return lookup_table[rating_diff_index]
# Cross-entropy loss
def calculate_loss_for_one_game(p_win: np.float32, game_won: bool):
if game_won:
loss = -np.log(p_win)
else:
loss = -np.log(1 - p_win)
return loss
def calculate_variant_index_in_rating_list(
player_count: int, variant_id: int, global_info: GlobalInfo
):
return (
len(global_info.rated_ids)
+ global_info.player_counts.index(player_count) * len(global_info.variant_ids)
+ global_info.variant_ids.index(variant_id)
)
def calculate_player_count_and_variant_id_from_variant_index(
index: int, global_info: GlobalInfo
):
num_players = global_info.player_counts[index // len(global_info.variant_ids)]
variant_id = global_info.variant_ids[index % len(global_info.variant_ids)]
return num_players, variant_id
def write_data(random_init_rating_lists, cv_rating_lists):
with open("../results/min_loss_rating_list.json", "w") as f:
rating_list = np.average(random_init_rating_lists, axis=0)
rating_stdevs = np.std(cv_rating_lists, axis=0, ddof=1)
players = []
for rated_id, user_name, rating, rating_stdev in zip(
global_info.rated_ids, global_info.user_names, rating_list, rating_stdevs
):
players.append(
{
"id": rated_id,
"rating": rating,
"username": user_name,
"stdev": rating_stdev,
}
)
variants = []
for i, (rating, rating_stdev) in enumerate(
zip(
rating_list[len(global_info.rated_ids) :],
rating_stdevs[len(global_info.rated_ids) :],
)
):
(
num_players,
variant_id,
) = calculate_player_count_and_variant_id_from_variant_index(i, global_info)
variant_name = None
for game in game_list:
if (
len(game["users"]) == num_players
and game["variant_id"] == variant_id
):
variant_name = game["variant_name"]
num_suits = game["num_suits"]
variants.append(
{
"id": variant_id,
"rating": rating,
"num_players": num_players,
"name": variant_name,
"num_suits": num_suits,
"stdev": rating_stdev,
}
)
s = json.dumps({"players": players, "variants": variants})
f.write(s)
def read_data():
with open("../data/games.json") as game_data:
game_list = json.loads(game_data.read())
p_win_lookup_table = calculate_p_win_lookup_table(
np.arange(-MAX_RATING_DIFF * LCM_FACTOR, MAX_RATING_DIFF * LCM_FACTOR + 1)
)
config = config_manager.get_config()
rated_ids_dupes = [ID for game in game_list for ID in game["user_ids"]]
rated_ids = list(set(rated_ids_dupes))
variant_ids_dupes = [game["variant_id"] for game in game_list]
variant_ids = list(set(variant_ids_dupes))
nums_players = [game["num_players"] for game in game_list]
player_counts = list(range(config.min_player_count, config.max_player_count + 1))
game_counts = np.concatenate(
(
[rated_ids_dupes.count(ID) for ID in rated_ids],
np.zeros(len(variant_ids) * len(player_counts)),
),
)
user_names = list(np.full((len(rated_ids),), ""))
for game in game_list:
for user_id, user_name in zip(game["user_ids"], game["users"]):
user_names[rated_ids.index(user_id)] = user_name
rated_id_indices_in_rating_list = {
id: index for index, id in dict(enumerate(rated_ids)).items()
}
global_info = GlobalInfo(
rated_ids,
variant_ids,
player_counts,
user_names,
game_counts,
rated_id_indices_in_rating_list,
)
for variant_id, num_players in zip(variant_ids_dupes, nums_players):
game_counts[
calculate_variant_index_in_rating_list(num_players, variant_id, global_info)
] += 1
global_info.game_counts = game_counts
return game_list, global_info, p_win_lookup_table
if __name__ == "__main__":
game_list, global_info, p_win_lookup_table = read_data()
random_init_rating_lists = []
cv_rating_lists = []
for _ in range(NUM_RANDOM_INITS):
random_init_rating_lists.append(
calculate_minloss_ratings(
game_list,
global_info,
p_win_lookup_table,
variant_rating_priors=VARIANT_RATING_PRIORS,
player_random_init_stdev=200,
variant_random_init_stdev=200,
).x
)
for _ in range(NUM_CV_ITERS):
cv_game_list = np.random.choice(game_list, len(game_list), replace=True)
cv_rating_lists.append(
calculate_minloss_ratings(
cv_game_list,
global_info,
p_win_lookup_table,
variant_rating_priors=VARIANT_RATING_PRIORS,
player_random_init_stdev=200,
variant_random_init_stdev=200,
).x
)
write_data(random_init_rating_lists, cv_rating_lists)