Add code to fetch new games
This commit is contained in:
parent
01ebfa380d
commit
20ec64ef04
5 changed files with 226 additions and 3 deletions
23
config.py
23
config.py
|
@ -201,3 +201,26 @@ def create_config() -> None:
|
|||
logger.info("Created default hanabi league config file at {}".format(config_path))
|
||||
else:
|
||||
logger.info("Hanabi league config file at {} already exists".format(config_path))
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
"""
|
||||
This class manages lazily loading each config exactly once.
|
||||
"""
|
||||
def __init__(self):
|
||||
self._db_config = None
|
||||
self._config = None
|
||||
|
||||
def get_db_config(self):
|
||||
if self._db_config is None:
|
||||
self._db_config = read_db_config()
|
||||
return self._db_config
|
||||
|
||||
def get_config(self):
|
||||
if self._config is None:
|
||||
self._config = read_config()
|
||||
return self._config
|
||||
|
||||
|
||||
# Global config manager
|
||||
config_manager = ConfigManager()
|
||||
|
|
|
@ -32,3 +32,11 @@ DEFAULT_CONFIG_PATH = 'install/default_config.yaml'
|
|||
|
||||
VARIANTS_JSON_URL = 'https://raw.githubusercontent.com/Hanabi-Live/hanabi-live/main/packages/data/src/json/variants.json'
|
||||
|
||||
FORBIDDEN_GAME_OPTIONS = [
|
||||
"deckPlays"
|
||||
, "emptyClues"
|
||||
, "oneExtraCard"
|
||||
, "oneLessCard"
|
||||
, "allOrNothing"
|
||||
, "detrimentalCharacters"
|
||||
]
|
||||
|
|
13
database.py
13
database.py
|
@ -98,6 +98,11 @@ def fetch_and_initialize_variants():
|
|||
conn_manager.get_connection().commit()
|
||||
|
||||
|
||||
def normalize_username(username: str) -> str:
|
||||
decoded = unidecode.unidecode(username)
|
||||
return decoded.lower()
|
||||
|
||||
|
||||
def add_player_name(player_name: str):
|
||||
conn = conn_manager.get_connection()
|
||||
cur = conn.cursor()
|
||||
|
@ -110,7 +115,7 @@ def add_player_name(player_name: str):
|
|||
|
||||
|
||||
def add_user_name_to_player(hanabi_username: str, player_name: str):
|
||||
normalized_username = unidecode.unidecode(hanabi_username).lower()
|
||||
normalized_username = normalize_username(hanabi_username)
|
||||
cur = conn_manager.get_new_cursor()
|
||||
cur.execute("SELECT id FROM users WHERE player_name = (%s)", (player_name,))
|
||||
user_id = cur.fetchone()
|
||||
|
@ -147,3 +152,9 @@ def add_user_name_to_player(hanabi_username: str, player_name: str):
|
|||
def add_player(player_name: str, user_name: str):
|
||||
add_player_name(player_name)
|
||||
add_user_name_to_player(user_name, player_name)
|
||||
|
||||
|
||||
def get_variant_ids():
|
||||
cur = conn_manager.get_new_cursor()
|
||||
cur.execute("SELECT id FROM variants")
|
||||
return [var_id for (var_id,) in cur.fetchall()]
|
||||
|
|
176
fetch_games.py
Normal file
176
fetch_games.py
Normal file
|
@ -0,0 +1,176 @@
|
|||
import json
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import platformdirs
|
||||
import requests_cache
|
||||
import psycopg2.extras
|
||||
|
||||
import constants
|
||||
import database
|
||||
from database import conn_manager
|
||||
from log_setup import logger
|
||||
from config import config_manager
|
||||
|
||||
# This ensures that we do not pose too many requests to the website, especially while in development where we
|
||||
# might frequently re-initialize the database.
|
||||
session = requests_cache.CachedSession(
|
||||
platformdirs.user_cache_dir(constants.APP_NAME) + 'hanab.live.requests-cache',
|
||||
urls_expire_after={
|
||||
# Game exports will never change, so cache them forever
|
||||
'hanab.live/export/*': requests_cache.NEVER_EXPIRE
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class GameInfo:
|
||||
def __init__(self, game_id: int, num_players: int, variant_id: int, seed: str, score: int, num_turns: int, user_ids: List[int], normalized_usernames: List[str]):
|
||||
self.game_id = game_id
|
||||
self.num_players = num_players
|
||||
self.variant_id = variant_id
|
||||
self.seed = seed
|
||||
self.score = score
|
||||
self.num_turns = num_turns
|
||||
self.user_ids = user_ids
|
||||
self.normalized_usernames = normalized_usernames
|
||||
|
||||
def fetch_games_for_player(username: str, latest_game_id: int):
|
||||
logger.verbose("Fetching games for username {}".format(username))
|
||||
url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, latest_game_id + 1)
|
||||
response = session.get(url)
|
||||
if not response.status_code == 200:
|
||||
err_msg = "Failed to fetch games for username {}, requested URL {}".format(username, url)
|
||||
logger.error(err_msg)
|
||||
raise ConnectionError(err_msg)
|
||||
return json.loads(response.text)
|
||||
|
||||
|
||||
def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[int]) -> Optional[GameInfo]:
|
||||
logger.debug("Processing entry {}".format(game_json))
|
||||
config = config_manager.get_config()
|
||||
# Check if the game is one that we accept
|
||||
game_id = game_json["id"]
|
||||
players = game_json["playerNames"]
|
||||
num_players = len(players)
|
||||
seed = game_json["seed"]
|
||||
score = game_json["score"]
|
||||
num_turns = game_json["numTurns"]
|
||||
|
||||
game_options = game_json["options"]
|
||||
var_id = game_options["variantID"]
|
||||
|
||||
normalized_usernames = [database.normalize_username(username) for username in players]
|
||||
|
||||
# Check that the game has no special options enabled
|
||||
for forbidden_option in constants.FORBIDDEN_GAME_OPTIONS:
|
||||
if game_options.get(forbidden_option, False):
|
||||
logger.debug("Rejected game {} due to option {} set".format(game_id, forbidden_option))
|
||||
return
|
||||
|
||||
# Check if player count matches
|
||||
if not (config.min_player_count <= num_players <= config.max_player_count):
|
||||
logger.debug("Rejected game {} due to invalid number of players ({})".format(game_id, num_players))
|
||||
return
|
||||
|
||||
# Check if the variant was ok
|
||||
if var_id not in variant_ids:
|
||||
logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
|
||||
return
|
||||
|
||||
# Everything matches, so we can parse the participants now
|
||||
user_ids = [] # This will be a list of the (league specific) user_id's that played this game.
|
||||
for normalized_username in normalized_usernames:
|
||||
user_id = username_dict.get(normalized_username, None)
|
||||
if user_id is None:
|
||||
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, normalized_username))
|
||||
return
|
||||
user_ids.append(user_id)
|
||||
|
||||
return GameInfo(game_id, num_players, 0, seed, score, num_turns, user_ids, normalized_usernames)
|
||||
|
||||
|
||||
def fetch_games_for_all_players():
|
||||
logger.info("Fetching new games.")
|
||||
cur = conn_manager.get_new_cursor()
|
||||
cur.execute("SELECT user_accounts.normalized_username, user_accounts.user_id, downloads.latest_game_id "
|
||||
"FROM user_accounts "
|
||||
"LEFT OUTER JOIN downloads "
|
||||
" ON user_accounts.normalized_username = downloads.normalized_username"
|
||||
)
|
||||
# This will be a mapping of normalized username -> user ID that we built from the DB data
|
||||
username_dict = {}
|
||||
|
||||
# This will be a mapping of game id -> JSON data that we get from hanab.live, where we will collect all the
|
||||
# possibly relevant games now
|
||||
games: Dict[int, Dict] = {}
|
||||
for username, user_id, latest_game_id in cur.fetchall():
|
||||
username_dict[username] = user_id
|
||||
# Use the starting id as fallback if we have not downloaded any games for this player yet.
|
||||
if latest_game_id is None:
|
||||
latest_game_id = config_manager.get_config().starting_game_id
|
||||
|
||||
player_games = fetch_games_for_player(username, latest_game_id)
|
||||
for game in player_games:
|
||||
games[game['id']] = game
|
||||
|
||||
allowed_variants = database.get_variant_ids()
|
||||
|
||||
# This will hold the processed games that we will add to the database.
|
||||
good_games: Dict[int, GameInfo] = {}
|
||||
|
||||
for game_id, game in games.items():
|
||||
game_info = process_game_entry(game, username_dict, allowed_variants)
|
||||
if game_info is not None:
|
||||
good_games[game_id] = game_info
|
||||
|
||||
return good_games
|
||||
|
||||
|
||||
def store_new_games(games: Dict[int, GameInfo]):
|
||||
conn = conn_manager.get_connection()
|
||||
cur = conn.cursor()
|
||||
games_vals = []
|
||||
game_participants_vals = []
|
||||
latest_game_ids: Dict[str, int] = {}
|
||||
|
||||
# Now, iterate over all games and convert to tuples for insertion
|
||||
for game in sorted(games.values(), key=lambda game_info: game_info.game_id):
|
||||
tup = (game.game_id, game.num_players, game.variant_id, game.seed, game.score, game.num_turns)
|
||||
games_vals.append(tup)
|
||||
for player_id in game.user_ids:
|
||||
tup = (game.game_id, player_id)
|
||||
game_participants_vals.append(tup)
|
||||
|
||||
# Note that this gives the maximum in the end since we process the games in order
|
||||
for normalized_username in game.normalized_usernames:
|
||||
latest_game_ids[normalized_username] = game.game_id
|
||||
|
||||
# Do the insertions
|
||||
psycopg2.extras.execute_values(
|
||||
cur,
|
||||
"INSERT INTO games (id, num_players, variant_id, seed, score, num_turns) "
|
||||
"VALUES %s",
|
||||
games_vals
|
||||
)
|
||||
psycopg2.extras.execute_values(
|
||||
cur,
|
||||
"INSERT INTO game_participants (game_id, user_id) "
|
||||
"VALUES %s",
|
||||
game_participants_vals
|
||||
)
|
||||
psycopg2.extras.execute_values(
|
||||
cur,
|
||||
"INSERT INTO downloads (normalized_username, latest_game_id) "
|
||||
"VALUES %s",
|
||||
latest_game_ids.items()
|
||||
)
|
||||
conn.commit()
|
||||
"""
|
||||
cur.execute("INSERT INTO games (id, num_players, variant_id, seed, score, num_turns) "
|
||||
"VALUES %s""
|
||||
)
|
||||
game_participant_args = b", ".join(game_participants_vals)
|
||||
cur.execute("INSERT INTO game_participants (game_id, user_id, seat) "
|
||||
"VALUES {}".format(game_participants_vals))
|
||||
|
||||
conn.commit()
|
||||
"""
|
|
@ -98,7 +98,7 @@ CREATE TABLE user_accounts (
|
|||
DROP TABLE IF EXISTS downloads;
|
||||
CREATE TABLE downloads (
|
||||
/** Notice this has to be a hanab.live username, not a user_id */
|
||||
username TEXT PRIMARY KEY REFERENCES user_accounts,
|
||||
normalized_username TEXT PRIMARY KEY REFERENCES user_accounts (normalized_username),
|
||||
latest_game_id INTEGER NOT NULL
|
||||
);
|
||||
|
||||
|
@ -129,6 +129,7 @@ CREATE TABLE games (
|
|||
/** Same here: seed from hanab.live */
|
||||
seed TEXT NOT NULL,
|
||||
score SMALLINT NOT NULL,
|
||||
num_turns SMALLINT NOT NULL,
|
||||
/**
|
||||
* This is the league id mentioned above that will represent the ordering of games regarding being processed by ELO.
|
||||
* Note that this means when fetching new data from hanab.live, we have to fetch *all* of it and insert the games sorted
|
||||
|
@ -154,8 +155,12 @@ CREATE TABLE game_participants (
|
|||
* Hanab.live stores this as well, I don't think that we need it currently, but there's no harm done in storing this, since it would enable us to know
|
||||
* which player is which if we were to do per-player statistics also inside an individual game.
|
||||
* For example, I could imagine stuff like: 'Clues given' or 'Cards played', which might actually vary across players.
|
||||
* Unfortunately, when using the 'history-full' api hook, player names are reported alphabetically, so to retrieve
|
||||
* the seat order, we have to use the 'export' endpoint of the api, which requires an extra request for each game.
|
||||
* This is why we allow for this entry to be null in general, so that we can easily store the games without the seat
|
||||
* being known.
|
||||
*/
|
||||
seat SMALLINT NOT NULL,
|
||||
seat SMALLINT,
|
||||
FOREIGN KEY (game_id) REFERENCES games (id),
|
||||
FOREIGN KEY (user_id) REFERENCES users (id),
|
||||
CONSTRAINT game_participants_unique UNIQUE (game_id, user_id)
|
||||
|
|
Loading…
Reference in a new issue