Add code to fetch new games

config.py

@@ -201,3 +201,26 @@ def create_config() -> None:
         logger.info("Created default hanabi league config file at {}".format(config_path))
     else:
         logger.info("Hanabi league config file at {} already exists".format(config_path))
+
+
+class ConfigManager:
+    """
+    This class manages lazily loading each config exactly once.
+    """
+    def __init__(self):
+        self._db_config = None
+        self._config = None
+
+    def get_db_config(self):
+        if self._db_config is None:
+            self._db_config = read_db_config()
+        return self._db_config
+
+    def get_config(self):
+        if self._config is None:
+            self._config = read_config()
+        return self._config
+
+
+# Global config manager
+config_manager = ConfigManager()

constants.py

@@ -32,3 +32,11 @@ DEFAULT_CONFIG_PATH = 'install/default_config.yaml'
 
 VARIANTS_JSON_URL = 'https://raw.githubusercontent.com/Hanabi-Live/hanabi-live/main/packages/data/src/json/variants.json'
 
+FORBIDDEN_GAME_OPTIONS = [
+    "deckPlays"
+    , "emptyClues"
+    , "oneExtraCard"
+    , "oneLessCard"
+    , "allOrNothing"
+    , "detrimentalCharacters"
+]

database.py

@@ -98,6 +98,11 @@ def fetch_and_initialize_variants():
     conn_manager.get_connection().commit()
 
 
+def normalize_username(username: str) -> str:
+    decoded = unidecode.unidecode(username)
+    return decoded.lower()
+
+
 def add_player_name(player_name: str):
     conn = conn_manager.get_connection()
     cur = conn.cursor()
@@ -110,7 +115,7 @@ def add_player_name(player_name: str):
 
 
 def add_user_name_to_player(hanabi_username: str, player_name: str):
-    normalized_username = unidecode.unidecode(hanabi_username).lower()
+    normalized_username = normalize_username(hanabi_username)
     cur = conn_manager.get_new_cursor()
     cur.execute("SELECT id FROM users WHERE player_name = (%s)", (player_name,))
     user_id = cur.fetchone()
@@ -147,3 +152,9 @@ def add_user_name_to_player(hanabi_username: str, player_name: str):
 def add_player(player_name: str, user_name: str):
     add_player_name(player_name)
     add_user_name_to_player(user_name, player_name)
+
+
+def get_variant_ids():
+    cur = conn_manager.get_new_cursor()
+    cur.execute("SELECT id FROM variants")
+    return [var_id for (var_id,) in cur.fetchall()]

fetch_games.py

@@ -0,0 +1,176 @@
+import json
+from typing import Dict, List, Optional
+
+import platformdirs
+import requests_cache
+import psycopg2.extras
+
+import constants
+import database
+from database import conn_manager
+from log_setup import logger
+from config import config_manager
+
+# This ensures that we do not pose too many requests to the website, especially while in development where we
+# might frequently re-initialize the database.
+session = requests_cache.CachedSession(
+    platformdirs.user_cache_dir(constants.APP_NAME) + 'hanab.live.requests-cache',
+    urls_expire_after={
+        # Game exports will never change, so cache them forever
+        'hanab.live/export/*': requests_cache.NEVER_EXPIRE
+    }
+)
+
+
+class GameInfo:
+    def __init__(self, game_id: int, num_players: int, variant_id: int, seed: str, score: int, num_turns: int, user_ids: List[int], normalized_usernames: List[str]):
+        self.game_id = game_id
+        self.num_players = num_players
+        self.variant_id = variant_id
+        self.seed = seed
+        self.score = score
+        self.num_turns = num_turns
+        self.user_ids = user_ids
+        self.normalized_usernames = normalized_usernames
+
+def fetch_games_for_player(username: str, latest_game_id: int):
+    logger.verbose("Fetching games for username {}".format(username))
+    url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, latest_game_id + 1)
+    response = session.get(url)
+    if not response.status_code == 200:
+        err_msg = "Failed to fetch games for username {}, requested URL {}".format(username, url)
+        logger.error(err_msg)
+        raise ConnectionError(err_msg)
+    return json.loads(response.text)
+
+
+def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[int]) -> Optional[GameInfo]:
+    logger.debug("Processing entry {}".format(game_json))
+    config = config_manager.get_config()
+    # Check if the game is one that we accept
+    game_id = game_json["id"]
+    players = game_json["playerNames"]
+    num_players = len(players)
+    seed = game_json["seed"]
+    score = game_json["score"]
+    num_turns = game_json["numTurns"]
+
+    game_options = game_json["options"]
+    var_id = game_options["variantID"]
+
+    normalized_usernames = [database.normalize_username(username) for username in players]
+
+    # Check that the game has no special options enabled
+    for forbidden_option in constants.FORBIDDEN_GAME_OPTIONS:
+        if game_options.get(forbidden_option, False):
+            logger.debug("Rejected game {} due to option {} set".format(game_id, forbidden_option))
+            return
+
+    # Check if player count matches
+    if not (config.min_player_count <= num_players <= config.max_player_count):
+        logger.debug("Rejected game {} due to invalid number of players ({})".format(game_id, num_players))
+        return
+
+    # Check if the variant was ok
+    if var_id not in variant_ids:
+        logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
+        return
+
+    # Everything matches, so we can parse the participants now
+    user_ids = []  # This will be a list of the (league specific) user_id's that played this game.
+    for normalized_username in normalized_usernames:
+        user_id = username_dict.get(normalized_username, None)
+        if user_id is None:
+            logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, normalized_username))
+            return
+        user_ids.append(user_id)
+
+    return GameInfo(game_id, num_players, 0, seed, score, num_turns, user_ids, normalized_usernames)
+
+
+def fetch_games_for_all_players():
+    logger.info("Fetching new games.")
+    cur = conn_manager.get_new_cursor()
+    cur.execute("SELECT user_accounts.normalized_username, user_accounts.user_id, downloads.latest_game_id "
+                "FROM user_accounts "
+                "LEFT OUTER JOIN downloads "
+                "  ON user_accounts.normalized_username = downloads.normalized_username"
+                )
+    # This will be a mapping of normalized username -> user ID that we built from the DB data
+    username_dict = {}
+
+    # This will be a mapping of game id -> JSON data that we get from hanab.live, where we will collect all the
+    # possibly relevant games now
+    games: Dict[int, Dict] = {}
+    for username, user_id, latest_game_id in cur.fetchall():
+        username_dict[username] = user_id
+        # Use the starting id as fallback if we have not downloaded any games for this player yet.
+        if latest_game_id is None:
+            latest_game_id = config_manager.get_config().starting_game_id
+
+        player_games = fetch_games_for_player(username, latest_game_id)
+        for game in player_games:
+            games[game['id']] = game
+
+    allowed_variants = database.get_variant_ids()
+
+    # This will hold the processed games that we will add to the database.
+    good_games: Dict[int, GameInfo] = {}
+
+    for game_id, game in games.items():
+        game_info = process_game_entry(game, username_dict, allowed_variants)
+        if game_info is not None:
+            good_games[game_id] = game_info
+
+    return good_games
+
+
+def store_new_games(games: Dict[int, GameInfo]):
+    conn = conn_manager.get_connection()
+    cur = conn.cursor()
+    games_vals = []
+    game_participants_vals = []
+    latest_game_ids: Dict[str, int] = {}
+
+    # Now, iterate over all games and convert to tuples for insertion
+    for game in sorted(games.values(), key=lambda game_info: game_info.game_id):
+        tup = (game.game_id, game.num_players, game.variant_id, game.seed, game.score, game.num_turns)
+        games_vals.append(tup)
+        for player_id in game.user_ids:
+            tup = (game.game_id, player_id)
+            game_participants_vals.append(tup)
+
+        # Note that this gives the maximum in the end since we process the games in order
+        for normalized_username in game.normalized_usernames:
+            latest_game_ids[normalized_username] = game.game_id
+
+    # Do the insertions
+    psycopg2.extras.execute_values(
+        cur,
+        "INSERT INTO games (id, num_players, variant_id, seed, score, num_turns) "
+        "VALUES %s",
+        games_vals
+    )
+    psycopg2.extras.execute_values(
+        cur,
+        "INSERT INTO game_participants (game_id, user_id) "
+        "VALUES %s",
+        game_participants_vals
+    )
+    psycopg2.extras.execute_values(
+        cur,
+        "INSERT INTO downloads (normalized_username, latest_game_id) "
+        "VALUES %s",
+        latest_game_ids.items()
+    )
+    conn.commit()
+    """
+    cur.execute("INSERT INTO games (id, num_players, variant_id, seed, score, num_turns) "
+                "VALUES %s""
+                )
+    game_participant_args = b", ".join(game_participants_vals)
+    cur.execute("INSERT INTO game_participants (game_id, user_id, seat) "
+                "VALUES {}".format(game_participants_vals))
+
+    conn.commit()
+"""

install/database_schema.sql

@@ -98,7 +98,7 @@ CREATE TABLE user_accounts (
 DROP TABLE IF EXISTS downloads;
 CREATE TABLE downloads (
   /** Notice this has to be a hanab.live username, not a user_id */
-  username TEXT PRIMARY KEY REFERENCES user_accounts,
+  normalized_username TEXT PRIMARY KEY REFERENCES user_accounts (normalized_username),
   latest_game_id INTEGER NOT NULL
 );
 
@@ -129,6 +129,7 @@ CREATE TABLE games (
   /** Same here: seed from hanab.live */
   seed TEXT NOT NULL,
   score SMALLINT NOT NULL,
+  num_turns SMALLINT NOT NULL,
   /**
   * This is the league id mentioned above that will represent the ordering of games regarding being processed by ELO.
   * Note that this means when fetching new data from hanab.live, we have to fetch *all* of it and insert the games sorted
@@ -154,8 +155,12 @@ CREATE TABLE game_participants (
   * Hanab.live stores this as well, I don't think that we need it currently, but there's no harm done in storing this, since it would enable us to know
   * which player is which if we were to do per-player statistics also inside an individual game.
   * For example, I could imagine stuff like: 'Clues given' or 'Cards played', which might actually vary across players.
+  * Unfortunately, when using the 'history-full' api hook, player names are reported alphabetically, so to retrieve
+  * the seat order, we have to use the 'export' endpoint of the api, which requires an extra request for each game.
+  * This is why we allow for this entry to be null in general, so that we can easily store the games without the seat
+  * being known.
   */
-  seat SMALLINT NOT NULL,
+  seat SMALLINT,
   FOREIGN KEY (game_id) REFERENCES games (id),
   FOREIGN KEY (user_id) REFERENCES users (id),
   CONSTRAINT game_participants_unique UNIQUE (game_id, user_id)