rework game fetching: deduplicate code, fetch timestamps

src/database.py

@@ -246,7 +246,7 @@ def add_player(user_name: str, player_name: str, base_rating: Optional[int] = No
     init_player_base_rating(player_name, base_rating)
 
 
-def get_variant_ids():
+def get_variant_ids() -> List[int]:
     cur = conn_manager.get_new_cursor()
     cur.execute("SELECT id FROM variants")
     return [var_id for (var_id,) in cur.fetchall()]

src/fetch_games.py

@@ -43,9 +43,12 @@ class GameInfo:
     datetime_ended: str
 
 
-def fetch_games_for_player(username: str, latest_game_id: int):
+def fetch_games_for_player(username: str, first_game_id: int, last_game_id: Optional[int] = None) -> Dict:
-    logger.verbose("Fetching games for username {} more recent than id {}".format(username, latest_game_id))
+    logger.verbose("Fetching games for username {} more recent than id {}".format(username, first_game_id))
-    url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, latest_game_id + 1)
+    url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, first_game_id)
+    if last_game_id is not None:
+        url += "&end={}".format(last_game_id)
+    print(url)
     response = session.get(url)
     if not response.status_code == 200:
         err_msg = "Failed to fetch games for username {}, requested URL {}".format(username, url)
@@ -54,9 +57,8 @@ def fetch_games_for_player(username: str, latest_game_id: int):
     return json.loads(response.text)
 
 
-def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[int]) -> Optional[GameInfo]:
+def process_game_entry(game_json: Dict) -> Optional[GameInfo]:
     logger.debug("Processing entry {}".format(game_json))
-    config = config_manager.get_config()
 
     # Parse game properties
     game_id = game_json["id"]
@@ -72,7 +74,6 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i
     var_id = game_options["variantID"]
 
     normalized_usernames = [utils.normalize_username(username) for username in players]
-
     # Now, check if the game is one that we accept for league
 
     if not all([
@@ -83,63 +84,59 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i
         ]):
         return
 
-    if var_id not in variant_ids:
+    database.get_variant_ids()
+
+    if var_id not in database.get_variant_ids():
         logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
         return
 
     # Everything matches, so we can parse the participants now
-    user_ids = []  # This will be a list of the (league specific) user_id's that played this game.
+    user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames)
-    for normalized_username in normalized_usernames:
+    # The return value here is a str if there was an unregistered participant
-        user_id = username_dict.get(normalized_username, None)
+    if type(user_ids) is str:
-        if user_id is None:
+        logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids))
-            logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, normalized_username))
         return
-        user_ids.append(user_id)
 
     return GameInfo(game_id, num_players, var_id, seed, score, num_turns, user_ids, normalized_usernames, start_time, end_time)
 
 
-def fetch_games_for_all_players():
+def fetch_games_for_all_players() -> List[GameInfo]:
     logger.info("Fetching new games.")
     cur = conn_manager.get_new_cursor()
-    cur.execute("SELECT user_accounts.normalized_username, user_accounts.user_id, downloads.latest_game_id "
+    cur.execute("SELECT"
+                "  user_accounts.normalized_username,"
+                "  user_accounts.user_id,"
+                # Use the starting id as fallback if we have not downloaded any games for this player yet.
+                "  COALESCE(downloads.latest_game_id, %s) "
                 "FROM user_accounts "
                 "LEFT OUTER JOIN downloads "
-                "  ON user_accounts.normalized_username = downloads.normalized_username"
+                "  ON user_accounts.normalized_username = downloads.normalized_username",
+                (config_manager.get_config().starting_game_id,)
                 )
-    # This will be a mapping of normalized username -> user ID that we built from the DB data
-    username_dict = {}
 
     # This will be a mapping of game id -> JSON data that we get from hanab.live, where we will collect all the
     # possibly relevant games now
     games: Dict[int, Dict] = {}
     for username, user_id, latest_game_id in cur.fetchall():
-        username_dict[username] = user_id
+        player_games = fetch_games_for_player(username, latest_game_id + 1)
-        # Use the starting id as fallback if we have not downloaded any games for this player yet.
-        if latest_game_id is None:
-            latest_game_id = config_manager.get_config().starting_game_id
-
-        player_games = fetch_games_for_player(username, latest_game_id)
         for game in player_games:
             games[game['id']] = game
 
     logger.info("Found {} potential league game(s) to process.".format(len(games)))
 
-    allowed_variants = database.get_variant_ids()
-
     # This will hold the processed games that we will add to the database.
-    good_games: Dict[int, GameInfo] = {}
+    good_games: List[GameInfo] = []
 
     for game_id, game in games.items():
-        game_info = process_game_entry(game, username_dict, allowed_variants)
+        game_info = process_game_entry(game)
         if game_info is not None:
-            good_games[game_id] = game_info
+            good_games.append(game_info)
 
     logger.verbose("Found {} valid league game(s).".format(len(good_games)))
     return good_games
 
 
-def store_new_games(games: Dict[int, GameInfo]):
+def store_new_games(games: List[GameInfo]):
     conn = conn_manager.get_connection()
     cur = conn.cursor()
     games_vals = []
@@ -147,7 +144,7 @@ def store_new_games(games: Dict[int, GameInfo]):
     latest_game_ids: Dict[str, int] = {}
 
     # Now, iterate over all games and convert to tuples for insertion
-    for game in sorted(games.values(), key=lambda game_info: game_info.game_id):
+    for game in sorted(games, key=lambda game_info: game_info.game_id):
         tup = (game.game_id, game.num_players, game.variant_id, game.seed, game.score, game.num_turns, game.datetime_started, game.datetime_ended)
         games_vals.append(tup)
         for player_id in game.user_ids:
@@ -193,7 +190,6 @@ def store_new_games(games: Dict[int, GameInfo]):
 def detailed_fetch_game(game_id: int) -> bool:
     """
     Fetches full game details from the server and stores it in local DB if this game is a league game.
-    @warning: Game data has to be present in database already, game details will then be added.
     @param game_id: Game ID from hanab.live
     @return: Whether the processed game was accepted as a league game, i.e. inserted into the DB
     """
@@ -206,36 +202,20 @@ def detailed_fetch_game(game_id: int) -> bool:
         raise ConnectionError(err_msg)
     game_json = json.loads(response.text)
 
-    instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
-
     game_id = game_json["id"]
     players = game_json["players"]
-    num_players = len(players)
-    seed = game_json["seed"]
-    game_options = game_json.get("options", {})
-    var_name = game_options.get("variant", "No Variant")
 
-    if not utils.are_game_options_allowed(game_id, game_options):
+    games = fetch_games_for_player(players[0], game_id, game_id)
+    if not len(games) == 1:
+        print(games)
+        exit(1)
+    assert len(games) == 1
+    game_entry = process_game_entry(games[0])
+    if game_entry is None:
         return False
 
-    if not utils.is_player_count_allowed(game_id, num_players):
+    # Now make sure that the game exists in the database
-        return False
+    store_new_games([game_entry])
-
-    if not utils.is_game_id_allowed(game_id):
-        return False
-
-    var_id = database.get_variant_id(var_name)
-    if var_id is None:
-        logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
-        return False
-
-    # All game options are ok, now check if the number of players is okay.
-    normalized_usernames = [utils.normalize_username(username) for username in players]
-    user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames)
-    # The return value here is a str if there was an unregistered participant
-    if type(user_ids) is str:
-        logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids))
-        return False
 
     # Now, we can start to actually process the game details
     instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
@@ -244,19 +224,16 @@ def detailed_fetch_game(game_id: int) -> bool:
     for action in actions:
         game.make_action(action)
 
-    conn = conn_manager.get_connection()
-    cur = conn_manager.get_new_cursor()
-    # Note that we assume the game is present in the database already
-
-
     game_participants_vals = []
-    for seat, user_id in enumerate(user_ids):
+    for seat, user_id in enumerate(game_entry.user_ids):
         tup = (game_id, user_id, seat)
         game_participants_vals.append(tup)
 
     # This inserts the game participants now.
     # Note that the participants might already be stored, but not their seat (since we do not know the seat when only
     # getting a game list from the server on fetching the games played by some player)
+    conn = conn_manager.get_connection()
+    cur = conn.cursor()
     psycopg2.extras.execute_values(
         cur,
         "INSERT INTO game_participants "
@@ -271,26 +248,28 @@ def detailed_fetch_game(game_id: int) -> bool:
 
     # It remains to store the seed and action data for this game
     # Note that we store the seed first. This ensures that whenever we have actions stored, we also have the seed stored.
-    games_db_interface.store_deck_for_seed(seed, instance.deck)
+    games_db_interface.store_deck_for_seed(game_entry.seed, instance.deck)
     games_db_interface.store_actions(game_id, actions)
     logger.debug("Fetched all game details of game {}.".format(game_id))
 
     # Do some sanity checks that loading the stored data did not change it
     assert actions == games_db_interface.load_actions(game_id)
-    assert instance.deck == games_db_interface.load_deck(seed)
+    assert instance.deck == games_db_interface.load_deck(game_entry.seed)
     return True
 
 
-def fetch_all_game_details():
+def fetch_all_game_details(force_refresh=False):
     logger.info("Fetching detailed game data for all games.")
     cur = conn_manager.get_new_cursor()
 
-    # Get all games with no actions
+    query = "SELECT DiSTINCT ON (id) id FROM games "                 \
-    cur.execute("SELECT id FROM games "
+            "LEFT OUTER JOIN game_actions"          \
-                "LEFT OUTER JOIN game_actions"
             "  ON games.id = game_actions.game_id"
-                "WHERE game_actions.game_id IS NULL"
+    if not force_refresh:
-                )
+        query += " WHERE game_actions.game_id IS NULL"
+
+    # Get all games
+    cur.execute(query)
     for (game_id,) in cur.fetchall():
         detailed_fetch_game(game_id)