diff --git a/fetch_games.py b/fetch_games.py index 8867ab7..e11b342 100644 --- a/fetch_games.py +++ b/fetch_games.py @@ -28,7 +28,8 @@ session = requests_cache.CachedSession( class GameInfo: - def __init__(self, game_id: int, num_players: int, variant_id: int, seed: str, score: int, num_turns: int, user_ids: List[int], normalized_usernames: List[str]): + def __init__(self, game_id: int, num_players: int, variant_id: int, seed: str, score: int, num_turns: int, + user_ids: List[int], normalized_usernames: List[str]): self.game_id = game_id self.num_players = num_players self.variant_id = variant_id @@ -188,6 +189,7 @@ def detailed_fetch_game(game_id: int) -> bool: @param game_id: Game ID from hanab.live @return: Whether the processed game was accepted as a league game, i.e. inserted into the DB """ + logger.verbose("Fetching details on game {}.".format(game_id)) url = "https://hanab.live/export/{}".format(game_id) response = session.get(url) if not response.status_code == 200: @@ -263,6 +265,36 @@ def detailed_fetch_game(game_id: int) -> bool: conn.commit() # It remains to store the seed and action data for this game + # Note that we store the seed first. This ensures that whenever we have actions stored, we also have the seed stored. games_db_interface.store_deck_for_seed(seed, instance.deck) games_db_interface.store_actions(game_id, actions) + logger.debug("Fetched all game details of game {}.".format(game_id)) return True + + +def fetch_all_game_details(): + logger.info("Fetching detailed game data for all games.") + cur = conn_manager.get_new_cursor() + + # Get all games with no actions + cur.execute("SELECT id FROM games " + "LEFT OUTER JOIN game_actions" + " ON games.id = game_actions.game_id " + "WHERE game_actions.game_id IS NULL" + ) + for (game_id,) in cur.fetchall(): + detailed_fetch_game(game_id) + + # Just to make sure, check that we have data on all seeds + cur.execute("SELECT id FROM games " + "LEFT OUTER JOIN seeds" + " ON seeds.seed = games.seed " + "WHERE seeds.seed IS NULL" + ) + games_without_seeds = [game_id for (game_id,) in cur.fetchall()] + if len(games_without_seeds) != 0: + logger.warn("Detected the following games without seed data, but with actions already stored: {}." + "Fetching them now".format(", ".join(games_without_seeds)) + ) + for game_id in games_without_seeds: + detailed_fetch_game(game_id)