From 78bdd8397ce76a6d2c8a5f4ff22c6d07e3263a57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Ke=C3=9Fler?= Date: Tue, 26 Dec 2023 17:19:02 +0100 Subject: [PATCH] rework game fetching: deduplicate code, fetch timestamps --- src/database.py | 2 +- src/fetch_games.py | 125 +++++++++++++++++++-------------------------- 2 files changed, 53 insertions(+), 74 deletions(-) diff --git a/src/database.py b/src/database.py index cf7efea..b18127a 100644 --- a/src/database.py +++ b/src/database.py @@ -246,7 +246,7 @@ def add_player(user_name: str, player_name: str, base_rating: Optional[int] = No init_player_base_rating(player_name, base_rating) -def get_variant_ids(): +def get_variant_ids() -> List[int]: cur = conn_manager.get_new_cursor() cur.execute("SELECT id FROM variants") return [var_id for (var_id,) in cur.fetchall()] diff --git a/src/fetch_games.py b/src/fetch_games.py index a6c8df8..ac4fd37 100644 --- a/src/fetch_games.py +++ b/src/fetch_games.py @@ -43,9 +43,12 @@ class GameInfo: datetime_ended: str -def fetch_games_for_player(username: str, latest_game_id: int): - logger.verbose("Fetching games for username {} more recent than id {}".format(username, latest_game_id)) - url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, latest_game_id + 1) +def fetch_games_for_player(username: str, first_game_id: int, last_game_id: Optional[int] = None) -> Dict: + logger.verbose("Fetching games for username {} more recent than id {}".format(username, first_game_id)) + url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, first_game_id) + if last_game_id is not None: + url += "&end={}".format(last_game_id) + print(url) response = session.get(url) if not response.status_code == 200: err_msg = "Failed to fetch games for username {}, requested URL {}".format(username, url) @@ -54,9 +57,8 @@ def fetch_games_for_player(username: str, latest_game_id: int): return json.loads(response.text) -def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[int]) -> Optional[GameInfo]: +def process_game_entry(game_json: Dict) -> Optional[GameInfo]: logger.debug("Processing entry {}".format(game_json)) - config = config_manager.get_config() # Parse game properties game_id = game_json["id"] @@ -72,7 +74,6 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i var_id = game_options["variantID"] normalized_usernames = [utils.normalize_username(username) for username in players] - # Now, check if the game is one that we accept for league if not all([ @@ -83,63 +84,59 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i ]): return - if var_id not in variant_ids: + database.get_variant_ids() + + if var_id not in database.get_variant_ids(): logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id)) return # Everything matches, so we can parse the participants now - user_ids = [] # This will be a list of the (league specific) user_id's that played this game. - for normalized_username in normalized_usernames: - user_id = username_dict.get(normalized_username, None) - if user_id is None: - logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, normalized_username)) - return - user_ids.append(user_id) + user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames) + # The return value here is a str if there was an unregistered participant + if type(user_ids) is str: + logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids)) + return return GameInfo(game_id, num_players, var_id, seed, score, num_turns, user_ids, normalized_usernames, start_time, end_time) -def fetch_games_for_all_players(): +def fetch_games_for_all_players() -> List[GameInfo]: logger.info("Fetching new games.") cur = conn_manager.get_new_cursor() - cur.execute("SELECT user_accounts.normalized_username, user_accounts.user_id, downloads.latest_game_id " + cur.execute("SELECT" + " user_accounts.normalized_username," + " user_accounts.user_id," + # Use the starting id as fallback if we have not downloaded any games for this player yet. + " COALESCE(downloads.latest_game_id, %s) " "FROM user_accounts " "LEFT OUTER JOIN downloads " - " ON user_accounts.normalized_username = downloads.normalized_username" + " ON user_accounts.normalized_username = downloads.normalized_username", + (config_manager.get_config().starting_game_id,) ) - # This will be a mapping of normalized username -> user ID that we built from the DB data - username_dict = {} # This will be a mapping of game id -> JSON data that we get from hanab.live, where we will collect all the # possibly relevant games now games: Dict[int, Dict] = {} for username, user_id, latest_game_id in cur.fetchall(): - username_dict[username] = user_id - # Use the starting id as fallback if we have not downloaded any games for this player yet. - if latest_game_id is None: - latest_game_id = config_manager.get_config().starting_game_id - - player_games = fetch_games_for_player(username, latest_game_id) + player_games = fetch_games_for_player(username, latest_game_id + 1) for game in player_games: games[game['id']] = game logger.info("Found {} potential league game(s) to process.".format(len(games))) - allowed_variants = database.get_variant_ids() - # This will hold the processed games that we will add to the database. - good_games: Dict[int, GameInfo] = {} + good_games: List[GameInfo] = [] for game_id, game in games.items(): - game_info = process_game_entry(game, username_dict, allowed_variants) + game_info = process_game_entry(game) if game_info is not None: - good_games[game_id] = game_info + good_games.append(game_info) logger.verbose("Found {} valid league game(s).".format(len(good_games))) return good_games -def store_new_games(games: Dict[int, GameInfo]): +def store_new_games(games: List[GameInfo]): conn = conn_manager.get_connection() cur = conn.cursor() games_vals = [] @@ -147,7 +144,7 @@ def store_new_games(games: Dict[int, GameInfo]): latest_game_ids: Dict[str, int] = {} # Now, iterate over all games and convert to tuples for insertion - for game in sorted(games.values(), key=lambda game_info: game_info.game_id): + for game in sorted(games, key=lambda game_info: game_info.game_id): tup = (game.game_id, game.num_players, game.variant_id, game.seed, game.score, game.num_turns, game.datetime_started, game.datetime_ended) games_vals.append(tup) for player_id in game.user_ids: @@ -193,7 +190,6 @@ def store_new_games(games: Dict[int, GameInfo]): def detailed_fetch_game(game_id: int) -> bool: """ Fetches full game details from the server and stores it in local DB if this game is a league game. - @warning: Game data has to be present in database already, game details will then be added. @param game_id: Game ID from hanab.live @return: Whether the processed game was accepted as a league game, i.e. inserted into the DB """ @@ -206,36 +202,20 @@ def detailed_fetch_game(game_id: int) -> bool: raise ConnectionError(err_msg) game_json = json.loads(response.text) - instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False) - game_id = game_json["id"] players = game_json["players"] - num_players = len(players) - seed = game_json["seed"] - game_options = game_json.get("options", {}) - var_name = game_options.get("variant", "No Variant") - if not utils.are_game_options_allowed(game_id, game_options): + games = fetch_games_for_player(players[0], game_id, game_id) + if not len(games) == 1: + print(games) + exit(1) + assert len(games) == 1 + game_entry = process_game_entry(games[0]) + if game_entry is None: return False - if not utils.is_player_count_allowed(game_id, num_players): - return False - - if not utils.is_game_id_allowed(game_id): - return False - - var_id = database.get_variant_id(var_name) - if var_id is None: - logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id)) - return False - - # All game options are ok, now check if the number of players is okay. - normalized_usernames = [utils.normalize_username(username) for username in players] - user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames) - # The return value here is a str if there was an unregistered participant - if type(user_ids) is str: - logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids)) - return False + # Now make sure that the game exists in the database + store_new_games([game_entry]) # Now, we can start to actually process the game details instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False) @@ -244,19 +224,16 @@ def detailed_fetch_game(game_id: int) -> bool: for action in actions: game.make_action(action) - conn = conn_manager.get_connection() - cur = conn_manager.get_new_cursor() - # Note that we assume the game is present in the database already - - game_participants_vals = [] - for seat, user_id in enumerate(user_ids): + for seat, user_id in enumerate(game_entry.user_ids): tup = (game_id, user_id, seat) game_participants_vals.append(tup) # This inserts the game participants now. # Note that the participants might already be stored, but not their seat (since we do not know the seat when only # getting a game list from the server on fetching the games played by some player) + conn = conn_manager.get_connection() + cur = conn.cursor() psycopg2.extras.execute_values( cur, "INSERT INTO game_participants " @@ -271,26 +248,28 @@ def detailed_fetch_game(game_id: int) -> bool: # It remains to store the seed and action data for this game # Note that we store the seed first. This ensures that whenever we have actions stored, we also have the seed stored. - games_db_interface.store_deck_for_seed(seed, instance.deck) + games_db_interface.store_deck_for_seed(game_entry.seed, instance.deck) games_db_interface.store_actions(game_id, actions) logger.debug("Fetched all game details of game {}.".format(game_id)) # Do some sanity checks that loading the stored data did not change it assert actions == games_db_interface.load_actions(game_id) - assert instance.deck == games_db_interface.load_deck(seed) + assert instance.deck == games_db_interface.load_deck(game_entry.seed) return True -def fetch_all_game_details(): +def fetch_all_game_details(force_refresh=False): logger.info("Fetching detailed game data for all games.") cur = conn_manager.get_new_cursor() - # Get all games with no actions - cur.execute("SELECT id FROM games " - "LEFT OUTER JOIN game_actions" - " ON games.id = game_actions.game_id " - "WHERE game_actions.game_id IS NULL" - ) + query = "SELECT DiSTINCT ON (id) id FROM games " \ + "LEFT OUTER JOIN game_actions" \ + " ON games.id = game_actions.game_id" + if not force_refresh: + query += " WHERE game_actions.game_id IS NULL" + + # Get all games + cur.execute(query) for (game_id,) in cur.fetchall(): detailed_fetch_game(game_id)