rework game fetching: deduplicate code, fetch timestamps

This commit is contained in:
Maximilian Keßler 2023-12-26 17:19:02 +01:00
parent e01b7fe823
commit 78bdd8397c
Signed by: max
GPG key ID: BCC5A619923C0BA5
2 changed files with 53 additions and 74 deletions

View file

@ -246,7 +246,7 @@ def add_player(user_name: str, player_name: str, base_rating: Optional[int] = No
init_player_base_rating(player_name, base_rating) init_player_base_rating(player_name, base_rating)
def get_variant_ids(): def get_variant_ids() -> List[int]:
cur = conn_manager.get_new_cursor() cur = conn_manager.get_new_cursor()
cur.execute("SELECT id FROM variants") cur.execute("SELECT id FROM variants")
return [var_id for (var_id,) in cur.fetchall()] return [var_id for (var_id,) in cur.fetchall()]

View file

@ -43,9 +43,12 @@ class GameInfo:
datetime_ended: str datetime_ended: str
def fetch_games_for_player(username: str, latest_game_id: int): def fetch_games_for_player(username: str, first_game_id: int, last_game_id: Optional[int] = None) -> Dict:
logger.verbose("Fetching games for username {} more recent than id {}".format(username, latest_game_id)) logger.verbose("Fetching games for username {} more recent than id {}".format(username, first_game_id))
url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, latest_game_id + 1) url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, first_game_id)
if last_game_id is not None:
url += "&end={}".format(last_game_id)
print(url)
response = session.get(url) response = session.get(url)
if not response.status_code == 200: if not response.status_code == 200:
err_msg = "Failed to fetch games for username {}, requested URL {}".format(username, url) err_msg = "Failed to fetch games for username {}, requested URL {}".format(username, url)
@ -54,9 +57,8 @@ def fetch_games_for_player(username: str, latest_game_id: int):
return json.loads(response.text) return json.loads(response.text)
def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[int]) -> Optional[GameInfo]: def process_game_entry(game_json: Dict) -> Optional[GameInfo]:
logger.debug("Processing entry {}".format(game_json)) logger.debug("Processing entry {}".format(game_json))
config = config_manager.get_config()
# Parse game properties # Parse game properties
game_id = game_json["id"] game_id = game_json["id"]
@ -72,7 +74,6 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i
var_id = game_options["variantID"] var_id = game_options["variantID"]
normalized_usernames = [utils.normalize_username(username) for username in players] normalized_usernames = [utils.normalize_username(username) for username in players]
# Now, check if the game is one that we accept for league # Now, check if the game is one that we accept for league
if not all([ if not all([
@ -83,63 +84,59 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i
]): ]):
return return
if var_id not in variant_ids: database.get_variant_ids()
if var_id not in database.get_variant_ids():
logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id)) logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
return return
# Everything matches, so we can parse the participants now # Everything matches, so we can parse the participants now
user_ids = [] # This will be a list of the (league specific) user_id's that played this game. user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames)
for normalized_username in normalized_usernames: # The return value here is a str if there was an unregistered participant
user_id = username_dict.get(normalized_username, None) if type(user_ids) is str:
if user_id is None: logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids))
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, normalized_username)) return
return
user_ids.append(user_id)
return GameInfo(game_id, num_players, var_id, seed, score, num_turns, user_ids, normalized_usernames, start_time, end_time) return GameInfo(game_id, num_players, var_id, seed, score, num_turns, user_ids, normalized_usernames, start_time, end_time)
def fetch_games_for_all_players(): def fetch_games_for_all_players() -> List[GameInfo]:
logger.info("Fetching new games.") logger.info("Fetching new games.")
cur = conn_manager.get_new_cursor() cur = conn_manager.get_new_cursor()
cur.execute("SELECT user_accounts.normalized_username, user_accounts.user_id, downloads.latest_game_id " cur.execute("SELECT"
" user_accounts.normalized_username,"
" user_accounts.user_id,"
# Use the starting id as fallback if we have not downloaded any games for this player yet.
" COALESCE(downloads.latest_game_id, %s) "
"FROM user_accounts " "FROM user_accounts "
"LEFT OUTER JOIN downloads " "LEFT OUTER JOIN downloads "
" ON user_accounts.normalized_username = downloads.normalized_username" " ON user_accounts.normalized_username = downloads.normalized_username",
(config_manager.get_config().starting_game_id,)
) )
# This will be a mapping of normalized username -> user ID that we built from the DB data
username_dict = {}
# This will be a mapping of game id -> JSON data that we get from hanab.live, where we will collect all the # This will be a mapping of game id -> JSON data that we get from hanab.live, where we will collect all the
# possibly relevant games now # possibly relevant games now
games: Dict[int, Dict] = {} games: Dict[int, Dict] = {}
for username, user_id, latest_game_id in cur.fetchall(): for username, user_id, latest_game_id in cur.fetchall():
username_dict[username] = user_id player_games = fetch_games_for_player(username, latest_game_id + 1)
# Use the starting id as fallback if we have not downloaded any games for this player yet.
if latest_game_id is None:
latest_game_id = config_manager.get_config().starting_game_id
player_games = fetch_games_for_player(username, latest_game_id)
for game in player_games: for game in player_games:
games[game['id']] = game games[game['id']] = game
logger.info("Found {} potential league game(s) to process.".format(len(games))) logger.info("Found {} potential league game(s) to process.".format(len(games)))
allowed_variants = database.get_variant_ids()
# This will hold the processed games that we will add to the database. # This will hold the processed games that we will add to the database.
good_games: Dict[int, GameInfo] = {} good_games: List[GameInfo] = []
for game_id, game in games.items(): for game_id, game in games.items():
game_info = process_game_entry(game, username_dict, allowed_variants) game_info = process_game_entry(game)
if game_info is not None: if game_info is not None:
good_games[game_id] = game_info good_games.append(game_info)
logger.verbose("Found {} valid league game(s).".format(len(good_games))) logger.verbose("Found {} valid league game(s).".format(len(good_games)))
return good_games return good_games
def store_new_games(games: Dict[int, GameInfo]): def store_new_games(games: List[GameInfo]):
conn = conn_manager.get_connection() conn = conn_manager.get_connection()
cur = conn.cursor() cur = conn.cursor()
games_vals = [] games_vals = []
@ -147,7 +144,7 @@ def store_new_games(games: Dict[int, GameInfo]):
latest_game_ids: Dict[str, int] = {} latest_game_ids: Dict[str, int] = {}
# Now, iterate over all games and convert to tuples for insertion # Now, iterate over all games and convert to tuples for insertion
for game in sorted(games.values(), key=lambda game_info: game_info.game_id): for game in sorted(games, key=lambda game_info: game_info.game_id):
tup = (game.game_id, game.num_players, game.variant_id, game.seed, game.score, game.num_turns, game.datetime_started, game.datetime_ended) tup = (game.game_id, game.num_players, game.variant_id, game.seed, game.score, game.num_turns, game.datetime_started, game.datetime_ended)
games_vals.append(tup) games_vals.append(tup)
for player_id in game.user_ids: for player_id in game.user_ids:
@ -193,7 +190,6 @@ def store_new_games(games: Dict[int, GameInfo]):
def detailed_fetch_game(game_id: int) -> bool: def detailed_fetch_game(game_id: int) -> bool:
""" """
Fetches full game details from the server and stores it in local DB if this game is a league game. Fetches full game details from the server and stores it in local DB if this game is a league game.
@warning: Game data has to be present in database already, game details will then be added.
@param game_id: Game ID from hanab.live @param game_id: Game ID from hanab.live
@return: Whether the processed game was accepted as a league game, i.e. inserted into the DB @return: Whether the processed game was accepted as a league game, i.e. inserted into the DB
""" """
@ -206,36 +202,20 @@ def detailed_fetch_game(game_id: int) -> bool:
raise ConnectionError(err_msg) raise ConnectionError(err_msg)
game_json = json.loads(response.text) game_json = json.loads(response.text)
instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
game_id = game_json["id"] game_id = game_json["id"]
players = game_json["players"] players = game_json["players"]
num_players = len(players)
seed = game_json["seed"]
game_options = game_json.get("options", {})
var_name = game_options.get("variant", "No Variant")
if not utils.are_game_options_allowed(game_id, game_options): games = fetch_games_for_player(players[0], game_id, game_id)
if not len(games) == 1:
print(games)
exit(1)
assert len(games) == 1
game_entry = process_game_entry(games[0])
if game_entry is None:
return False return False
if not utils.is_player_count_allowed(game_id, num_players): # Now make sure that the game exists in the database
return False store_new_games([game_entry])
if not utils.is_game_id_allowed(game_id):
return False
var_id = database.get_variant_id(var_name)
if var_id is None:
logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
return False
# All game options are ok, now check if the number of players is okay.
normalized_usernames = [utils.normalize_username(username) for username in players]
user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames)
# The return value here is a str if there was an unregistered participant
if type(user_ids) is str:
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids))
return False
# Now, we can start to actually process the game details # Now, we can start to actually process the game details
instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False) instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
@ -244,19 +224,16 @@ def detailed_fetch_game(game_id: int) -> bool:
for action in actions: for action in actions:
game.make_action(action) game.make_action(action)
conn = conn_manager.get_connection()
cur = conn_manager.get_new_cursor()
# Note that we assume the game is present in the database already
game_participants_vals = [] game_participants_vals = []
for seat, user_id in enumerate(user_ids): for seat, user_id in enumerate(game_entry.user_ids):
tup = (game_id, user_id, seat) tup = (game_id, user_id, seat)
game_participants_vals.append(tup) game_participants_vals.append(tup)
# This inserts the game participants now. # This inserts the game participants now.
# Note that the participants might already be stored, but not their seat (since we do not know the seat when only # Note that the participants might already be stored, but not their seat (since we do not know the seat when only
# getting a game list from the server on fetching the games played by some player) # getting a game list from the server on fetching the games played by some player)
conn = conn_manager.get_connection()
cur = conn.cursor()
psycopg2.extras.execute_values( psycopg2.extras.execute_values(
cur, cur,
"INSERT INTO game_participants " "INSERT INTO game_participants "
@ -271,26 +248,28 @@ def detailed_fetch_game(game_id: int) -> bool:
# It remains to store the seed and action data for this game # It remains to store the seed and action data for this game
# Note that we store the seed first. This ensures that whenever we have actions stored, we also have the seed stored. # Note that we store the seed first. This ensures that whenever we have actions stored, we also have the seed stored.
games_db_interface.store_deck_for_seed(seed, instance.deck) games_db_interface.store_deck_for_seed(game_entry.seed, instance.deck)
games_db_interface.store_actions(game_id, actions) games_db_interface.store_actions(game_id, actions)
logger.debug("Fetched all game details of game {}.".format(game_id)) logger.debug("Fetched all game details of game {}.".format(game_id))
# Do some sanity checks that loading the stored data did not change it # Do some sanity checks that loading the stored data did not change it
assert actions == games_db_interface.load_actions(game_id) assert actions == games_db_interface.load_actions(game_id)
assert instance.deck == games_db_interface.load_deck(seed) assert instance.deck == games_db_interface.load_deck(game_entry.seed)
return True return True
def fetch_all_game_details(): def fetch_all_game_details(force_refresh=False):
logger.info("Fetching detailed game data for all games.") logger.info("Fetching detailed game data for all games.")
cur = conn_manager.get_new_cursor() cur = conn_manager.get_new_cursor()
# Get all games with no actions query = "SELECT DiSTINCT ON (id) id FROM games " \
cur.execute("SELECT id FROM games " "LEFT OUTER JOIN game_actions" \
"LEFT OUTER JOIN game_actions" " ON games.id = game_actions.game_id"
" ON games.id = game_actions.game_id " if not force_refresh:
"WHERE game_actions.game_id IS NULL" query += " WHERE game_actions.game_id IS NULL"
)
# Get all games
cur.execute(query)
for (game_id,) in cur.fetchall(): for (game_id,) in cur.fetchall():
detailed_fetch_game(game_id) detailed_fetch_game(game_id)