Hanabi/hanabi-league
1
0
Fork 2
Code Issues Pull requests Projects Releases Packages Wiki Activity

rework game fetching: deduplicate code, fetch timestamps

Browse source
This commit is contained in:
Maximilian Keßler 2023-12-26 17:19:02 +01:00
parent e01b7fe823
commit 78bdd8397c
Signed by: max
GPG key ID: BCC5A619923C0BA5
2 changed files with 53 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/database.py
View file

@ -246,7 +246,7 @@ def add_player(user_name: str, player_name: str, base_rating: Optional[int] = No
init_player_base_rating(player_name, base_rating)
def get_variant_ids():
def get_variant_ids() -> List[int]:
cur = conn_manager.get_new_cursor()
cur.execute("SELECT id FROM variants")
return [var_id for (var_id,) in cur.fetchall()]

123
src/fetch_games.py
View file

@ -43,9 +43,12 @@ class GameInfo:
datetime_ended: str
def fetch_games_for_player(username: str, latest_game_id: int):
logger.verbose("Fetching games for username {} more recent than id {}".format(username, latest_game_id))
url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, latest_game_id + 1)
def fetch_games_for_player(username: str, first_game_id: int, last_game_id: Optional[int] = None) -> Dict:
logger.verbose("Fetching games for username {} more recent than id {}".format(username, first_game_id))
url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, first_game_id)
if last_game_id is not None:
url += "&end={}".format(last_game_id)
print(url)
response = session.get(url)
if not response.status_code == 200:
err_msg = "Failed to fetch games for username {}, requested URL {}".format(username, url)
@ -54,9 +57,8 @@ def fetch_games_for_player(username: str, latest_game_id: int):
return json.loads(response.text)
def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[int]) -> Optional[GameInfo]:
def process_game_entry(game_json: Dict) -> Optional[GameInfo]:
logger.debug("Processing entry {}".format(game_json))
config = config_manager.get_config()
# Parse game properties
game_id = game_json["id"]
@ -72,7 +74,6 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i
var_id = game_options["variantID"]
normalized_usernames = [utils.normalize_username(username) for username in players]
# Now, check if the game is one that we accept for league
if not all([
@ -83,63 +84,59 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i
]):
return
if var_id not in variant_ids:
database.get_variant_ids()
if var_id not in database.get_variant_ids():
logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
return
# Everything matches, so we can parse the participants now
user_ids = [] # This will be a list of the (league specific) user_id's that played this game.
for normalized_username in normalized_usernames:
user_id = username_dict.get(normalized_username, None)
if user_id is None:
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, normalized_username))
user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames)
# The return value here is a str if there was an unregistered participant
if type(user_ids) is str:
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids))
return
user_ids.append(user_id)
return GameInfo(game_id, num_players, var_id, seed, score, num_turns, user_ids, normalized_usernames, start_time, end_time)
def fetch_games_for_all_players():
def fetch_games_for_all_players() -> List[GameInfo]:
logger.info("Fetching new games.")
cur = conn_manager.get_new_cursor()
cur.execute("SELECT user_accounts.normalized_username, user_accounts.user_id, downloads.latest_game_id "
cur.execute("SELECT"
" user_accounts.normalized_username,"
" user_accounts.user_id,"
# Use the starting id as fallback if we have not downloaded any games for this player yet.
" COALESCE(downloads.latest_game_id, %s) "
"FROM user_accounts "
"LEFT OUTER JOIN downloads "
" ON user_accounts.normalized_username = downloads.normalized_username"
" ON user_accounts.normalized_username = downloads.normalized_username",
(config_manager.get_config().starting_game_id,)
)
# This will be a mapping of normalized username -> user ID that we built from the DB data
username_dict = {}
# This will be a mapping of game id -> JSON data that we get from hanab.live, where we will collect all the
# possibly relevant games now
games: Dict[int, Dict] = {}
for username, user_id, latest_game_id in cur.fetchall():
username_dict[username] = user_id
# Use the starting id as fallback if we have not downloaded any games for this player yet.
if latest_game_id is None:
latest_game_id = config_manager.get_config().starting_game_id
player_games = fetch_games_for_player(username, latest_game_id)
player_games = fetch_games_for_player(username, latest_game_id + 1)
for game in player_games:
games[game['id']] = game
logger.info("Found {} potential league game(s) to process.".format(len(games)))
allowed_variants = database.get_variant_ids()
# This will hold the processed games that we will add to the database.
good_games: Dict[int, GameInfo] = {}
good_games: List[GameInfo] = []
for game_id, game in games.items():
game_info = process_game_entry(game, username_dict, allowed_variants)
game_info = process_game_entry(game)
if game_info is not None:
good_games[game_id] = game_info
good_games.append(game_info)
logger.verbose("Found {} valid league game(s).".format(len(good_games)))
return good_games
def store_new_games(games: Dict[int, GameInfo]):
def store_new_games(games: List[GameInfo]):
conn = conn_manager.get_connection()
cur = conn.cursor()
games_vals = []
@ -147,7 +144,7 @@ def store_new_games(games: Dict[int, GameInfo]):
latest_game_ids: Dict[str, int] = {}
# Now, iterate over all games and convert to tuples for insertion
for game in sorted(games.values(), key=lambda game_info: game_info.game_id):
for game in sorted(games, key=lambda game_info: game_info.game_id):
tup = (game.game_id, game.num_players, game.variant_id, game.seed, game.score, game.num_turns, game.datetime_started, game.datetime_ended)
games_vals.append(tup)
for player_id in game.user_ids:
@ -193,7 +190,6 @@ def store_new_games(games: Dict[int, GameInfo]):
def detailed_fetch_game(game_id: int) -> bool:
"""
Fetches full game details from the server and stores it in local DB if this game is a league game.
@warning: Game data has to be present in database already, game details will then be added.
@param game_id: Game ID from hanab.live
@return: Whether the processed game was accepted as a league game, i.e. inserted into the DB
"""
@ -206,36 +202,20 @@ def detailed_fetch_game(game_id: int) -> bool:
raise ConnectionError(err_msg)
game_json = json.loads(response.text)
instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
game_id = game_json["id"]
players = game_json["players"]
num_players = len(players)
seed = game_json["seed"]
game_options = game_json.get("options", {})
var_name = game_options.get("variant", "No Variant")
if not utils.are_game_options_allowed(game_id, game_options):
games = fetch_games_for_player(players[0], game_id, game_id)
if not len(games) == 1:
print(games)
exit(1)
assert len(games) == 1
game_entry = process_game_entry(games[0])
if game_entry is None:
return False
if not utils.is_player_count_allowed(game_id, num_players):
return False
if not utils.is_game_id_allowed(game_id):
return False
var_id = database.get_variant_id(var_name)
if var_id is None:
logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
return False
# All game options are ok, now check if the number of players is okay.
normalized_usernames = [utils.normalize_username(username) for username in players]
user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames)
# The return value here is a str if there was an unregistered participant
if type(user_ids) is str:
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids))
return False
# Now make sure that the game exists in the database
store_new_games([game_entry])
# Now, we can start to actually process the game details
instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
@ -244,19 +224,16 @@ def detailed_fetch_game(game_id: int) -> bool:
for action in actions:
game.make_action(action)
conn = conn_manager.get_connection()
cur = conn_manager.get_new_cursor()
# Note that we assume the game is present in the database already
game_participants_vals = []
for seat, user_id in enumerate(user_ids):
for seat, user_id in enumerate(game_entry.user_ids):
tup = (game_id, user_id, seat)
game_participants_vals.append(tup)
# This inserts the game participants now.
# Note that the participants might already be stored, but not their seat (since we do not know the seat when only
# getting a game list from the server on fetching the games played by some player)
conn = conn_manager.get_connection()
cur = conn.cursor()
psycopg2.extras.execute_values(
cur,
"INSERT INTO game_participants "
@ -271,26 +248,28 @@ def detailed_fetch_game(game_id: int) -> bool:
# It remains to store the seed and action data for this game
# Note that we store the seed first. This ensures that whenever we have actions stored, we also have the seed stored.
games_db_interface.store_deck_for_seed(seed, instance.deck)
games_db_interface.store_deck_for_seed(game_entry.seed, instance.deck)
games_db_interface.store_actions(game_id, actions)
logger.debug("Fetched all game details of game {}.".format(game_id))
# Do some sanity checks that loading the stored data did not change it
assert actions == games_db_interface.load_actions(game_id)
assert instance.deck == games_db_interface.load_deck(seed)
assert instance.deck == games_db_interface.load_deck(game_entry.seed)
return True
def fetch_all_game_details():
def fetch_all_game_details(force_refresh=False):
logger.info("Fetching detailed game data for all games.")
cur = conn_manager.get_new_cursor()
# Get all games with no actions
cur.execute("SELECT id FROM games "
"LEFT OUTER JOIN game_actions"
" ON games.id = game_actions.game_id "
"WHERE game_actions.game_id IS NULL"
)
query = "SELECT DiSTINCT ON (id) id FROM games " \
"LEFT OUTER JOIN game_actions" \
" ON games.id = game_actions.game_id"
if not force_refresh:
query += " WHERE game_actions.game_id IS NULL"
# Get all games
cur.execute(query)
for (game_id,) in cur.fetchall():
detailed_fetch_game(game_id)