forked from Hanabi/hanabi-league
rework game fetching: deduplicate code, fetch timestamps
This commit is contained in:
parent
e01b7fe823
commit
78bdd8397c
2 changed files with 53 additions and 74 deletions
|
@ -246,7 +246,7 @@ def add_player(user_name: str, player_name: str, base_rating: Optional[int] = No
|
|||
init_player_base_rating(player_name, base_rating)
|
||||
|
||||
|
||||
def get_variant_ids():
|
||||
def get_variant_ids() -> List[int]:
|
||||
cur = conn_manager.get_new_cursor()
|
||||
cur.execute("SELECT id FROM variants")
|
||||
return [var_id for (var_id,) in cur.fetchall()]
|
||||
|
|
|
@ -43,9 +43,12 @@ class GameInfo:
|
|||
datetime_ended: str
|
||||
|
||||
|
||||
def fetch_games_for_player(username: str, latest_game_id: int):
|
||||
logger.verbose("Fetching games for username {} more recent than id {}".format(username, latest_game_id))
|
||||
url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, latest_game_id + 1)
|
||||
def fetch_games_for_player(username: str, first_game_id: int, last_game_id: Optional[int] = None) -> Dict:
|
||||
logger.verbose("Fetching games for username {} more recent than id {}".format(username, first_game_id))
|
||||
url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, first_game_id)
|
||||
if last_game_id is not None:
|
||||
url += "&end={}".format(last_game_id)
|
||||
print(url)
|
||||
response = session.get(url)
|
||||
if not response.status_code == 200:
|
||||
err_msg = "Failed to fetch games for username {}, requested URL {}".format(username, url)
|
||||
|
@ -54,9 +57,8 @@ def fetch_games_for_player(username: str, latest_game_id: int):
|
|||
return json.loads(response.text)
|
||||
|
||||
|
||||
def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[int]) -> Optional[GameInfo]:
|
||||
def process_game_entry(game_json: Dict) -> Optional[GameInfo]:
|
||||
logger.debug("Processing entry {}".format(game_json))
|
||||
config = config_manager.get_config()
|
||||
|
||||
# Parse game properties
|
||||
game_id = game_json["id"]
|
||||
|
@ -72,7 +74,6 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i
|
|||
var_id = game_options["variantID"]
|
||||
|
||||
normalized_usernames = [utils.normalize_username(username) for username in players]
|
||||
|
||||
# Now, check if the game is one that we accept for league
|
||||
|
||||
if not all([
|
||||
|
@ -83,63 +84,59 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i
|
|||
]):
|
||||
return
|
||||
|
||||
if var_id not in variant_ids:
|
||||
database.get_variant_ids()
|
||||
|
||||
if var_id not in database.get_variant_ids():
|
||||
logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
|
||||
return
|
||||
|
||||
# Everything matches, so we can parse the participants now
|
||||
user_ids = [] # This will be a list of the (league specific) user_id's that played this game.
|
||||
for normalized_username in normalized_usernames:
|
||||
user_id = username_dict.get(normalized_username, None)
|
||||
if user_id is None:
|
||||
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, normalized_username))
|
||||
return
|
||||
user_ids.append(user_id)
|
||||
user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames)
|
||||
# The return value here is a str if there was an unregistered participant
|
||||
if type(user_ids) is str:
|
||||
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids))
|
||||
return
|
||||
|
||||
return GameInfo(game_id, num_players, var_id, seed, score, num_turns, user_ids, normalized_usernames, start_time, end_time)
|
||||
|
||||
|
||||
def fetch_games_for_all_players():
|
||||
def fetch_games_for_all_players() -> List[GameInfo]:
|
||||
logger.info("Fetching new games.")
|
||||
cur = conn_manager.get_new_cursor()
|
||||
cur.execute("SELECT user_accounts.normalized_username, user_accounts.user_id, downloads.latest_game_id "
|
||||
cur.execute("SELECT"
|
||||
" user_accounts.normalized_username,"
|
||||
" user_accounts.user_id,"
|
||||
# Use the starting id as fallback if we have not downloaded any games for this player yet.
|
||||
" COALESCE(downloads.latest_game_id, %s) "
|
||||
"FROM user_accounts "
|
||||
"LEFT OUTER JOIN downloads "
|
||||
" ON user_accounts.normalized_username = downloads.normalized_username"
|
||||
" ON user_accounts.normalized_username = downloads.normalized_username",
|
||||
(config_manager.get_config().starting_game_id,)
|
||||
)
|
||||
# This will be a mapping of normalized username -> user ID that we built from the DB data
|
||||
username_dict = {}
|
||||
|
||||
# This will be a mapping of game id -> JSON data that we get from hanab.live, where we will collect all the
|
||||
# possibly relevant games now
|
||||
games: Dict[int, Dict] = {}
|
||||
for username, user_id, latest_game_id in cur.fetchall():
|
||||
username_dict[username] = user_id
|
||||
# Use the starting id as fallback if we have not downloaded any games for this player yet.
|
||||
if latest_game_id is None:
|
||||
latest_game_id = config_manager.get_config().starting_game_id
|
||||
|
||||
player_games = fetch_games_for_player(username, latest_game_id)
|
||||
player_games = fetch_games_for_player(username, latest_game_id + 1)
|
||||
for game in player_games:
|
||||
games[game['id']] = game
|
||||
|
||||
logger.info("Found {} potential league game(s) to process.".format(len(games)))
|
||||
|
||||
allowed_variants = database.get_variant_ids()
|
||||
|
||||
# This will hold the processed games that we will add to the database.
|
||||
good_games: Dict[int, GameInfo] = {}
|
||||
good_games: List[GameInfo] = []
|
||||
|
||||
for game_id, game in games.items():
|
||||
game_info = process_game_entry(game, username_dict, allowed_variants)
|
||||
game_info = process_game_entry(game)
|
||||
if game_info is not None:
|
||||
good_games[game_id] = game_info
|
||||
good_games.append(game_info)
|
||||
|
||||
logger.verbose("Found {} valid league game(s).".format(len(good_games)))
|
||||
return good_games
|
||||
|
||||
|
||||
def store_new_games(games: Dict[int, GameInfo]):
|
||||
def store_new_games(games: List[GameInfo]):
|
||||
conn = conn_manager.get_connection()
|
||||
cur = conn.cursor()
|
||||
games_vals = []
|
||||
|
@ -147,7 +144,7 @@ def store_new_games(games: Dict[int, GameInfo]):
|
|||
latest_game_ids: Dict[str, int] = {}
|
||||
|
||||
# Now, iterate over all games and convert to tuples for insertion
|
||||
for game in sorted(games.values(), key=lambda game_info: game_info.game_id):
|
||||
for game in sorted(games, key=lambda game_info: game_info.game_id):
|
||||
tup = (game.game_id, game.num_players, game.variant_id, game.seed, game.score, game.num_turns, game.datetime_started, game.datetime_ended)
|
||||
games_vals.append(tup)
|
||||
for player_id in game.user_ids:
|
||||
|
@ -193,7 +190,6 @@ def store_new_games(games: Dict[int, GameInfo]):
|
|||
def detailed_fetch_game(game_id: int) -> bool:
|
||||
"""
|
||||
Fetches full game details from the server and stores it in local DB if this game is a league game.
|
||||
@warning: Game data has to be present in database already, game details will then be added.
|
||||
@param game_id: Game ID from hanab.live
|
||||
@return: Whether the processed game was accepted as a league game, i.e. inserted into the DB
|
||||
"""
|
||||
|
@ -206,36 +202,20 @@ def detailed_fetch_game(game_id: int) -> bool:
|
|||
raise ConnectionError(err_msg)
|
||||
game_json = json.loads(response.text)
|
||||
|
||||
instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
|
||||
|
||||
game_id = game_json["id"]
|
||||
players = game_json["players"]
|
||||
num_players = len(players)
|
||||
seed = game_json["seed"]
|
||||
game_options = game_json.get("options", {})
|
||||
var_name = game_options.get("variant", "No Variant")
|
||||
|
||||
if not utils.are_game_options_allowed(game_id, game_options):
|
||||
games = fetch_games_for_player(players[0], game_id, game_id)
|
||||
if not len(games) == 1:
|
||||
print(games)
|
||||
exit(1)
|
||||
assert len(games) == 1
|
||||
game_entry = process_game_entry(games[0])
|
||||
if game_entry is None:
|
||||
return False
|
||||
|
||||
if not utils.is_player_count_allowed(game_id, num_players):
|
||||
return False
|
||||
|
||||
if not utils.is_game_id_allowed(game_id):
|
||||
return False
|
||||
|
||||
var_id = database.get_variant_id(var_name)
|
||||
if var_id is None:
|
||||
logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
|
||||
return False
|
||||
|
||||
# All game options are ok, now check if the number of players is okay.
|
||||
normalized_usernames = [utils.normalize_username(username) for username in players]
|
||||
user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames)
|
||||
# The return value here is a str if there was an unregistered participant
|
||||
if type(user_ids) is str:
|
||||
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids))
|
||||
return False
|
||||
# Now make sure that the game exists in the database
|
||||
store_new_games([game_entry])
|
||||
|
||||
# Now, we can start to actually process the game details
|
||||
instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
|
||||
|
@ -244,19 +224,16 @@ def detailed_fetch_game(game_id: int) -> bool:
|
|||
for action in actions:
|
||||
game.make_action(action)
|
||||
|
||||
conn = conn_manager.get_connection()
|
||||
cur = conn_manager.get_new_cursor()
|
||||
# Note that we assume the game is present in the database already
|
||||
|
||||
|
||||
game_participants_vals = []
|
||||
for seat, user_id in enumerate(user_ids):
|
||||
for seat, user_id in enumerate(game_entry.user_ids):
|
||||
tup = (game_id, user_id, seat)
|
||||
game_participants_vals.append(tup)
|
||||
|
||||
# This inserts the game participants now.
|
||||
# Note that the participants might already be stored, but not their seat (since we do not know the seat when only
|
||||
# getting a game list from the server on fetching the games played by some player)
|
||||
conn = conn_manager.get_connection()
|
||||
cur = conn.cursor()
|
||||
psycopg2.extras.execute_values(
|
||||
cur,
|
||||
"INSERT INTO game_participants "
|
||||
|
@ -271,26 +248,28 @@ def detailed_fetch_game(game_id: int) -> bool:
|
|||
|
||||
# It remains to store the seed and action data for this game
|
||||
# Note that we store the seed first. This ensures that whenever we have actions stored, we also have the seed stored.
|
||||
games_db_interface.store_deck_for_seed(seed, instance.deck)
|
||||
games_db_interface.store_deck_for_seed(game_entry.seed, instance.deck)
|
||||
games_db_interface.store_actions(game_id, actions)
|
||||
logger.debug("Fetched all game details of game {}.".format(game_id))
|
||||
|
||||
# Do some sanity checks that loading the stored data did not change it
|
||||
assert actions == games_db_interface.load_actions(game_id)
|
||||
assert instance.deck == games_db_interface.load_deck(seed)
|
||||
assert instance.deck == games_db_interface.load_deck(game_entry.seed)
|
||||
return True
|
||||
|
||||
|
||||
def fetch_all_game_details():
|
||||
def fetch_all_game_details(force_refresh=False):
|
||||
logger.info("Fetching detailed game data for all games.")
|
||||
cur = conn_manager.get_new_cursor()
|
||||
|
||||
# Get all games with no actions
|
||||
cur.execute("SELECT id FROM games "
|
||||
"LEFT OUTER JOIN game_actions"
|
||||
" ON games.id = game_actions.game_id "
|
||||
"WHERE game_actions.game_id IS NULL"
|
||||
)
|
||||
query = "SELECT DiSTINCT ON (id) id FROM games " \
|
||||
"LEFT OUTER JOIN game_actions" \
|
||||
" ON games.id = game_actions.game_id"
|
||||
if not force_refresh:
|
||||
query += " WHERE game_actions.game_id IS NULL"
|
||||
|
||||
# Get all games
|
||||
cur.execute(query)
|
||||
for (game_id,) in cur.fetchall():
|
||||
detailed_fetch_game(game_id)
|
||||
|
||||
|
|
Loading…
Reference in a new issue