forked from Hanabi/hanabi-league
rework game fetching: deduplicate code, fetch timestamps
This commit is contained in:
parent
e01b7fe823
commit
78bdd8397c
2 changed files with 53 additions and 74 deletions
|
@ -246,7 +246,7 @@ def add_player(user_name: str, player_name: str, base_rating: Optional[int] = No
|
||||||
init_player_base_rating(player_name, base_rating)
|
init_player_base_rating(player_name, base_rating)
|
||||||
|
|
||||||
|
|
||||||
def get_variant_ids():
|
def get_variant_ids() -> List[int]:
|
||||||
cur = conn_manager.get_new_cursor()
|
cur = conn_manager.get_new_cursor()
|
||||||
cur.execute("SELECT id FROM variants")
|
cur.execute("SELECT id FROM variants")
|
||||||
return [var_id for (var_id,) in cur.fetchall()]
|
return [var_id for (var_id,) in cur.fetchall()]
|
||||||
|
|
|
@ -43,9 +43,12 @@ class GameInfo:
|
||||||
datetime_ended: str
|
datetime_ended: str
|
||||||
|
|
||||||
|
|
||||||
def fetch_games_for_player(username: str, latest_game_id: int):
|
def fetch_games_for_player(username: str, first_game_id: int, last_game_id: Optional[int] = None) -> Dict:
|
||||||
logger.verbose("Fetching games for username {} more recent than id {}".format(username, latest_game_id))
|
logger.verbose("Fetching games for username {} more recent than id {}".format(username, first_game_id))
|
||||||
url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, latest_game_id + 1)
|
url = "https://hanab.live/api/v1/history-full/{}?start={}".format(username, first_game_id)
|
||||||
|
if last_game_id is not None:
|
||||||
|
url += "&end={}".format(last_game_id)
|
||||||
|
print(url)
|
||||||
response = session.get(url)
|
response = session.get(url)
|
||||||
if not response.status_code == 200:
|
if not response.status_code == 200:
|
||||||
err_msg = "Failed to fetch games for username {}, requested URL {}".format(username, url)
|
err_msg = "Failed to fetch games for username {}, requested URL {}".format(username, url)
|
||||||
|
@ -54,9 +57,8 @@ def fetch_games_for_player(username: str, latest_game_id: int):
|
||||||
return json.loads(response.text)
|
return json.loads(response.text)
|
||||||
|
|
||||||
|
|
||||||
def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[int]) -> Optional[GameInfo]:
|
def process_game_entry(game_json: Dict) -> Optional[GameInfo]:
|
||||||
logger.debug("Processing entry {}".format(game_json))
|
logger.debug("Processing entry {}".format(game_json))
|
||||||
config = config_manager.get_config()
|
|
||||||
|
|
||||||
# Parse game properties
|
# Parse game properties
|
||||||
game_id = game_json["id"]
|
game_id = game_json["id"]
|
||||||
|
@ -72,7 +74,6 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i
|
||||||
var_id = game_options["variantID"]
|
var_id = game_options["variantID"]
|
||||||
|
|
||||||
normalized_usernames = [utils.normalize_username(username) for username in players]
|
normalized_usernames = [utils.normalize_username(username) for username in players]
|
||||||
|
|
||||||
# Now, check if the game is one that we accept for league
|
# Now, check if the game is one that we accept for league
|
||||||
|
|
||||||
if not all([
|
if not all([
|
||||||
|
@ -83,63 +84,59 @@ def process_game_entry(game_json: Dict, username_dict: Dict, variant_ids: List[i
|
||||||
]):
|
]):
|
||||||
return
|
return
|
||||||
|
|
||||||
if var_id not in variant_ids:
|
database.get_variant_ids()
|
||||||
|
|
||||||
|
if var_id not in database.get_variant_ids():
|
||||||
logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
|
logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
|
||||||
return
|
return
|
||||||
|
|
||||||
# Everything matches, so we can parse the participants now
|
# Everything matches, so we can parse the participants now
|
||||||
user_ids = [] # This will be a list of the (league specific) user_id's that played this game.
|
user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames)
|
||||||
for normalized_username in normalized_usernames:
|
# The return value here is a str if there was an unregistered participant
|
||||||
user_id = username_dict.get(normalized_username, None)
|
if type(user_ids) is str:
|
||||||
if user_id is None:
|
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids))
|
||||||
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, normalized_username))
|
return
|
||||||
return
|
|
||||||
user_ids.append(user_id)
|
|
||||||
|
|
||||||
return GameInfo(game_id, num_players, var_id, seed, score, num_turns, user_ids, normalized_usernames, start_time, end_time)
|
return GameInfo(game_id, num_players, var_id, seed, score, num_turns, user_ids, normalized_usernames, start_time, end_time)
|
||||||
|
|
||||||
|
|
||||||
def fetch_games_for_all_players():
|
def fetch_games_for_all_players() -> List[GameInfo]:
|
||||||
logger.info("Fetching new games.")
|
logger.info("Fetching new games.")
|
||||||
cur = conn_manager.get_new_cursor()
|
cur = conn_manager.get_new_cursor()
|
||||||
cur.execute("SELECT user_accounts.normalized_username, user_accounts.user_id, downloads.latest_game_id "
|
cur.execute("SELECT"
|
||||||
|
" user_accounts.normalized_username,"
|
||||||
|
" user_accounts.user_id,"
|
||||||
|
# Use the starting id as fallback if we have not downloaded any games for this player yet.
|
||||||
|
" COALESCE(downloads.latest_game_id, %s) "
|
||||||
"FROM user_accounts "
|
"FROM user_accounts "
|
||||||
"LEFT OUTER JOIN downloads "
|
"LEFT OUTER JOIN downloads "
|
||||||
" ON user_accounts.normalized_username = downloads.normalized_username"
|
" ON user_accounts.normalized_username = downloads.normalized_username",
|
||||||
|
(config_manager.get_config().starting_game_id,)
|
||||||
)
|
)
|
||||||
# This will be a mapping of normalized username -> user ID that we built from the DB data
|
|
||||||
username_dict = {}
|
|
||||||
|
|
||||||
# This will be a mapping of game id -> JSON data that we get from hanab.live, where we will collect all the
|
# This will be a mapping of game id -> JSON data that we get from hanab.live, where we will collect all the
|
||||||
# possibly relevant games now
|
# possibly relevant games now
|
||||||
games: Dict[int, Dict] = {}
|
games: Dict[int, Dict] = {}
|
||||||
for username, user_id, latest_game_id in cur.fetchall():
|
for username, user_id, latest_game_id in cur.fetchall():
|
||||||
username_dict[username] = user_id
|
player_games = fetch_games_for_player(username, latest_game_id + 1)
|
||||||
# Use the starting id as fallback if we have not downloaded any games for this player yet.
|
|
||||||
if latest_game_id is None:
|
|
||||||
latest_game_id = config_manager.get_config().starting_game_id
|
|
||||||
|
|
||||||
player_games = fetch_games_for_player(username, latest_game_id)
|
|
||||||
for game in player_games:
|
for game in player_games:
|
||||||
games[game['id']] = game
|
games[game['id']] = game
|
||||||
|
|
||||||
logger.info("Found {} potential league game(s) to process.".format(len(games)))
|
logger.info("Found {} potential league game(s) to process.".format(len(games)))
|
||||||
|
|
||||||
allowed_variants = database.get_variant_ids()
|
|
||||||
|
|
||||||
# This will hold the processed games that we will add to the database.
|
# This will hold the processed games that we will add to the database.
|
||||||
good_games: Dict[int, GameInfo] = {}
|
good_games: List[GameInfo] = []
|
||||||
|
|
||||||
for game_id, game in games.items():
|
for game_id, game in games.items():
|
||||||
game_info = process_game_entry(game, username_dict, allowed_variants)
|
game_info = process_game_entry(game)
|
||||||
if game_info is not None:
|
if game_info is not None:
|
||||||
good_games[game_id] = game_info
|
good_games.append(game_info)
|
||||||
|
|
||||||
logger.verbose("Found {} valid league game(s).".format(len(good_games)))
|
logger.verbose("Found {} valid league game(s).".format(len(good_games)))
|
||||||
return good_games
|
return good_games
|
||||||
|
|
||||||
|
|
||||||
def store_new_games(games: Dict[int, GameInfo]):
|
def store_new_games(games: List[GameInfo]):
|
||||||
conn = conn_manager.get_connection()
|
conn = conn_manager.get_connection()
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
games_vals = []
|
games_vals = []
|
||||||
|
@ -147,7 +144,7 @@ def store_new_games(games: Dict[int, GameInfo]):
|
||||||
latest_game_ids: Dict[str, int] = {}
|
latest_game_ids: Dict[str, int] = {}
|
||||||
|
|
||||||
# Now, iterate over all games and convert to tuples for insertion
|
# Now, iterate over all games and convert to tuples for insertion
|
||||||
for game in sorted(games.values(), key=lambda game_info: game_info.game_id):
|
for game in sorted(games, key=lambda game_info: game_info.game_id):
|
||||||
tup = (game.game_id, game.num_players, game.variant_id, game.seed, game.score, game.num_turns, game.datetime_started, game.datetime_ended)
|
tup = (game.game_id, game.num_players, game.variant_id, game.seed, game.score, game.num_turns, game.datetime_started, game.datetime_ended)
|
||||||
games_vals.append(tup)
|
games_vals.append(tup)
|
||||||
for player_id in game.user_ids:
|
for player_id in game.user_ids:
|
||||||
|
@ -193,7 +190,6 @@ def store_new_games(games: Dict[int, GameInfo]):
|
||||||
def detailed_fetch_game(game_id: int) -> bool:
|
def detailed_fetch_game(game_id: int) -> bool:
|
||||||
"""
|
"""
|
||||||
Fetches full game details from the server and stores it in local DB if this game is a league game.
|
Fetches full game details from the server and stores it in local DB if this game is a league game.
|
||||||
@warning: Game data has to be present in database already, game details will then be added.
|
|
||||||
@param game_id: Game ID from hanab.live
|
@param game_id: Game ID from hanab.live
|
||||||
@return: Whether the processed game was accepted as a league game, i.e. inserted into the DB
|
@return: Whether the processed game was accepted as a league game, i.e. inserted into the DB
|
||||||
"""
|
"""
|
||||||
|
@ -206,36 +202,20 @@ def detailed_fetch_game(game_id: int) -> bool:
|
||||||
raise ConnectionError(err_msg)
|
raise ConnectionError(err_msg)
|
||||||
game_json = json.loads(response.text)
|
game_json = json.loads(response.text)
|
||||||
|
|
||||||
instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
|
|
||||||
|
|
||||||
game_id = game_json["id"]
|
game_id = game_json["id"]
|
||||||
players = game_json["players"]
|
players = game_json["players"]
|
||||||
num_players = len(players)
|
|
||||||
seed = game_json["seed"]
|
|
||||||
game_options = game_json.get("options", {})
|
|
||||||
var_name = game_options.get("variant", "No Variant")
|
|
||||||
|
|
||||||
if not utils.are_game_options_allowed(game_id, game_options):
|
games = fetch_games_for_player(players[0], game_id, game_id)
|
||||||
|
if not len(games) == 1:
|
||||||
|
print(games)
|
||||||
|
exit(1)
|
||||||
|
assert len(games) == 1
|
||||||
|
game_entry = process_game_entry(games[0])
|
||||||
|
if game_entry is None:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if not utils.is_player_count_allowed(game_id, num_players):
|
# Now make sure that the game exists in the database
|
||||||
return False
|
store_new_games([game_entry])
|
||||||
|
|
||||||
if not utils.is_game_id_allowed(game_id):
|
|
||||||
return False
|
|
||||||
|
|
||||||
var_id = database.get_variant_id(var_name)
|
|
||||||
if var_id is None:
|
|
||||||
logger.debug("Rejected game {} due to invalid variant id {}".format(game_id, var_id))
|
|
||||||
return False
|
|
||||||
|
|
||||||
# All game options are ok, now check if the number of players is okay.
|
|
||||||
normalized_usernames = [utils.normalize_username(username) for username in players]
|
|
||||||
user_ids = database.get_user_ids_from_normalized_usernames(normalized_usernames)
|
|
||||||
# The return value here is a str if there was an unregistered participant
|
|
||||||
if type(user_ids) is str:
|
|
||||||
logger.debug("Rejected game {} due to unregistered participant {}".format(game_id, user_ids))
|
|
||||||
return False
|
|
||||||
|
|
||||||
# Now, we can start to actually process the game details
|
# Now, we can start to actually process the game details
|
||||||
instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
|
instance, actions = hanabi.live.hanab_live.parse_json_game(game_json, False)
|
||||||
|
@ -244,19 +224,16 @@ def detailed_fetch_game(game_id: int) -> bool:
|
||||||
for action in actions:
|
for action in actions:
|
||||||
game.make_action(action)
|
game.make_action(action)
|
||||||
|
|
||||||
conn = conn_manager.get_connection()
|
|
||||||
cur = conn_manager.get_new_cursor()
|
|
||||||
# Note that we assume the game is present in the database already
|
|
||||||
|
|
||||||
|
|
||||||
game_participants_vals = []
|
game_participants_vals = []
|
||||||
for seat, user_id in enumerate(user_ids):
|
for seat, user_id in enumerate(game_entry.user_ids):
|
||||||
tup = (game_id, user_id, seat)
|
tup = (game_id, user_id, seat)
|
||||||
game_participants_vals.append(tup)
|
game_participants_vals.append(tup)
|
||||||
|
|
||||||
# This inserts the game participants now.
|
# This inserts the game participants now.
|
||||||
# Note that the participants might already be stored, but not their seat (since we do not know the seat when only
|
# Note that the participants might already be stored, but not their seat (since we do not know the seat when only
|
||||||
# getting a game list from the server on fetching the games played by some player)
|
# getting a game list from the server on fetching the games played by some player)
|
||||||
|
conn = conn_manager.get_connection()
|
||||||
|
cur = conn.cursor()
|
||||||
psycopg2.extras.execute_values(
|
psycopg2.extras.execute_values(
|
||||||
cur,
|
cur,
|
||||||
"INSERT INTO game_participants "
|
"INSERT INTO game_participants "
|
||||||
|
@ -271,26 +248,28 @@ def detailed_fetch_game(game_id: int) -> bool:
|
||||||
|
|
||||||
# It remains to store the seed and action data for this game
|
# It remains to store the seed and action data for this game
|
||||||
# Note that we store the seed first. This ensures that whenever we have actions stored, we also have the seed stored.
|
# Note that we store the seed first. This ensures that whenever we have actions stored, we also have the seed stored.
|
||||||
games_db_interface.store_deck_for_seed(seed, instance.deck)
|
games_db_interface.store_deck_for_seed(game_entry.seed, instance.deck)
|
||||||
games_db_interface.store_actions(game_id, actions)
|
games_db_interface.store_actions(game_id, actions)
|
||||||
logger.debug("Fetched all game details of game {}.".format(game_id))
|
logger.debug("Fetched all game details of game {}.".format(game_id))
|
||||||
|
|
||||||
# Do some sanity checks that loading the stored data did not change it
|
# Do some sanity checks that loading the stored data did not change it
|
||||||
assert actions == games_db_interface.load_actions(game_id)
|
assert actions == games_db_interface.load_actions(game_id)
|
||||||
assert instance.deck == games_db_interface.load_deck(seed)
|
assert instance.deck == games_db_interface.load_deck(game_entry.seed)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def fetch_all_game_details():
|
def fetch_all_game_details(force_refresh=False):
|
||||||
logger.info("Fetching detailed game data for all games.")
|
logger.info("Fetching detailed game data for all games.")
|
||||||
cur = conn_manager.get_new_cursor()
|
cur = conn_manager.get_new_cursor()
|
||||||
|
|
||||||
# Get all games with no actions
|
query = "SELECT DiSTINCT ON (id) id FROM games " \
|
||||||
cur.execute("SELECT id FROM games "
|
"LEFT OUTER JOIN game_actions" \
|
||||||
"LEFT OUTER JOIN game_actions"
|
" ON games.id = game_actions.game_id"
|
||||||
" ON games.id = game_actions.game_id "
|
if not force_refresh:
|
||||||
"WHERE game_actions.game_id IS NULL"
|
query += " WHERE game_actions.game_id IS NULL"
|
||||||
)
|
|
||||||
|
# Get all games
|
||||||
|
cur.execute(query)
|
||||||
for (game_id,) in cur.fetchall():
|
for (game_id,) in cur.fetchall():
|
||||||
detailed_fetch_game(game_id)
|
detailed_fetch_game(game_id)
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue