Hanabi/Py-Hanabi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

when downloading: refresh api queries on row count mismatch

Browse Source
This commit is contained in:
Maximilian Keßler 2023-07-07 08:29:12 +02:00
parent 71db1e9d26
commit 301cfe10e8
Signed by: max
GPG Key ID: BCC5A619923C0BA5
1 changed files with 28 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
hanabi/live/download_data.py
View File

@ -250,19 +250,31 @@ def download_games(var_id, export_all_games: bool = False):
enrich_print=False
) as bar:
for page in range(next_page, last_page + 1):
r = site_api.api(url + "?col[0]=0&page={}".format(page), refresh=page == last_page)
rows = r.get('rows', [])
if page == next_page:
rows = rows[num_already_downloaded_games % 100:]
if not (page == last_page or len(rows) == page_size):
logger.warn('WARN: received unexpected row count ({}) on page {}'.format(len(rows), page))
for row in rows:
_process_game_row(row, var_id, export_all_games)
bar()
database.cur.execute(
"INSERT INTO variant_game_downloads (variant_id, last_game_id) VALUES"
"(%s, %s)"
"ON CONFLICT (variant_id) DO UPDATE SET last_game_id = EXCLUDED.last_game_id",
(var_id, r['rows'][-1]['id'])
)
database.conn.commit()
for refresh in [False, True]:
r = site_api.api(url + "?col[0]=0&page={}".format(page), refresh=(page == last_page) or refresh)
rows = r.get('rows', [])
if page == next_page:
rows = rows[num_already_downloaded_games % 100:]
if not (page == last_page or len(rows) == page_size):
if not refresh:
# row count does not match, maybe this is due to an old cached version of the api query,
# try again with a forced refresh of the query
logger.verbose("refreshing page {} due to unexpected row count".format(page))
continue
# If refreshing did not fix the error, log a warning
logger.warn('WARN: received unexpected row count ({}, expected {}) on page {}'.format(
len(rows), page_size, page)
)
for row in rows:
_process_game_row(row, var_id, export_all_games)
bar()
database.cur.execute(
"INSERT INTO variant_game_downloads (variant_id, last_game_id) VALUES"
"(%s, %s)"
"ON CONFLICT (variant_id) DO UPDATE SET last_game_id = EXCLUDED.last_game_id",
(var_id, r['rows'][-1]['id'])
)
database.conn.commit()
# we need this so that we don't execute the iteration with forced refresh
# if stuff already checked out without refreshing
break