Hanabi/Py-Hanabi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

when downloading: refresh api queries on row count mismatch

Browse source
This commit is contained in:
Maximilian Keßler 2023-07-07 08:29:12 +02:00
parent 71db1e9d26
commit 301cfe10e8
Signed by: max
GPG key ID: BCC5A619923C0BA5
1 changed files with 28 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
hanabi/live/download_data.py
View file

@ -250,12 +250,21 @@ def download_games(var_id, export_all_games: bool = False):
enrich_print=False enrich_print=False
) as bar: ) as bar:
for page in range(next_page, last_page + 1): for page in range(next_page, last_page + 1):
r = site_api.api(url + "?col[0]=0&page={}".format(page), refresh=page == last_page) for refresh in [False, True]:
r = site_api.api(url + "?col[0]=0&page={}".format(page), refresh=(page == last_page) or refresh)
rows = r.get('rows', []) rows = r.get('rows', [])
if page == next_page: if page == next_page:
rows = rows[num_already_downloaded_games % 100:] rows = rows[num_already_downloaded_games % 100:]
if not (page == last_page or len(rows) == page_size): if not (page == last_page or len(rows) == page_size):
logger.warn('WARN: received unexpected row count ({}) on page {}'.format(len(rows), page)) if not refresh:
# row count does not match, maybe this is due to an old cached version of the api query,
# try again with a forced refresh of the query
logger.verbose("refreshing page {} due to unexpected row count".format(page))
continue
# If refreshing did not fix the error, log a warning
logger.warn('WARN: received unexpected row count ({}, expected {}) on page {}'.format(
len(rows), page_size, page)
)
for row in rows: for row in rows:
_process_game_row(row, var_id, export_all_games) _process_game_row(row, var_id, export_all_games)
bar() bar()
@ -266,3 +275,6 @@ def download_games(var_id, export_all_games: bool = False):
(var_id, r['rows'][-1]['id']) (var_id, r['rows'][-1]['id'])
) )
database.conn.commit() database.conn.commit()
# we need this so that we don't execute the iteration with forced refresh
# if stuff already checked out without refreshing
break