Hanabi/Py-Hanabi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

when downloading: refresh api queries on row count mismatch

Browse source
This commit is contained in:
Maximilian Keßler 2023-07-07 08:29:12 +02:00
parent 71db1e9d26
commit 301cfe10e8
Signed by: max
GPG key ID: BCC5A619923C0BA5
1 changed files with 28 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
hanabi/live/download_data.py
View file

@ -250,19 +250,31 @@ def download_games(var_id, export_all_games: bool = False):
enrich_print=False enrich_print=False
) as bar: ) as bar:
for page in range(next_page, last_page + 1): for page in range(next_page, last_page + 1):
r = site_api.api(url + "?col[0]=0&page={}".format(page), refresh=page == last_page) for refresh in [False, True]:
rows = r.get('rows', []) r = site_api.api(url + "?col[0]=0&page={}".format(page), refresh=(page == last_page) or refresh)
if page == next_page: rows = r.get('rows', [])
rows = rows[num_already_downloaded_games % 100:] if page == next_page:
if not (page == last_page or len(rows) == page_size): rows = rows[num_already_downloaded_games % 100:]
logger.warn('WARN: received unexpected row count ({}) on page {}'.format(len(rows), page)) if not (page == last_page or len(rows) == page_size):
for row in rows: if not refresh:
_process_game_row(row, var_id, export_all_games) # row count does not match, maybe this is due to an old cached version of the api query,
bar() # try again with a forced refresh of the query
database.cur.execute( logger.verbose("refreshing page {} due to unexpected row count".format(page))
"INSERT INTO variant_game_downloads (variant_id, last_game_id) VALUES" continue
"(%s, %s)" # If refreshing did not fix the error, log a warning
"ON CONFLICT (variant_id) DO UPDATE SET last_game_id = EXCLUDED.last_game_id", logger.warn('WARN: received unexpected row count ({}, expected {}) on page {}'.format(
(var_id, r['rows'][-1]['id']) len(rows), page_size, page)
) )
database.conn.commit() for row in rows:
_process_game_row(row, var_id, export_all_games)
bar()
database.cur.execute(
"INSERT INTO variant_game_downloads (variant_id, last_game_id) VALUES"
"(%s, %s)"
"ON CONFLICT (variant_id) DO UPDATE SET last_game_id = EXCLUDED.last_game_id",
(var_id, r['rows'][-1]['id'])
)
database.conn.commit()
# we need this so that we don't execute the iteration with forced refresh
# if stuff already checked out without refreshing
break