Hanabi/Py-Hanabi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add progress bar to downloading data

Browse source
This commit is contained in:
Maximilian Keßler 2023-05-14 10:40:47 +02:00
parent 78348c10b5
commit 9b7623b173
Signed by: max
GPG key ID: BCC5A619923C0BA5
1 changed files with 22 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
download_data.py
View file

@ -1,4 +1,4 @@
import json import alive_progress
from typing import Dict, Optional from typing import Dict, Optional
import psycopg2.errors import psycopg2.errors
@ -146,18 +146,24 @@ def download_games(var_id):
) )
) )
for page in range(next_page, last_page + 1): with alive_progress.alive_bar(
r = api(url + "?col[0]=0&page={}".format(page)) total=num_entries - num_already_downloaded_games,
rows = r.get('rows', []) title='Downloading games for variant id {} [{}]'.format(var_id, name)
assert page == last_page or len(rows) == page_size, \ ) as bar:
"Received unexpected row count ({}) when querying page {}".format(len(rows), page) for page in range(next_page, last_page + 1):
for row in rows: r = api(url + "?col[0]=0&page={}".format(page))
process_game_row(row, var_id) rows = r.get('rows', [])
cur.execute( if not (page == last_page or len(rows) == page_size):
"INSERT INTO variant_game_downloads (variant_id, last_game_id) VALUES" print('WARN: received unexpected row count ({}) on page {}'.format(len(rows), page))
"(%s, %s)" # assert page == last_page or len(rows) == page_size, \
"ON CONFLICT (variant_id) DO UPDATE SET last_game_id = EXCLUDED.last_game_id", # "Received unexpected row count ({}) when querying page {}".format(len(rows), page)
(var_id, r['rows'][-1]['id']) for row in rows:
) process_game_row(row, var_id)
conn.commit() bar()
print('Downloaded and processed page {}'.format(page)) cur.execute(
"INSERT INTO variant_game_downloads (variant_id, last_game_id) VALUES"
"(%s, %s)"
"ON CONFLICT (variant_id) DO UPDATE SET last_game_id = EXCLUDED.last_game_id",
(var_id, r['rows'][-1]['id'])
)
conn.commit()