Hanabi/Py-Hanabi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add progress bar to downloading data

Browse source
This commit is contained in:
Maximilian Keßler 2023-05-14 10:40:47 +02:00
parent 78348c10b5
commit 9b7623b173
Signed by: max
GPG key ID: BCC5A619923C0BA5
1 changed files with 22 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
download_data.py
View file

@ -1,4 +1,4 @@
import json import alive_progress
from typing import Dict, Optional from typing import Dict, Optional
import psycopg2.errors import psycopg2.errors
@ -146,13 +146,20 @@ def download_games(var_id):
) )
) )
with alive_progress.alive_bar(
total=num_entries - num_already_downloaded_games,
title='Downloading games for variant id {} [{}]'.format(var_id, name)
) as bar:
for page in range(next_page, last_page + 1): for page in range(next_page, last_page + 1):
r = api(url + "?col[0]=0&page={}".format(page)) r = api(url + "?col[0]=0&page={}".format(page))
rows = r.get('rows', []) rows = r.get('rows', [])
assert page == last_page or len(rows) == page_size, \ if not (page == last_page or len(rows) == page_size):
"Received unexpected row count ({}) when querying page {}".format(len(rows), page) print('WARN: received unexpected row count ({}) on page {}'.format(len(rows), page))
# assert page == last_page or len(rows) == page_size, \
# "Received unexpected row count ({}) when querying page {}".format(len(rows), page)
for row in rows: for row in rows:
process_game_row(row, var_id) process_game_row(row, var_id)
bar()
cur.execute( cur.execute(
"INSERT INTO variant_game_downloads (variant_id, last_game_id) VALUES" "INSERT INTO variant_game_downloads (variant_id, last_game_id) VALUES"
"(%s, %s)" "(%s, %s)"
@ -160,4 +167,3 @@ def download_games(var_id):
(var_id, r['rows'][-1]['id']) (var_id, r['rows'][-1]['id'])
) )
conn.commit() conn.commit()
print('Downloaded and processed page {}'.format(page))