fz0x1/HPI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

loading

Browse source
This commit is contained in:
Dima Gerasimov 2019-04-30 13:22:19 +02:00
parent 9545db4aaf
commit 1a765129cb
1 changed files with 11 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
sql.py
View file

@ -38,13 +38,14 @@ def save_locs(db_path: Path):
# locs = list(_load_locations(fo)) # locs = list(_load_locations(fo))
# TODO fuck. do I really need to split myself?? # TODO fuck. do I really need to split myself??
# sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) too many SQL variables # sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) too many SQL variables
# TODO err wtf?? jsust 80000??? # TODO count deprecated??
for chunk in ichunks(_load_locations(fo), 10000): for chunk in ichunks(_load_locations(fo), 10000):
engine.execute(table.insert().values(chunk)) engine.execute(table.insert().values(chunk))
print(engine.execute(table.count()).fetchone())
# TODO maintain order during insertion? # TODO maintain order during insertion?
def load_locs(db_path: Path): def iter_db_locs(db_path: Path):
db = sa.create_engine(f'sqlite:///{db_path}') db = sa.create_engine(f'sqlite:///{db_path}')
engine = db.connect() # TODO do I need to tear anything down?? engine = db.connect() # TODO do I need to tear anything down??
meta = sa.MetaData(engine) meta = sa.MetaData(engine)
@ -53,24 +54,27 @@ def load_locs(db_path: Path):
meta.create_all() meta.create_all()
table = sa.table('locations', *schema) table = sa.table('locations', *schema)
return engine.execute(table.select()).fetchall() datas = engine.execute(table.select()).fetchall()
yield from (Location(**d) for d in datas)
def main(): def main():
from kython import setup_logzero from kython import setup_logzero
setup_logzero(get_logger(), level=logging.DEBUG) setup_logzero(get_logger(), level=logging.DEBUG)
db_path = Path('test2.sqlite') db_path = Path('test3.sqlite')
# if db_path.exists(): # if db_path.exists():
# db_path.unlink() # db_path.unlink()
locs = [Location(**d) for d in load_locs(db_path)][:10] locs = iter_db_locs(db_path)
print(locs) print(len(list(locs)))
# TODO is it quicker to insert anyway? needs unique policy # TODO is it quicker to insert anyway? needs unique policy
# ok, very nice. the whold db is just 4mb now # ok, very nice. the whold db is just 20mb now
# nice, and loads in seconds basically
# TODO FIXME just need to check timezone
if __name__ == '__main__': if __name__ == '__main__':
main() main()