From 1a765129cba9c91ff1d60c3becd0b5a06a7479d8 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 30 Apr 2019 13:22:19 +0200 Subject: [PATCH] loading --- sql.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/sql.py b/sql.py index 35e3df8..590cc89 100755 --- a/sql.py +++ b/sql.py @@ -38,13 +38,14 @@ def save_locs(db_path: Path): # locs = list(_load_locations(fo)) # TODO fuck. do I really need to split myself?? # sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) too many SQL variables - # TODO err wtf?? jsust 80000??? + # TODO count deprecated?? for chunk in ichunks(_load_locations(fo), 10000): engine.execute(table.insert().values(chunk)) + print(engine.execute(table.count()).fetchone()) # TODO maintain order during insertion? -def load_locs(db_path: Path): +def iter_db_locs(db_path: Path): db = sa.create_engine(f'sqlite:///{db_path}') engine = db.connect() # TODO do I need to tear anything down?? meta = sa.MetaData(engine) @@ -53,24 +54,27 @@ def load_locs(db_path: Path): meta.create_all() table = sa.table('locations', *schema) - return engine.execute(table.select()).fetchall() + datas = engine.execute(table.select()).fetchall() + yield from (Location(**d) for d in datas) def main(): from kython import setup_logzero setup_logzero(get_logger(), level=logging.DEBUG) - db_path = Path('test2.sqlite') + db_path = Path('test3.sqlite') # if db_path.exists(): # db_path.unlink() - locs = [Location(**d) for d in load_locs(db_path)][:10] - print(locs) + locs = iter_db_locs(db_path) + print(len(list(locs))) # TODO is it quicker to insert anyway? needs unique policy - # ok, very nice. the whold db is just 4mb now + # ok, very nice. the whold db is just 20mb now + # nice, and loads in seconds basically + # TODO FIXME just need to check timezone if __name__ == '__main__': main()