loading

sql.py

@@ -38,13 +38,14 @@ def save_locs(db_path: Path):
         # locs = list(_load_locations(fo))
         # TODO fuck. do I really need to split myself??
         # sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) too many SQL variables
-        # TODO err wtf?? jsust 80000???
+        # TODO count deprecated??
         for chunk in ichunks(_load_locations(fo), 10000):
             engine.execute(table.insert().values(chunk))
+            print(engine.execute(table.count()).fetchone())
 
     # TODO maintain order during insertion?
 
-def load_locs(db_path: Path):
+def iter_db_locs(db_path: Path):
     db = sa.create_engine(f'sqlite:///{db_path}')
     engine = db.connect() # TODO do I need to tear anything down??
     meta = sa.MetaData(engine)
@@ -53,24 +54,27 @@ def load_locs(db_path: Path):
     meta.create_all()
     table = sa.table('locations', *schema)
 
-    return engine.execute(table.select()).fetchall()
+    datas = engine.execute(table.select()).fetchall()
+    yield from (Location(**d) for d in datas)
 
 
 def main():
     from kython import setup_logzero
     setup_logzero(get_logger(), level=logging.DEBUG)
 
-    db_path = Path('test2.sqlite')
+    db_path = Path('test3.sqlite')
     # if db_path.exists():
     #     db_path.unlink()
 
-    locs = [Location(**d) for d in load_locs(db_path)][:10]
-    print(locs)
+    locs = iter_db_locs(db_path)
+    print(len(list(locs)))
 
 
     # TODO is it quicker to insert anyway? needs unique policy
 
-    # ok, very nice. the whold db is just 4mb now
+    # ok, very nice. the whold db is just 20mb now
+    # nice, and loads in seconds basically
+    # TODO FIXME just need to check timezone
 
 if __name__ == '__main__':
     main()