fz0x1/HPI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

general: improve logging during file processing in various modules

Browse source
This commit is contained in:
karlicoss 2023-10-29 00:11:58 +01:00
parent bd27bd4c24
commit ea195e3d17
4 changed files with 29 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

13
my/hackernews/materialistic.py
View file

@ -7,7 +7,7 @@ from typing import Any, Dict, Iterator, NamedTuple, Sequence
from more_itertools import unique_everseen
from my.core import get_files, datetime_aware
from my.core import get_files, datetime_aware, make_logger
from my.core.sqlite import sqlite_connection
from my.config import materialistic as config # todo migrate config to my.hackernews.materialistic
@ -15,6 +15,9 @@ from my.config import materialistic as config # todo migrate config to my.hacke
from .common import hackernews_link
logger = make_logger(__name__)
def inputs() -> Sequence[Path]:
return get_files(config.export_path)
@ -51,8 +54,12 @@ class Saved(NamedTuple):
def _all_raw() -> Iterator[Row]:
for db in inputs():
with sqlite_connection(db, immutable=True, row_factory='dict') as conn:
paths = inputs()
total = len(paths)
width = len(str(total))
for idx, path in enumerate(paths):
logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}')
with sqlite_connection(path, immutable=True, row_factory='dict') as conn:
yield from conn.execute('SELECT * FROM saved ORDER BY time')