From ea195e3d17b3065f5a700773161072b6e025962c Mon Sep 17 00:00:00 2001
From: karlicoss <karlicoss@gmail.com>
Date: Sun, 29 Oct 2023 00:11:58 +0100
Subject: [PATCH] general: improve logging during file processing in various
 modules

---
 my/hackernews/harmonic.py      |  9 ++++++---
 my/hackernews/materialistic.py | 13 ++++++++++---
 my/tinder/android.py           | 10 ++++++----
 my/whatsapp/android.py         | 12 +++++++-----
 4 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/my/hackernews/harmonic.py b/my/hackernews/harmonic.py
index f78c3ef..6070510 100644
--- a/my/hackernews/harmonic.py
+++ b/my/hackernews/harmonic.py
@@ -81,10 +81,13 @@ _PREFIX = 'com.simon.harmonichackernews.KEY_SHARED_PREFERENCES'
 
 
 def _saved() -> Iterator[Res[Saved]]:
-    for p in inputs():
-        logger.info(f'processing: {p}')
+    paths = inputs()
+    total = len(paths)
+    width = len(str(total))
+    for idx, path in enumerate(paths):
+        logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}')
         # TODO defensive for each item!
-        tr = etree.parse(p)
+        tr = etree.parse(path)
 
         res = one(cast(List[Any], tr.xpath(f'//*[@name="{_PREFIX}_CACHED_STORIES_STRINGS"]')))
         cached_ids = [x.text.split('-')[0] for x in res]
diff --git a/my/hackernews/materialistic.py b/my/hackernews/materialistic.py
index eddf053..4d5cd47 100644
--- a/my/hackernews/materialistic.py
+++ b/my/hackernews/materialistic.py
@@ -7,7 +7,7 @@ from typing import Any, Dict, Iterator, NamedTuple, Sequence
 
 from more_itertools import unique_everseen
 
-from my.core import get_files, datetime_aware
+from my.core import get_files, datetime_aware, make_logger
 from my.core.sqlite import sqlite_connection
 
 from my.config import materialistic as config  # todo migrate config to my.hackernews.materialistic
@@ -15,6 +15,9 @@ from my.config import materialistic as config  # todo migrate config to my.hacke
 from .common import hackernews_link
 
 
+logger = make_logger(__name__)
+
+
 def inputs() -> Sequence[Path]:
     return get_files(config.export_path)
 
@@ -51,8 +54,12 @@ class Saved(NamedTuple):
 
 
 def _all_raw() -> Iterator[Row]:
-    for db in inputs():
-        with sqlite_connection(db, immutable=True, row_factory='dict') as conn:
+    paths = inputs()
+    total = len(paths)
+    width = len(str(total))
+    for idx, path in enumerate(paths):
+        logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}')
+        with sqlite_connection(path, immutable=True, row_factory='dict') as conn:
             yield from conn.execute('SELECT * FROM saved ORDER BY time')
 
 
diff --git a/my/tinder/android.py b/my/tinder/android.py
index 9047a53..0ba9739 100644
--- a/my/tinder/android.py
+++ b/my/tinder/android.py
@@ -87,10 +87,12 @@ Entity = Union[Person, Match, Message]
 
 
 def _entities() -> Iterator[Res[_Entity]]:
-    dbs = inputs()
-    for i, db_file in enumerate(dbs):
-        logger.info(f'processing {db_file} {i}/{len(dbs)}')
-        with sqlite_connection(db_file, immutable=True, row_factory='row') as db:
+    paths = inputs()
+    total = len(paths)
+    width = len(str(total))
+    for idx, path in enumerate(paths):
+        logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}')
+        with sqlite_connection(path, immutable=True, row_factory='row') as db:
             yield from _handle_db(db)
 
 
diff --git a/my/whatsapp/android.py b/my/whatsapp/android.py
index e7c266c..b82c353 100644
--- a/my/whatsapp/android.py
+++ b/my/whatsapp/android.py
@@ -189,14 +189,16 @@ def _process_db(db: sqlite3.Connection):
 
 
 def _messages() -> Iterator[Res[Message]]:
-    dbs = inputs()
-    for i, f in enumerate(dbs):
-        logger.info(f'processing {f} {i}/{len(dbs)}')
-        with sqlite_connection(f, immutable=True, row_factory='row') as db:
+    paths = inputs()
+    total = len(paths)
+    width = len(str(total))
+    for idx, path in enumerate(paths):
+        logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}')
+        with sqlite_connection(path, immutable=True, row_factory='row') as db:
             try:
                 yield from _process_db(db)
             except Exception as e:
-                yield echain(RuntimeError(f'While processing {f}'), cause=e)
+                yield echain(RuntimeError(f'While processing {path}'), cause=e)
 
 
 def messages() -> Iterator[Res[Message]]: