diff --git a/my/bluemaestro.py b/my/bluemaestro.py index b49e9e0..6b58d05 100644 --- a/my/bluemaestro.py +++ b/my/bluemaestro.py @@ -10,15 +10,13 @@ import re import sqlite3 from typing import Iterable, Sequence, Set, Optional -from my.core import get_files, LazyLogger, dataclass, Res +from my.core import get_files, make_logger, dataclass, Res from my.core.sqlite import sqlite_connect_immutable from my.config import bluemaestro as config -# todo control level via env variable? -# i.e. HPI_LOGGING_MY_BLUEMAESTRO_LEVEL=debug -logger = LazyLogger(__name__, level='debug') +logger = make_logger(__name__) def inputs() -> Sequence[Path]: @@ -63,7 +61,7 @@ def measurements() -> Iterable[Res[Measurement]]: # tables are immutable, so can save on processing.. processed_tables: Set[str] = set() for f in dbs: - logger.debug('processing %s', f) + logger.info('processing %s', f) tot = 0 new = 0 # todo assert increasing timestamp? diff --git a/my/github/gdpr.py b/my/github/gdpr.py index 6f7efe4..3d23565 100644 --- a/my/github/gdpr.py +++ b/my/github/gdpr.py @@ -25,8 +25,8 @@ class github(user_config): ### -from ..core import LazyLogger -logger = LazyLogger(__name__) +from ..core import make_logger +logger = make_logger(__name__) from ..core.cfg import make_config @@ -85,6 +85,7 @@ def events() -> Iterable[Res[Event]]: 'repository_files_': None, # repository artifacts, probs not very useful } for f in files: + logger.info(f'{f} : processing...') handler: Any for prefix, h in handler_map.items(): if not f.name.startswith(prefix): diff --git a/my/instagram/android.py b/my/instagram/android.py index 48e8021..e1db55a 100644 --- a/my/instagram/android.py +++ b/my/instagram/android.py @@ -14,7 +14,7 @@ from more_itertools import unique_everseen from my.core import ( get_files, Paths, make_config, - LazyLogger, + make_logger, datetime_naive, Json, Res, assert_never, @@ -24,7 +24,7 @@ from my.core.sqlite import sqlite_connect_immutable, select from my.config import instagram as user_config -logger = LazyLogger(__name__, level='debug') +logger = make_logger(__name__) @dataclass class instagram_android_config(user_config.android): @@ -132,7 +132,9 @@ def _entities() -> Iterator[Res[Union[User, _Message]]]: # NOTE: definitely need to merge multiple, app seems to recycle old messages # TODO: hmm hard to guarantee timestamp ordering when we use synthetic input data... # todo use TypedDict? - for f in inputs(): + dbs = inputs() + for f in dbs: + logger.info(f'{f} : processing...') with sqlite_connect_immutable(f) as db: # TODO ugh. seems like no way to extract username? # sometimes messages (e.g. media_share) contain it in message field diff --git a/my/instagram/gdpr.py b/my/instagram/gdpr.py index b9f8780..348d69d 100644 --- a/my/instagram/gdpr.py +++ b/my/instagram/gdpr.py @@ -15,14 +15,14 @@ from my.core import ( datetime_naive, Res, assert_never, - LazyLogger, + make_logger, ) from my.core.kompress import ZipPath from my.config import instagram as user_config -logger = LazyLogger(__name__, level='debug') +logger = make_logger(__name__) @dataclass @@ -113,6 +113,7 @@ def _entities() -> Iterator[Res[Union[User, _Message]]]: for fname, ffiles in file_map.items(): for ffile in sorted(ffiles, key=lambda p: int(p.stem.split('_')[-1])): + logger.info(f'{ffile} : processing...') j = json.loads(ffile.read_text()) id_len = 10 diff --git a/my/reddit/rexport.py b/my/reddit/rexport.py index 2d2b9a3..f20d00e 100644 --- a/my/reddit/rexport.py +++ b/my/reddit/rexport.py @@ -64,10 +64,11 @@ except ModuleNotFoundError as e: ############################ from typing import List, Sequence, Mapping, Iterator, Any -from my.core.common import mcachew, get_files, LazyLogger, make_dict, Stats +from my.core import make_logger +from my.core.common import mcachew, get_files, make_dict, Stats -logger = LazyLogger(__name__, level='info') +logger = make_logger(__name__) from pathlib import Path @@ -85,8 +86,8 @@ Upvote = dal.Upvote def _dal() -> dal.DAL: inp = list(inputs()) return dal.DAL(inp) -cache = mcachew(depends_on=inputs, logger=logger) # depends on inputs only +cache = mcachew(depends_on=inputs) @cache def saved() -> Iterator[Save]: diff --git a/my/rescuetime.py b/my/rescuetime.py index c986d89..75684d9 100644 --- a/my/rescuetime.py +++ b/my/rescuetime.py @@ -9,14 +9,14 @@ from pathlib import Path from datetime import timedelta from typing import Sequence, Iterable -from .core import get_files, LazyLogger -from .core.common import mcachew -from .core.error import Res, split_errors +from my.core import get_files, make_logger +from my.core.common import mcachew +from my.core.error import Res, split_errors from my.config import rescuetime as config -log = LazyLogger(__name__, level='info') +logger = make_logger(__name__) def inputs() -> Sequence[Path]: diff --git a/my/twitter/archive.py b/my/twitter/archive.py index d9ba562..44ebc5f 100644 --- a/my/twitter/archive.py +++ b/my/twitter/archive.py @@ -44,11 +44,11 @@ from typing import List, Optional, NamedTuple, Sequence, Iterator from pathlib import Path import json -from ..core.common import get_files, LazyLogger, Json +from my.core import get_files, make_logger, Json -logger = LazyLogger(__name__, level="warning") +logger = make_logger(__name__) def inputs() -> Sequence[Path]: @@ -175,7 +175,7 @@ class ZipExport: self.old_format = True def raw(self, what: str, *, fname: Optional[str]=None) -> Iterator[Json]: - logger.info('processing: %s %s', self.zpath, what) + logger.info(f'{self.zpath} : processing {what}') path = fname or what if not self.old_format: diff --git a/my/zotero.py b/my/zotero.py index 3afc512..4440aae 100644 --- a/my/zotero.py +++ b/my/zotero.py @@ -5,11 +5,11 @@ from typing import Iterator, Optional, Dict, Any, Sequence from pathlib import Path import sqlite3 -from my.core import LazyLogger, Res, datetime_aware +from my.core import make_logger, Res, datetime_aware from my.core.sqlite import sqlite_copy_and_open -logger = LazyLogger(__name__, level='debug') +logger = make_logger(__name__) def inputs() -> Sequence[Path]: