diff --git a/my/core/common.py b/my/core/common.py index 918f4b2..fcbeabb 100644 --- a/my/core/common.py +++ b/my/core/common.py @@ -134,7 +134,8 @@ def get_files(pp: Paths, glob: str=DEFAULT_GLOB, sort: bool=True) -> Tuple[Path, warnings.warn(f"Treating {ss} as glob path. Explicit glob={glob} argument is ignored!") paths.extend(map(Path, do_glob(ss))) else: - assert src.is_file(), src + if not src.is_file(): + raise RuntimeError(f"Expected '{src}' to exist") # todo assert matches glob?? paths.append(src) diff --git a/my/kython/konsume.py b/my/kython/konsume.py index 6e829d3..679755c 100644 --- a/my/kython/konsume.py +++ b/my/kython/konsume.py @@ -11,7 +11,7 @@ def zoom(w, *keys): # TODO need to support lists class Zoomable: - def __init__(self, parent, *args, **kwargs): + def __init__(self, parent, *args, **kwargs) -> None: super().__init__(*args, **kwargs) # type: ignore self.parent = parent @@ -21,19 +21,19 @@ class Zoomable: def dependants(self): raise NotImplementedError - def ignore(self): + def ignore(self) -> None: self.consume_all() - def consume_all(self): + def consume_all(self) -> None: for d in self.dependants: d.consume_all() self.consume() - def consume(self): + def consume(self) -> None: assert self.parent is not None self.parent._remove(self) - def zoom(self): + def zoom(self) -> 'Zoomable': self.consume() return self @@ -56,6 +56,8 @@ class Wdict(Zoomable, OrderedDict): def this_consumed(self): return len(self) == 0 + # TODO specify mypy type for the index special method? + class Wlist(Zoomable, list): def _remove(self, xx): @@ -83,7 +85,8 @@ class Wvalue(Zoomable): def __repr__(self): return 'WValue{' + repr(self.value) + '}' -def _wrap(j, parent=None): +from typing import Tuple +def _wrap(j, parent=None) -> Tuple[Zoomable, List[Zoomable]]: res: Zoomable cc: List[Zoomable] if isinstance(j, dict): @@ -109,13 +112,14 @@ def _wrap(j, parent=None): raise RuntimeError(f'Unexpected type: {type(j)} {j}') from contextlib import contextmanager +from typing import Iterator class UnconsumedError(Exception): pass # TODO think about error policy later... @contextmanager -def wrap(j, throw=True): +def wrap(j, throw=True) -> Iterator[Zoomable]: w, children = _wrap(j) yield w @@ -128,28 +132,33 @@ def wrap(j, throw=True): # TODO log? pass - +from typing import cast def test_unconsumed(): import pytest # type: ignore with pytest.raises(UnconsumedError): with wrap({'a': 1234}) as w: + w = cast(Wdict, w) pass with pytest.raises(UnconsumedError): with wrap({'c': {'d': 2222}}) as w: + w = cast(Wdict, w) d = w['c']['d'].zoom() def test_consumed(): with wrap({'a': 1234}) as w: + w = cast(Wdict, w) a = w['a'].zoom() with wrap({'c': {'d': 2222}}) as w: + w = cast(Wdict, w) c = w['c'].zoom() d = c['d'].zoom() def test_types(): # (string, number, object, array, boolean or nul with wrap({'string': 'string', 'number': 3.14, 'boolean': True, 'null': None, 'list': [1, 2, 3]}) as w: + w = cast(Wdict, w) w['string'].zoom() w['number'].consume() w['boolean'].zoom() @@ -159,5 +168,8 @@ def test_types(): def test_consume_all(): with wrap({'aaa': {'bbb': {'hi': 123}}}) as w: + w = cast(Wdict, w) aaa = w['aaa'].zoom() aaa['bbb'].consume_all() + +# TODO type check this... diff --git a/my/reading/polar.py b/my/reading/polar.py index 4f79fcf..a38662d 100755 --- a/my/reading/polar.py +++ b/my/reading/polar.py @@ -35,27 +35,20 @@ config = make_config(polar) # https://github.com/burtonator/polar-bookshelf/issues/296 from datetime import datetime -from typing import List, Dict, Iterator, NamedTuple, Sequence, Optional +from typing import List, Dict, Iterable, NamedTuple, Sequence, Optional import json import pytz -from ..core import get_files, LazyLogger - -from ..error import Res, echain, unwrap, sort_res_by -from ..kython.konsume import wrap, zoom, ignore +from ..core import LazyLogger, Json +from ..core.common import isoparse +from ..error import Res, echain, sort_res_by +from ..kython.konsume import wrap, zoom, ignore, Zoomable, Wdict logger = LazyLogger(__name__) -# TODO use core.isoparse -def parse_dt(s: str) -> datetime: - return pytz.utc.localize(datetime.strptime(s, '%Y-%m-%dT%H:%M:%S.%fZ')) - -Uid = str - - # Ok I guess handling comment-level errors is a bit too much.. Cid = str class Comment(NamedTuple): @@ -71,6 +64,8 @@ class Highlight(NamedTuple): comments: Sequence[Comment] + +Uid = str class Book(NamedTuple): uid: Uid created: datetime @@ -80,8 +75,6 @@ class Book(NamedTuple): # think about it later. items: Sequence[Highlight] -Error = Exception # for backwards compat with Orger; can remove later - Result = Res[Book] class Loader: @@ -89,12 +82,13 @@ class Loader: self.path = p self.uid = self.path.parent.name - def error(self, cause, extra='') -> Exception: + def error(self, cause: Exception, extra: str ='') -> Exception: if len(extra) > 0: extra = '\n' + extra return echain(Exception(f'while processing {self.path}{extra}'), cause) - def load_item(self, meta) -> Iterator[Highlight]: + def load_item(self, meta: Zoomable) -> Iterable[Highlight]: + meta = cast(Wdict, meta) # TODO this should be destructive zoom? meta['notes'].zoom() meta['pagemarks'].zoom() @@ -134,7 +128,7 @@ class Loader: cmap[hlid] = ccs ccs.append(Comment( cid=cid.value, - created=parse_dt(crt.value), + created=isoparse(crt.value), text=html.value, # TODO perhaps coonvert from html to text or org? )) v.consume() @@ -162,7 +156,7 @@ class Loader: yield Highlight( hid=hid, - created=parse_dt(crt), + created=isoparse(crt), selection=text, comments=tuple(comments), ) @@ -174,12 +168,12 @@ class Loader: # TODO sort by date? - def load_items(self, metas) -> Iterator[Highlight]: + def load_items(self, metas: Json) -> Iterable[Highlight]: for p, meta in metas.items(): with wrap(meta, throw=False) as meta: yield from self.load_item(meta) - def load(self) -> Iterator[Result]: + def load(self) -> Iterable[Result]: logger.info('processing %s', self.path) j = json.loads(self.path.read_text()) @@ -193,14 +187,15 @@ class Loader: yield Book( uid=self.uid, - created=parse_dt(added), + created=isoparse(added), filename=filename, title=title, items=list(self.load_items(pm)), ) -def iter_entries() -> Iterator[Result]: +def iter_entries() -> Iterable[Result]: + from ..core import get_files for d in get_files(config.polar_dir, glob='*/state.json'): loader = Loader(d) try: @@ -213,16 +208,18 @@ def iter_entries() -> Iterator[Result]: def get_entries() -> List[Result]: # sorting by first annotation is reasonable I guess??? + # todo perhaps worth making it a pattern? X() returns iterable, get_X returns reasonably sorted list? return list(sort_res_by(iter_entries(), key=lambda e: e.created)) def main(): - for entry in iter_entries(): - try: - ee = unwrap(entry) - except Error as e: + for e in iter_entries(): + if isinstance(e, Exception): logger.exception(e) else: - logger.info('processed %s', ee.uid) - for i in ee.items: + logger.info('processed %s', e.uid) + for i in e.items: logger.info(i) + + +Error = Exception # for backwards compat with Orger; can remove later diff --git a/tests/extra/polar.py b/tests/extra/polar.py index 709f44f..3ed0342 100644 --- a/tests/extra/polar.py +++ b/tests/extra/polar.py @@ -1,6 +1,6 @@ from pathlib import Path -ROOT = Path(__file__).parent.parent.absolute() +ROOT = Path(__file__).parent.absolute() import pytest # type: ignore @@ -17,7 +17,7 @@ def test_hpi(dotpolar: str): if dotpolar != '': pdir = Path(ROOT / dotpolar) class user_config: - export_dir = pdir + polar_dir = pdir import my.config setattr(my.config, 'polar', user_config) @@ -30,4 +30,4 @@ def test_hpi(dotpolar: str): import my.reading.polar as polar from my.reading.polar import get_entries - assert len(list(get_entries())) > 10 + assert len(list(get_entries())) > 1