polar: minor improvements, konsume: more type annotations

my/core/common.py

@@ -134,7 +134,8 @@ def get_files(pp: Paths, glob: str=DEFAULT_GLOB, sort: bool=True) -> Tuple[Path,
                     warnings.warn(f"Treating {ss} as glob path. Explicit glob={glob} argument is ignored!")
                 paths.extend(map(Path, do_glob(ss)))
             else:
-                assert src.is_file(), src
+                if not src.is_file():
+                    raise RuntimeError(f"Expected '{src}' to exist")
                 # todo assert matches glob??
                 paths.append(src)
 

my/kython/konsume.py

@@ -11,7 +11,7 @@ def zoom(w, *keys):
 
 # TODO need to support lists
 class Zoomable:
-    def __init__(self, parent, *args, **kwargs):
+    def __init__(self, parent, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs) # type: ignore
         self.parent = parent
 
@@ -21,19 +21,19 @@ class Zoomable:
     def dependants(self):
         raise NotImplementedError
 
-    def ignore(self):
+    def ignore(self) -> None:
         self.consume_all()
 
-    def consume_all(self):
+    def consume_all(self) -> None:
         for d in self.dependants:
             d.consume_all()
         self.consume()
 
-    def consume(self):
+    def consume(self) -> None:
         assert self.parent is not None
         self.parent._remove(self)
 
-    def zoom(self):
+    def zoom(self) -> 'Zoomable':
         self.consume()
         return self
 
@@ -56,6 +56,8 @@ class Wdict(Zoomable, OrderedDict):
 
     def this_consumed(self):
         return len(self) == 0
+    # TODO specify mypy type for the index special method?
+
 
 class Wlist(Zoomable, list):
     def _remove(self, xx):
@@ -83,7 +85,8 @@ class Wvalue(Zoomable):
     def __repr__(self):
         return 'WValue{' + repr(self.value) + '}'
 
-def _wrap(j, parent=None):
+from typing import Tuple
+def _wrap(j, parent=None) -> Tuple[Zoomable, List[Zoomable]]:
     res: Zoomable
     cc: List[Zoomable]
     if isinstance(j, dict):
@@ -109,13 +112,14 @@ def _wrap(j, parent=None):
         raise RuntimeError(f'Unexpected type: {type(j)} {j}')
 
 from contextlib import contextmanager
+from typing import Iterator
 
 class UnconsumedError(Exception):
     pass
 
 # TODO think about error policy later...
 @contextmanager
-def wrap(j, throw=True):
+def wrap(j, throw=True) -> Iterator[Zoomable]:
     w, children = _wrap(j)
 
     yield w
@@ -128,28 +132,33 @@ def wrap(j, throw=True):
                 # TODO log?
                 pass
 
-
+from typing import cast
 def test_unconsumed():
     import pytest # type: ignore
     with pytest.raises(UnconsumedError):
         with wrap({'a': 1234}) as w:
+            w = cast(Wdict, w)
             pass
 
     with pytest.raises(UnconsumedError):
         with wrap({'c': {'d': 2222}}) as w:
+            w = cast(Wdict, w)
             d = w['c']['d'].zoom()
 
 def test_consumed():
     with wrap({'a': 1234}) as w:
+        w = cast(Wdict, w)
         a = w['a'].zoom()
 
     with wrap({'c': {'d': 2222}}) as w:
+        w = cast(Wdict, w)
         c = w['c'].zoom()
         d = c['d'].zoom()
 
 def test_types():
     # (string, number, object, array, boolean or nul
     with wrap({'string': 'string', 'number': 3.14, 'boolean': True, 'null': None, 'list': [1, 2, 3]}) as w:
+        w = cast(Wdict, w)
         w['string'].zoom()
         w['number'].consume()
         w['boolean'].zoom()
@@ -159,5 +168,8 @@ def test_types():
 
 def test_consume_all():
     with wrap({'aaa': {'bbb': {'hi': 123}}}) as w:
+        w = cast(Wdict, w)
         aaa = w['aaa'].zoom()
         aaa['bbb'].consume_all()
+
+# TODO type check this...

my/reading/polar.py

@@ -35,27 +35,20 @@ config = make_config(polar)
 # https://github.com/burtonator/polar-bookshelf/issues/296
 
 from datetime import datetime
-from typing import List, Dict, Iterator, NamedTuple, Sequence, Optional
+from typing import List, Dict, Iterable, NamedTuple, Sequence, Optional
 import json
 
 import pytz
 
-from ..core import get_files, LazyLogger
+from ..core import LazyLogger, Json
-
+from ..core.common import isoparse
-from ..error import Res, echain, unwrap, sort_res_by
+from ..error import Res, echain, sort_res_by
-from ..kython.konsume import wrap, zoom, ignore
+from ..kython.konsume import wrap, zoom, ignore, Zoomable, Wdict
 
 
 logger = LazyLogger(__name__)
 
 
-# TODO use core.isoparse
-def parse_dt(s: str) -> datetime:
-    return pytz.utc.localize(datetime.strptime(s, '%Y-%m-%dT%H:%M:%S.%fZ'))
-
-Uid = str
-
-
 # Ok I guess handling comment-level errors is a bit too much..
 Cid = str
 class Comment(NamedTuple):
@@ -71,6 +64,8 @@ class Highlight(NamedTuple):
     comments: Sequence[Comment]
 
 
+
+Uid = str
 class Book(NamedTuple):
     uid: Uid
     created: datetime
@@ -80,8 +75,6 @@ class Book(NamedTuple):
     # think about it later.
     items: Sequence[Highlight]
 
-Error = Exception # for backwards compat with Orger; can remove later
-
 Result = Res[Book]
 
 class Loader:
@@ -89,12 +82,13 @@ class Loader:
         self.path = p
         self.uid = self.path.parent.name
 
-    def error(self, cause, extra='') -> Exception:
+    def error(self, cause: Exception, extra: str ='') -> Exception:
         if len(extra) > 0:
             extra = '\n' + extra
         return echain(Exception(f'while processing {self.path}{extra}'), cause)
 
-    def load_item(self, meta) -> Iterator[Highlight]:
+    def load_item(self, meta: Zoomable) -> Iterable[Highlight]:
+        meta = cast(Wdict, meta)
         # TODO this should be destructive zoom?
         meta['notes'].zoom()
         meta['pagemarks'].zoom()
@@ -134,7 +128,7 @@ class Loader:
             cmap[hlid] = ccs
             ccs.append(Comment(
                 cid=cid.value,
-                created=parse_dt(crt.value),
+                created=isoparse(crt.value),
                 text=html.value, # TODO perhaps coonvert from html to text or org?
             ))
             v.consume()
@@ -162,7 +156,7 @@ class Loader:
 
             yield Highlight(
                 hid=hid,
-                created=parse_dt(crt),
+                created=isoparse(crt),
                 selection=text,
                 comments=tuple(comments),
             )
@@ -174,12 +168,12 @@ class Loader:
         # TODO sort by date?
 
 
-    def load_items(self, metas) -> Iterator[Highlight]:
+    def load_items(self, metas: Json) -> Iterable[Highlight]:
         for p, meta in metas.items():
             with wrap(meta, throw=False) as meta:
                 yield from self.load_item(meta)
 
-    def load(self) -> Iterator[Result]:
+    def load(self) -> Iterable[Result]:
         logger.info('processing %s', self.path)
         j = json.loads(self.path.read_text())
 
@@ -193,14 +187,15 @@ class Loader:
 
         yield Book(
             uid=self.uid,
-            created=parse_dt(added),
+            created=isoparse(added),
             filename=filename,
             title=title,
             items=list(self.load_items(pm)),
         )
 
 
-def iter_entries() -> Iterator[Result]:
+def iter_entries() -> Iterable[Result]:
+    from ..core import get_files
     for d in get_files(config.polar_dir, glob='*/state.json'):
         loader = Loader(d)
         try:
@@ -213,16 +208,18 @@ def iter_entries() -> Iterator[Result]:
 
 def get_entries() -> List[Result]:
     # sorting by first annotation is reasonable I guess???
+    # todo perhaps worth making it a pattern? X() returns iterable, get_X returns reasonably sorted list?
     return list(sort_res_by(iter_entries(), key=lambda e: e.created))
 
 
 def main():
-    for entry in iter_entries():
+    for e in iter_entries():
-        try:
+        if isinstance(e, Exception):
-            ee = unwrap(entry)
-        except Error as e:
             logger.exception(e)
         else:
-            logger.info('processed %s', ee.uid)
+            logger.info('processed %s', e.uid)
-            for i in ee.items:
+            for i in e.items:
                 logger.info(i)
+
+
+Error = Exception # for backwards compat with Orger; can remove later

tests/extra/polar.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-ROOT = Path(__file__).parent.parent.absolute()
+ROOT = Path(__file__).parent.absolute()
 
 
 import pytest # type: ignore
@@ -17,7 +17,7 @@ def test_hpi(dotpolar: str):
     if dotpolar != '':
         pdir = Path(ROOT / dotpolar)
         class user_config:
-            export_dir = pdir
+            polar_dir = pdir
 
         import my.config
         setattr(my.config, 'polar', user_config)
@@ -30,4 +30,4 @@ def test_hpi(dotpolar: str):
 
     import my.reading.polar as polar
     from my.reading.polar import get_entries
-    assert len(list(get_entries())) > 10
+    assert len(list(get_entries())) > 1