fz0x1/HPI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

polar: minor improvements, konsume: more type annotations

Browse source
This commit is contained in:
Dima Gerasimov 2020-05-15 08:40:12 +01:00
parent f3d5064ff2
commit 0f27071dcc
4 changed files with 50 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
my/core/common.py
View file

@ -134,7 +134,8 @@ def get_files(pp: Paths, glob: str=DEFAULT_GLOB, sort: bool=True) -> Tuple[Path,
warnings.warn(f"Treating {ss} as glob path. Explicit glob={glob} argument is ignored!")
paths.extend(map(Path, do_glob(ss)))
else:
assert src.is_file(), src
if not src.is_file():
raise RuntimeError(f"Expected '{src}' to exist")
# todo assert matches glob??
paths.append(src)

28
my/kython/konsume.py
View file

@ -11,7 +11,7 @@ def zoom(w, *keys):
# TODO need to support lists
class Zoomable:
def __init__(self, parent, *args, **kwargs):
def __init__(self, parent, *args, **kwargs) -> None:
super().__init__(*args, **kwargs) # type: ignore
self.parent = parent
@ -21,19 +21,19 @@ class Zoomable:
def dependants(self):
raise NotImplementedError
def ignore(self):
def ignore(self) -> None:
self.consume_all()
def consume_all(self):
def consume_all(self) -> None:
for d in self.dependants:
d.consume_all()
self.consume()
def consume(self):
def consume(self) -> None:
assert self.parent is not None
self.parent._remove(self)
def zoom(self):
def zoom(self) -> 'Zoomable':
self.consume()
return self
@ -56,6 +56,8 @@ class Wdict(Zoomable, OrderedDict):
def this_consumed(self):
return len(self) == 0
# TODO specify mypy type for the index special method?
class Wlist(Zoomable, list):
def _remove(self, xx):
@ -83,7 +85,8 @@ class Wvalue(Zoomable):
def __repr__(self):
return 'WValue{' + repr(self.value) + '}'
def _wrap(j, parent=None):
from typing import Tuple
def _wrap(j, parent=None) -> Tuple[Zoomable, List[Zoomable]]:
res: Zoomable
cc: List[Zoomable]
if isinstance(j, dict):
@ -109,13 +112,14 @@ def _wrap(j, parent=None):
raise RuntimeError(f'Unexpected type: {type(j)} {j}')
from contextlib import contextmanager
from typing import Iterator
class UnconsumedError(Exception):
pass
# TODO think about error policy later...
@contextmanager
def wrap(j, throw=True):
def wrap(j, throw=True) -> Iterator[Zoomable]:
w, children = _wrap(j)
yield w
@ -128,28 +132,33 @@ def wrap(j, throw=True):
# TODO log?
pass
from typing import cast
def test_unconsumed():
import pytest # type: ignore
with pytest.raises(UnconsumedError):
with wrap({'a': 1234}) as w:
w = cast(Wdict, w)
pass
with pytest.raises(UnconsumedError):
with wrap({'c': {'d': 2222}}) as w:
w = cast(Wdict, w)
d = w['c']['d'].zoom()
def test_consumed():
with wrap({'a': 1234}) as w:
w = cast(Wdict, w)
a = w['a'].zoom()
with wrap({'c': {'d': 2222}}) as w:
w = cast(Wdict, w)
c = w['c'].zoom()
d = c['d'].zoom()
def test_types():
# (string, number, object, array, boolean or nul
with wrap({'string': 'string', 'number': 3.14, 'boolean': True, 'null': None, 'list': [1, 2, 3]}) as w:
w = cast(Wdict, w)
w['string'].zoom()
w['number'].consume()
w['boolean'].zoom()
@ -159,5 +168,8 @@ def test_types():
def test_consume_all():
with wrap({'aaa': {'bbb': {'hi': 123}}}) as w:
w = cast(Wdict, w)
aaa = w['aaa'].zoom()
aaa['bbb'].consume_all()
# TODO type check this...

53
my/reading/polar.py
View file

@ -35,27 +35,20 @@ config = make_config(polar)
# https://github.com/burtonator/polar-bookshelf/issues/296
from datetime import datetime
from typing import List, Dict, Iterator, NamedTuple, Sequence, Optional
from typing import List, Dict, Iterable, NamedTuple, Sequence, Optional
import json
import pytz
from ..core import get_files, LazyLogger
from ..error import Res, echain, unwrap, sort_res_by
from ..kython.konsume import wrap, zoom, ignore
from ..core import LazyLogger, Json
from ..core.common import isoparse
from ..error import Res, echain, sort_res_by
from ..kython.konsume import wrap, zoom, ignore, Zoomable, Wdict
logger = LazyLogger(__name__)
# TODO use core.isoparse
def parse_dt(s: str) -> datetime:
return pytz.utc.localize(datetime.strptime(s, '%Y-%m-%dT%H:%M:%S.%fZ'))
Uid = str
# Ok I guess handling comment-level errors is a bit too much..
Cid = str
class Comment(NamedTuple):
@ -71,6 +64,8 @@ class Highlight(NamedTuple):
comments: Sequence[Comment]
Uid = str
class Book(NamedTuple):
uid: Uid
created: datetime
@ -80,8 +75,6 @@ class Book(NamedTuple):
# think about it later.
items: Sequence[Highlight]
Error = Exception # for backwards compat with Orger; can remove later
Result = Res[Book]
class Loader:
@ -89,12 +82,13 @@ class Loader:
self.path = p
self.uid = self.path.parent.name
def error(self, cause, extra='') -> Exception:
def error(self, cause: Exception, extra: str ='') -> Exception:
if len(extra) > 0:
extra = '\n' + extra
return echain(Exception(f'while processing {self.path}{extra}'), cause)
def load_item(self, meta) -> Iterator[Highlight]:
def load_item(self, meta: Zoomable) -> Iterable[Highlight]:
meta = cast(Wdict, meta)
# TODO this should be destructive zoom?
meta['notes'].zoom()
meta['pagemarks'].zoom()
@ -134,7 +128,7 @@ class Loader:
cmap[hlid] = ccs
ccs.append(Comment(
cid=cid.value,
created=parse_dt(crt.value),
created=isoparse(crt.value),
text=html.value, # TODO perhaps coonvert from html to text or org?
))
v.consume()
@ -162,7 +156,7 @@ class Loader:
yield Highlight(
hid=hid,
created=parse_dt(crt),
created=isoparse(crt),
selection=text,
comments=tuple(comments),
)
@ -174,12 +168,12 @@ class Loader:
# TODO sort by date?
def load_items(self, metas) -> Iterator[Highlight]:
def load_items(self, metas: Json) -> Iterable[Highlight]:
for p, meta in metas.items():
with wrap(meta, throw=False) as meta:
yield from self.load_item(meta)
def load(self) -> Iterator[Result]:
def load(self) -> Iterable[Result]:
logger.info('processing %s', self.path)
j = json.loads(self.path.read_text())
@ -193,14 +187,15 @@ class Loader:
yield Book(
uid=self.uid,
created=parse_dt(added),
created=isoparse(added),
filename=filename,
title=title,
items=list(self.load_items(pm)),
)
def iter_entries() -> Iterator[Result]:
def iter_entries() -> Iterable[Result]:
from ..core import get_files
for d in get_files(config.polar_dir, glob='*/state.json'):
loader = Loader(d)
try:
@ -213,16 +208,18 @@ def iter_entries() -> Iterator[Result]:
def get_entries() -> List[Result]:
# sorting by first annotation is reasonable I guess???
# todo perhaps worth making it a pattern? X() returns iterable, get_X returns reasonably sorted list?
return list(sort_res_by(iter_entries(), key=lambda e: e.created))
def main():
for entry in iter_entries():
try:
ee = unwrap(entry)
except Error as e:
for e in iter_entries():
if isinstance(e, Exception):
logger.exception(e)
else:
logger.info('processed %s', ee.uid)
for i in ee.items:
logger.info('processed %s', e.uid)
for i in e.items:
logger.info(i)
Error = Exception # for backwards compat with Orger; can remove later

6
tests/extra/polar.py
View file

@ -1,6 +1,6 @@
from pathlib import Path
ROOT = Path(__file__).parent.parent.absolute()
ROOT = Path(__file__).parent.absolute()
import pytest # type: ignore
@ -17,7 +17,7 @@ def test_hpi(dotpolar: str):
if dotpolar != '':
pdir = Path(ROOT / dotpolar)
class user_config:
export_dir = pdir
polar_dir = pdir
import my.config
setattr(my.config, 'polar', user_config)
@ -30,4 +30,4 @@ def test_hpi(dotpolar: str):
import my.reading.polar as polar
from my.reading.polar import get_entries
assert len(list(get_entries())) > 10
assert len(list(get_entries())) > 1