Merge pull request #62 from karlicoss/updates

updates: core & kobo

doc/MODULES.org

@@ -27,6 +27,7 @@ If you have some issues with the setup, see [[file:SETUP.org::#troubleshooting][
   - [[#myinstapaper][my.instapaper]]
   - [[#mygithubgdpr][my.github.gdpr]]
   - [[#mygithubghexport][my.github.ghexport]]
+  - [[#mykobo][my.kobo]]
 :END:
 
 * Intro
@@ -80,6 +81,7 @@ modules = [
     ('instapaper'     , 'my.instapaper'          ),
     ('github'         , 'my.github.gdpr'         ),
     ('github'         , 'my.github.ghexport'     ),
+    ('kobo'           , 'my.kobo'                ),
 ]
 
 def indent(s, spaces=4):
@@ -261,3 +263,15 @@ for cls, p in modules:
         # if omitted, will use /tmp
         cache_dir: Optional[PathIsh] = None
     #+end_src
+** [[file:../my/kobo.py][my.kobo]]
+
+    [[https://uk.kobobooks.com/products/kobo-aura-one][Kobo]] e-ink reader: annotations and reading stats
+
+    #+begin_src python
+    class kobo:
+        '''
+        Uses [[https://github.com/karlicoss/kobuddy#as-a-backup-tool][kobuddy]] outputs.
+        '''
+        # path[s]/glob to the exported databases
+        export_path: Paths
+    #+end_src

my/books/kobo.py

@@ -1,45 +1,5 @@
-"""
-[[https://uk.kobobooks.com/products/kobo-aura-one][Kobo]] e-ink reader: annotations and reading stats
-"""
-from typing import Callable, Union, List
+import warnings
 
-from my.config import kobo as config
-from my.config.repos.kobuddy.src.kobuddy import *
-# hmm, explicit imports make pylint a bit happier..
-from my.config.repos.kobuddy.src.kobuddy import Highlight, set_databases, get_highlights
+warnings.warn('my.books.kobo is deprecated! Please use my.kobo instead!')
 
-set_databases(config.export_dir)
-
-# TODO maybe type over T?
-_Predicate = Callable[[str], bool]
-Predicatish = Union[str, _Predicate]
-def from_predicatish(p: Predicatish) -> _Predicate:
-    if isinstance(p, str):
-        def ff(s):
-            return s == p
-        return ff
-    else:
-        return p
-
-
-def by_annotation(predicatish: Predicatish, **kwargs) -> List[Highlight]:
-    pred = from_predicatish(predicatish)
-
-    res: List[Highlight] = []
-    for h in get_highlights(**kwargs):
-        if pred(h.annotation):
-            res.append(h)
-    return res
-
-
-def get_todos():
-    def with_todo(ann):
-        if ann is None:
-            ann = ''
-        return 'todo' in ann.lower().split()
-    return by_annotation(with_todo)
-
-
-def test_todos():
-    todos = get_todos()
-    assert len(todos) > 3
+from ..kobo import *

my/coding/github.py

@@ -1,6 +1,7 @@
 import warnings
 
-warnings.warn('my.coding.github is deprecated! Please use my.github.all instead!', DeprecationWarning)
+warnings.warn('my.coding.github is deprecated! Please use my.github.all instead!')
+# todo why aren't DeprecationWarning shown by default??
 
 from ..github.all import events, get_events
 

my/core/__init__.py

@@ -1,5 +1,8 @@
 # this file only keeps the most common & critical types/utility functions
 from .common import PathIsh, Paths, Json
-from .common import get_files, LazyLogger
+from .common import get_files
+from .common import LazyLogger
 from .common import warn_if_empty
+from .common import stat
+
 from .cfg import make_config

my/core/__main__.py

@@ -133,6 +133,10 @@ def modules_check(args):
         stats = getattr(mod, 'stats', None)
         if stats is None:
             continue
+        from . import common
+        common.QUICK_STATS = True
+        # todo make it a cmdline option..
+
         try:
             res = stats()
         except Exception as ee:

my/core/common.py

@@ -338,3 +338,30 @@ def warn_if_empty(f):
         res = f(*args, **kwargs)
         return _warn_iterable(res, f=f)
     return wrapped # type: ignore
+
+
+# hacky hook to speed up for 'hpi doctor'
+# todo think about something better
+QUICK_STATS = False
+
+
+C = TypeVar('C')
+# todo not sure about return type...
+def stat(func: Callable[[], Iterable[C]]) -> Dict[str, Any]:
+    from more_itertools import ilen, take, first
+
+    it = iter(func())
+    res: Any
+    if QUICK_STATS:
+        initial = take(100, it)
+        res = len(initial)
+        if first(it, None) is not None: # todo can actually be none...
+            # haven't exhausted
+            res = f'{res}+'
+    else:
+        res = ilen(it)
+
+
+    return {
+        func.__name__: res,
+    }

my/github/gdpr.py

@@ -66,6 +66,13 @@ def events() -> Iterable[Res[Event]]:
                 yield e
 
 
+def stats():
+    from ..core import stat
+    return {
+        **stat(events),
+    }
+
+
 # TODO typing.TypedDict could be handy here..
 def _parse_common(d: Dict) -> Dict:
     url = d['url']

my/github/ghexport.py

@@ -29,7 +29,7 @@ class github(user_config):
     def dal_module(self):
         rpath = self.ghexport
         if rpath is not None:
-            from .core.common import import_dir
+            from ..core.common import import_dir
             return import_dir(rpath, '.dal')
         else:
             import my.config.repos.ghexport.dal as dal
@@ -81,6 +81,13 @@ def events(dal=_dal()) -> Results:
         yield _parse_event(d)
 
 
+def stats():
+    from ..core import stat
+    return {
+        **stat(events),
+    }
+
+
 # TODO hmm. need some sort of abstract syntax for this...
 # TODO split further, title too
 def _get_summary(e) -> Tuple[str, Optional[str], Optional[str]]:

my/kobo.py

@@ -0,0 +1,76 @@
+"""
+[[https://uk.kobobooks.com/products/kobo-aura-one][Kobo]] e-ink reader: annotations and reading stats
+"""
+
+# TODO require installing kobuddy, need to upload to pypi as well?
+from dataclasses import dataclass
+from .core import Paths
+from my.config import kobo as user_config
+@dataclass
+class kobo(user_config):
+    '''
+    Uses [[https://github.com/karlicoss/kobuddy#as-a-backup-tool][kobuddy]] outputs.
+    '''
+    # path[s]/glob to the exported databases
+    export_path: Paths
+
+
+from .core.cfg import make_config
+config = make_config(kobo)
+
+from .core import get_files
+import kobuddy
+# todo not sure about this glob..
+kobuddy.DATABASES = list(get_files(config.export_path, glob='*.sqlite'))
+
+#########################
+
+# hmm, explicit imports make pylint a bit happier?
+from kobuddy import Highlight, get_highlights
+from kobuddy import *
+
+
+
+def stats():
+    from .core import stat
+    return {
+        **stat(get_highlights),
+    }
+
+## TODO hmm. not sure if all this really belongs here?... perhaps orger?
+
+
+from typing import Callable, Union, List
+# TODO maybe type over T?
+_Predicate = Callable[[str], bool]
+Predicatish = Union[str, _Predicate]
+def from_predicatish(p: Predicatish) -> _Predicate:
+    if isinstance(p, str):
+        def ff(s):
+            return s == p
+        return ff
+    else:
+        return p
+
+
+def by_annotation(predicatish: Predicatish, **kwargs) -> List[Highlight]:
+    pred = from_predicatish(predicatish)
+
+    res: List[Highlight] = []
+    for h in get_highlights(**kwargs):
+        if pred(h.annotation):
+            res.append(h)
+    return res
+
+
+def get_todos():
+    def with_todo(ann):
+        if ann is None:
+            ann = ''
+        return 'todo' in ann.lower().split()
+    return by_annotation(with_todo)
+
+
+def test_todos():
+    todos = get_todos()
+    assert len(todos) > 3

my/kython/kompress.py

@@ -43,7 +43,8 @@ def kopen(path: PathIsh, *args, mode: str='rt', **kwargs) -> IO[str]:
         ifile.seekable = lambda: False # type: ignore
         ifile.read1    = ifile.read    # type: ignore
         # TODO pass all kwargs here??
-        return io.TextIOWrapper(ifile, encoding=encoding)
+        # todo 'expected "BinaryIO"'??
+        return io.TextIOWrapper(ifile, encoding=encoding) # type: ignore[arg-type]
     elif suf in {'.lz4'}:
         import lz4.frame # type: ignore
         return lz4.frame.open(str(pp), mode, *args, **kwargs)

my/photos/__init__.py

@@ -123,7 +123,8 @@ def _candidates() -> Iterable[str]:
             '.',
             *config.paths,
     ], stdout=PIPE) as p:
-        for line in p.stdout:
+        out = p.stdout; assert out is not None
+        for line in out:
             path = line.decode('utf8').rstrip('\n')
             mime = fastermime(path)
             tp = mime.split('/')[0]

my/twitter/archive.py

@@ -183,3 +183,11 @@ def tweets() -> Iterable[Tweet]:
 def likes() -> Iterable[Like]:
     for inp in inputs():
         yield from ZipExport(inp).likes()
+
+
+def stats():
+    from ..core import stat
+    return {
+        **stat(tweets),
+        **stat(likes),
+    }

my/twitter/twint.py

@@ -108,3 +108,11 @@ def likes() -> Iterable[Tweet]:
     db = _get_db()
     res = db.query(_QUERY.format(where='F.tweet_id IS NOT NULL'))
     yield from map(Tweet, res)
+
+
+def stats():
+    from ..core import stat
+    return {
+        **stat(tweets),
+        **stat(likes),
+    }

tests/github.py

@@ -3,6 +3,7 @@ from more_itertools import ilen
 
 from my.coding.github import get_events
 
+# todo test against stats? not sure.. maybe both
 
 def test_gdpr():
     import my.github.gdpr as gdpr