Merge pull request #51 from karlicoss/updates

Improve documentation for some modules

doc/MODULES.org

@@ -1,9 +1,11 @@
-This file is an overview of *documented* modules. There are many more, see [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules.
+This file is an overview of *documented* modules.
+There are many more, see [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules, I'm progressively working on documenting them.
 
 See [[file:SETUP.org][SETUP]] to find out how to set up your own config.
 
 Some explanations:
 
+- =MY_CONFIG= is whereever you are keeping your private configuration (usually =~/.config/my/=)
 - [[https://docs.python.org/3/library/pathlib.html#pathlib.Path][Path]] is a standard Python object to represent paths
 - [[https://github.com/karlicoss/HPI/blob/5f4acfddeeeba18237e8b039c8f62bcaa62a4ac2/my/core/common.py#L9][PathIsh]] is a helper type to allow using either =str=, or a =Path=
 - [[https://github.com/karlicoss/HPI/blob/5f4acfddeeeba18237e8b039c8f62bcaa62a4ac2/my/core/common.py#L108][Paths]] is another helper type for paths.
@@ -17,10 +19,12 @@ Some explanations:
 
   Typically, such variable will be passed to =get_files= to actually extract the list of real files to use. You can see usage examples [[https://github.com/karlicoss/HPI/blob/master/tests/get_files.py][here]].
 
-- if the field has a default value, you can omit it from your private config.
+- if the field has a default value, you can omit it from your private config altogether
 
 
-Modules:
+The config snippets below are meant to be modified accordingly and *pasted into your private configuration*, e.g =$MY_CONFIG/my/config.py=.
+
+You don't have to set them up all at once, it's recommended to do it gradually.
 
 #+begin_src python :dir .. :results output drawer :exports result
 # TODO ugh, pkgutil.walk_packages doesn't recurse and find packages like my.twitter.archive??
@@ -28,12 +32,14 @@ import importlib
 # from lint import all_modules # meh
 # TODO figure out how to discover configs automatically...
 modules = [
-    ('google' , 'my.google.takeout.paths'),
+    ('google'     , 'my.google.takeout.paths'),
-    ('reddit' , 'my.reddit'              ),
+    ('hypothesis' , 'my.hypothesis'          ),
-    ('twint'  , 'my.twitter.twint'       ),
+    ('reddit'     , 'my.reddit'              ),
-    ('twitter', 'my.twitter.archive'     ),
+    ('twint'      , 'my.twitter.twint'       ),
-    ('lastfm' , 'my.lastfm'              ),
+    ('twitter'    , 'my.twitter.archive'     ),
-    ('polar'  , 'my.reading.polar'       ),
+    ('lastfm'     , 'my.lastfm'              ),
+    ('polar'      , 'my.reading.polar'       ),
+    ('instapaper' , 'my.instapaper'          ),
 ]
 
 def indent(s, spaces=4):
@@ -78,16 +84,39 @@ for cls, p in modules:
     class google:
         takeout_path: Paths # path/paths/glob for the takeout zips
     #+end_src
+- [[file:../my/hypothesis.py][my.hypothesis]]
+
+    [[https://hypothes.is][Hypothes.is]] highlights and annotations
+
+    #+begin_src python
+    class hypothesis:
+        '''
+        Uses [[https://github.com/karlicoss/hypexport][hypexport]] outputs
+        '''
+
+        # paths[s]/glob to the exported JSON data
+        export_path: Paths
+
+        # path to a local clone of hypexport
+        # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/hypexport
+        hypexport  : Optional[PathIsh] = None
+    #+end_src
 - [[file:../my/reddit.py][my.reddit]]
 
     Reddit data: saved items/comments/upvotes/etc.
 
-    Uses [[https://github.com/karlicoss/rexport][rexport]] output.
-
     #+begin_src python
     class reddit:
-        export_path: Paths                     # path[s]/glob to the exported data
+        '''
-        rexport    : Optional[PathIsh] = None  # path to a local clone of rexport
+        Uses [[https://github.com/karlicoss/rexport][rexport]] output.
+        '''
+
+        # path[s]/glob to the exported JSON data
+        export_path: Paths
+
+        # path to a local clone of rexport
+        # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/rexport
+        rexport    : Optional[PathIsh] = None
     #+end_src
 - [[file:../my/twitter/twint.py][my.twitter.twint]]
 
@@ -127,6 +156,23 @@ for cls, p in modules:
         '''
         Polar config is optional, you only need it if you want to specify custom 'polar_dir'
         '''
-        polar_dir: Path = Path('~/.polar').expanduser()
+        polar_dir: PathIsh = Path('~/.polar').expanduser()
+        defensive: bool = True # pass False if you want it to fail faster on errors (useful for debugging)
+    #+end_src
+- [[file:../my/instapaper.py][my.instapaper]]
+
+    [[https://www.instapaper.com][Instapaper]] bookmarks, highlights and annotations
+
+    #+begin_src python
+    class instapaper:
+        '''
+        Uses [[https://github.com/karlicoss/instapexport][instapexport]] outputs.
+        '''
+        # path[s]/glob to the exported JSON data
+        export_path : Paths
+
+        # path to a local clone of instapexport
+        # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/instapexport
+        instapexport: Optional[PathIsh] = None
     #+end_src
 :end:

doc/SETUP.org

@@ -5,14 +5,14 @@ You'd be really helping me, I want to make the setup as straightforward as possi
 * Few notes
 I understand people may not super familiar with Python, PIP or generally unix, so here are some short notes:
 
-- only python3 is supported, and more specifically, ~python >= 3.5~.
+- only python3 is supported, and more specifically, ~python >= 3.6~.
 - I'm using ~pip3~ command, but on your system you might only have ~pip~.
 
   If your ~pip --version~ says python 3, feel free to use ~pip~.
 
 - similarly, I'm using =python3= in the documentation, but if your =python --version= says python3, it's okay to use =python=
 
-- when you are using ~pip install~, [[https://stackoverflow.com/a/42989020/706389][always pass]] =--user=
+- when you are using ~pip install~, [[https://stackoverflow.com/a/42989020/706389][always pass]] =--user=, and *never install third party packages with sudo* (unless you know what you are doing)
 - throughout the guide I'm assuming the config directory is =~/.config=, but it's different on Mac/Windows.
 
   See [[https://github.com/ActiveState/appdirs/blob/3fe6a83776843a46f20c2e5587afcffe05e03b39/appdirs.py#L187-L190][this]] if you're not sure what's your user config dir.

my/cfg.py

@@ -7,10 +7,9 @@ Usage:
 
 After that, you can set config attributes:
 
-  from types import SimpleNamespace
+  class user_config:
-  config.twitter = SimpleNamespace(
+      export_path = '/path/to/twitter/exports'
-      export_path='/path/to/twitter/exports',
+  config.twitter = user_config
-  )
 """
 # todo why do we bring this into scope? don't remember..
 import my.config as config

my/hypothesis.py

@@ -1,26 +1,63 @@
 """
 [[https://hypothes.is][Hypothes.is]] highlights and annotations
 """
-from .common import get_files
+from dataclasses import dataclass
-from .error import Res, sort_res_by
+from typing import Optional
 
-import my.config.repos.hypexport.dal as hypexport
+from .core import Paths, PathIsh
-from my.config import hypothesis as config
 
-###
+from my.config import hypothesis as user_config
+
+
+@dataclass
+class hypothesis(user_config):
+    '''
+    Uses [[https://github.com/karlicoss/hypexport][hypexport]] outputs
+    '''
+
+    # paths[s]/glob to the exported JSON data
+    export_path: Paths
+
+    # path to a local clone of hypexport
+    # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/hypexport
+    hypexport  : Optional[PathIsh] = None
+
+    @property
+    def dal_module(self):
+        rpath = self.hypexport
+        if rpath is not None:
+            from .cfg import set_repo
+            set_repo('hypexport', rpath)
+
+        import my.config.repos.hypexport.dal as dal
+        return dal
+
+
+from .core.cfg import make_config
+config = make_config(hypothesis)
+
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    import my.config.repos.hypexport.dal as dal
+else:
+    dal = config.dal_module
+
+############################
 
 from typing import List
+from .core.error import Res, sort_res_by
+
+Highlight = dal.Highlight
+Page      = dal.Page
 
 
-# TODO weird. not sure why e.g. from dal import Highlight doesn't work..
+def _dal() -> dal.DAL:
-Highlight = hypexport.Highlight
+    from .core import get_files
-Page      = hypexport.Page
+    sources = get_files(config.export_path)
+    return dal.DAL(sources)
 
 
-# TODO eh. not sure if I should rename everything to dao/DAO or not...
-def _dal() -> hypexport.DAL:
-    sources = get_files(config.export_path, '*.json')
-    return hypexport.DAL(sources)
 
 
 def highlights() -> List[Res[Highlight]]:
@@ -32,12 +69,6 @@ def pages() -> List[Res[Page]]:
     return sort_res_by(_dal().pages(), key=lambda h: h.created)
 
 
-# TODO move to side tests?
-def test():
-    list(pages())
-    list(highlights())
-
-
 def _main():
     for page in get_pages():
         print(page)

my/instapaper.py

@@ -1,18 +1,58 @@
 """
-Instapaper bookmarks, highlights and annotations
+[[https://www.instapaper.com][Instapaper]] bookmarks, highlights and annotations
 """
-from .common import get_files
+from dataclasses import dataclass
+from typing import Optional
+
+from .core import Paths, PathIsh
+
+from my.config import instapaper as user_config
 
 
-from my.config import instapaper as config
+@dataclass
-import my.config.repos.instapexport.dal as dal
+class instapaper(user_config):
+    '''
+    Uses [[https://github.com/karlicoss/instapexport][instapexport]] outputs.
+    '''
+    # path[s]/glob to the exported JSON data
+    export_path : Paths
 
+    # path to a local clone of instapexport
+    # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/instapexport
+    instapexport: Optional[PathIsh] = None
+
+    @property
+    def dal_module(self):
+        rpath = self.instapexport
+        if rpath is not None:
+            from .cfg import set_repo
+            set_repo('instapexport', rpath)
+
+        import my.config.repos.instapexport.dal as dal
+        return dal
+
+
+from .core.cfg import make_config
+config = make_config(instapaper)
+
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    import my.config.repos.instapexport.dal as dal
+else:
+    dal = config.dal_module
+
+############################
 
 Highlight = dal.Highlight
-Bookmark = dal.Bookmark
+Bookmark  = dal.Bookmark
+Page      = dal.Page
 
 
-def inputs():
+from typing import Sequence, Iterable
+from pathlib import Path
+from .core import get_files
+def inputs() -> Sequence[Path]:
     return get_files(config.export_path)
 
 
@@ -20,9 +60,8 @@ def _dal() -> dal.DAL:
     return dal.DAL(inputs())
 
 
-def pages():
+def pages() -> Iterable[Page]:
     return _dal().pages()
-get_pages = pages # todo also deprecate..
 
 
 # TODO dunno, move this to private?
@@ -30,3 +69,6 @@ def is_todo(hl: Highlight) -> bool:
     note = hl.note or ''
     note = note.lstrip().lower()
     return note.startswith('todo')
+
+
+get_pages = pages # todo also deprecate..

my/reddit.py

@@ -1,7 +1,5 @@
 """
 Reddit data: saved items/comments/upvotes/etc.
-
-Uses [[https://github.com/karlicoss/rexport][rexport]] output.
 """
 
 from typing import Optional
@@ -13,20 +11,26 @@ from dataclasses import dataclass
 
 @dataclass
 class reddit(uconfig):
-    export_path: Paths                     # path[s]/glob to the exported data
+    '''
-    rexport    : Optional[PathIsh] = None  # path to a local clone of rexport
+    Uses [[https://github.com/karlicoss/rexport][rexport]] output.
+    '''
+
+    # path[s]/glob to the exported JSON data
+    export_path: Paths
+
+    # path to a local clone of rexport
+    # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/rexport
+    rexport    : Optional[PathIsh] = None
 
     @property
-    def rexport_module(self) -> ModuleType:
+    def dal_module(self) -> ModuleType:
-        # todo return Type[rexport]??
-        # todo ModuleIsh?
         rpath = self.rexport
         if rpath is not None:
-            from my.cfg import set_repo
+            from .cfg import set_repo
             set_repo('rexport', rpath)
 
-        import my.config.repos.rexport.dal as m
+        import my.config.repos.rexport.dal as dal
-        return m
+        return dal
 
 
 from .core.cfg import make_config, Attrs
@@ -43,15 +47,16 @@ config = make_config(reddit, migration=migration)
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     # TODO not sure what is the right way to handle this..
-    import my.config.repos.rexport.dal as rexport
+    import my.config.repos.rexport.dal as dal
 else:
     # TODO ugh. this would import too early
     # but on the other hand we do want to bring the objects into the scope for easier imports, etc. ugh!
     # ok, fair enough I suppose. It makes sense to configure something before using it. can always figure it out later..
     # maybe, the config could dynamically detect change and reimport itself? dunno.
-    rexport = config.rexport_module
+    dal = config.dal_module
 ###
 
+############################
 
 from typing import List, Sequence, Mapping, Iterator
 from .core.common import mcachew, get_files, LazyLogger, make_dict
@@ -70,35 +75,35 @@ def inputs() -> Sequence[Path]:
     return tuple(res)
 
 
-Sid        = rexport.Sid
+Sid        = dal.Sid
-Save       = rexport.Save
+Save       = dal.Save
-Comment    = rexport.Comment
+Comment    = dal.Comment
-Submission = rexport.Submission
+Submission = dal.Submission
-Upvote     = rexport.Upvote
+Upvote     = dal.Upvote
 
 
-def dal() -> rexport.DAL:
+def _dal() -> dal.DAL:
-    return rexport.DAL(inputs())
+    return dal.DAL(inputs())
 
 
 @mcachew(hashf=lambda: inputs())
 def saved() -> Iterator[Save]:
-    return dal().saved()
+    return _dal().saved()
 
 
 @mcachew(hashf=lambda: inputs())
 def comments() -> Iterator[Comment]:
-    return dal().comments()
+    return _dal().comments()
 
 
 @mcachew(hashf=lambda: inputs())
 def submissions() -> Iterator[Submission]:
-    return dal().submissions()
+    return _dal().submissions()
 
 
 @mcachew(hashf=lambda: inputs())
 def upvoted() -> Iterator[Upvote]:
-    return dal().upvoted()
+    return _dal().upvoted()
 
 
 ### the rest of the file is some elaborate attempt of restoring favorite/unfavorite times
@@ -151,7 +156,7 @@ def _get_state(bfile: Path) -> Dict[Sid, SaveWithDt]:
 
     bdt = _get_bdate(bfile)
 
-    saves = [SaveWithDt(save, bdt) for save in rexport.DAL([bfile]).saved()]
+    saves = [SaveWithDt(save, bdt) for save in dal.DAL([bfile]).saved()]
     return make_dict(
         sorted(saves, key=lambda p: p.save.created),
         key=lambda s: s.save.sid,

setup.py

@@ -46,6 +46,7 @@ def main():
         author_email='karlicoss@gmail.com',
         description='A Python interface to my life',
 
+        python_requires='>=3.6',
         install_requires=INSTALL_REQUIRES,
         extras_require={
             'testing': [

tests/config.py

@@ -1,21 +1,18 @@
 from pathlib import Path
 
 
-# TODO switch these from using SimpleNamespace
-
 def setup_notes_path(notes: Path) -> None:
     # TODO reuse doc from my.cfg?
     from my.cfg import config
 
-    from types import SimpleNamespace
+    class user_config:
-    config.orgmode = SimpleNamespace( # type: ignore[misc,assignment]
+        roots = [notes]
-        roots=[notes],
+    config.orgmode = user_config # type: ignore[misc,assignment]
-    )
     # TODO FIXME ugh. this belongs to tz provider or global config or someting
     import pytz
-    config.weight  = SimpleNamespace( # type: ignore[misc,assignment]
+    class user_config_2:
         default_timezone = pytz.timezone('Europe/London')
-    )
+    config.weight  = user_config_2 # type: ignore[misc,assignment]
 
 
 def test_dynamic_configuration(notes: Path) -> None:
@@ -38,10 +35,9 @@ import pytest # type: ignore
 
 def test_set_repo(tmp_path: Path) -> None:
     from my.cfg import config
-    from types import SimpleNamespace
+    class user_config:
-    config.hypothesis = SimpleNamespace( # type: ignore[misc,assignment]
+        export_path = 'whatever',
-        export_path='whatever',
+    config.hypothesis = user_config # type: ignore[misc,assignment]
-    )
 
     # precondition:
     # should fail because can't find hypexport

tests/hypothesis.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from my.hypothesis import pages, highlights
+
+def test():
+    assert len(list(pages())) > 10
+    assert len(list(highlights())) > 10