diff --git a/doc/MODULES.org b/doc/MODULES.org index 0e01188..d896eaa 100644 --- a/doc/MODULES.org +++ b/doc/MODULES.org @@ -1,9 +1,11 @@ -This file is an overview of *documented* modules. There are many more, see [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules. +This file is an overview of *documented* modules. +There are many more, see [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules, I'm progressively working on documenting them. See [[file:SETUP.org][SETUP]] to find out how to set up your own config. Some explanations: +- =MY_CONFIG= is whereever you are keeping your private configuration (usually =~/.config/my/=) - [[https://docs.python.org/3/library/pathlib.html#pathlib.Path][Path]] is a standard Python object to represent paths - [[https://github.com/karlicoss/HPI/blob/5f4acfddeeeba18237e8b039c8f62bcaa62a4ac2/my/core/common.py#L9][PathIsh]] is a helper type to allow using either =str=, or a =Path= - [[https://github.com/karlicoss/HPI/blob/5f4acfddeeeba18237e8b039c8f62bcaa62a4ac2/my/core/common.py#L108][Paths]] is another helper type for paths. @@ -17,10 +19,12 @@ Some explanations: Typically, such variable will be passed to =get_files= to actually extract the list of real files to use. You can see usage examples [[https://github.com/karlicoss/HPI/blob/master/tests/get_files.py][here]]. -- if the field has a default value, you can omit it from your private config. +- if the field has a default value, you can omit it from your private config altogether -Modules: +The config snippets below are meant to be modified accordingly and *pasted into your private configuration*, e.g =$MY_CONFIG/my/config.py=. + +You don't have to set them up all at once, it's recommended to do it gradually. #+begin_src python :dir .. :results output drawer :exports result # TODO ugh, pkgutil.walk_packages doesn't recurse and find packages like my.twitter.archive?? @@ -28,12 +32,14 @@ import importlib # from lint import all_modules # meh # TODO figure out how to discover configs automatically... modules = [ - ('google' , 'my.google.takeout.paths'), - ('reddit' , 'my.reddit' ), - ('twint' , 'my.twitter.twint' ), - ('twitter', 'my.twitter.archive' ), - ('lastfm' , 'my.lastfm' ), - ('polar' , 'my.reading.polar' ), + ('google' , 'my.google.takeout.paths'), + ('hypothesis' , 'my.hypothesis' ), + ('reddit' , 'my.reddit' ), + ('twint' , 'my.twitter.twint' ), + ('twitter' , 'my.twitter.archive' ), + ('lastfm' , 'my.lastfm' ), + ('polar' , 'my.reading.polar' ), + ('instapaper' , 'my.instapaper' ), ] def indent(s, spaces=4): @@ -78,16 +84,39 @@ for cls, p in modules: class google: takeout_path: Paths # path/paths/glob for the takeout zips #+end_src +- [[file:../my/hypothesis.py][my.hypothesis]] + + [[https://hypothes.is][Hypothes.is]] highlights and annotations + + #+begin_src python + class hypothesis: + ''' + Uses [[https://github.com/karlicoss/hypexport][hypexport]] outputs + ''' + + # paths[s]/glob to the exported JSON data + export_path: Paths + + # path to a local clone of hypexport + # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/hypexport + hypexport : Optional[PathIsh] = None + #+end_src - [[file:../my/reddit.py][my.reddit]] Reddit data: saved items/comments/upvotes/etc. - Uses [[https://github.com/karlicoss/rexport][rexport]] output. - #+begin_src python class reddit: - export_path: Paths # path[s]/glob to the exported data - rexport : Optional[PathIsh] = None # path to a local clone of rexport + ''' + Uses [[https://github.com/karlicoss/rexport][rexport]] output. + ''' + + # path[s]/glob to the exported JSON data + export_path: Paths + + # path to a local clone of rexport + # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/rexport + rexport : Optional[PathIsh] = None #+end_src - [[file:../my/twitter/twint.py][my.twitter.twint]] @@ -127,6 +156,23 @@ for cls, p in modules: ''' Polar config is optional, you only need it if you want to specify custom 'polar_dir' ''' - polar_dir: Path = Path('~/.polar').expanduser() + polar_dir: PathIsh = Path('~/.polar').expanduser() + defensive: bool = True # pass False if you want it to fail faster on errors (useful for debugging) + #+end_src +- [[file:../my/instapaper.py][my.instapaper]] + + [[https://www.instapaper.com][Instapaper]] bookmarks, highlights and annotations + + #+begin_src python + class instapaper: + ''' + Uses [[https://github.com/karlicoss/instapexport][instapexport]] outputs. + ''' + # path[s]/glob to the exported JSON data + export_path : Paths + + # path to a local clone of instapexport + # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/instapexport + instapexport: Optional[PathIsh] = None #+end_src :end: diff --git a/doc/SETUP.org b/doc/SETUP.org index d03e2a8..90c0df8 100644 --- a/doc/SETUP.org +++ b/doc/SETUP.org @@ -5,14 +5,14 @@ You'd be really helping me, I want to make the setup as straightforward as possi * Few notes I understand people may not super familiar with Python, PIP or generally unix, so here are some short notes: -- only python3 is supported, and more specifically, ~python >= 3.5~. +- only python3 is supported, and more specifically, ~python >= 3.6~. - I'm using ~pip3~ command, but on your system you might only have ~pip~. If your ~pip --version~ says python 3, feel free to use ~pip~. - similarly, I'm using =python3= in the documentation, but if your =python --version= says python3, it's okay to use =python= -- when you are using ~pip install~, [[https://stackoverflow.com/a/42989020/706389][always pass]] =--user= +- when you are using ~pip install~, [[https://stackoverflow.com/a/42989020/706389][always pass]] =--user=, and *never install third party packages with sudo* (unless you know what you are doing) - throughout the guide I'm assuming the config directory is =~/.config=, but it's different on Mac/Windows. See [[https://github.com/ActiveState/appdirs/blob/3fe6a83776843a46f20c2e5587afcffe05e03b39/appdirs.py#L187-L190][this]] if you're not sure what's your user config dir. diff --git a/my/cfg.py b/my/cfg.py index 9e039b6..3447525 100644 --- a/my/cfg.py +++ b/my/cfg.py @@ -7,10 +7,9 @@ Usage: After that, you can set config attributes: - from types import SimpleNamespace - config.twitter = SimpleNamespace( - export_path='/path/to/twitter/exports', - ) + class user_config: + export_path = '/path/to/twitter/exports' + config.twitter = user_config """ # todo why do we bring this into scope? don't remember.. import my.config as config diff --git a/my/hypothesis.py b/my/hypothesis.py index 94d4edf..738ba6e 100644 --- a/my/hypothesis.py +++ b/my/hypothesis.py @@ -1,26 +1,63 @@ """ [[https://hypothes.is][Hypothes.is]] highlights and annotations """ -from .common import get_files -from .error import Res, sort_res_by +from dataclasses import dataclass +from typing import Optional -import my.config.repos.hypexport.dal as hypexport -from my.config import hypothesis as config +from .core import Paths, PathIsh -### +from my.config import hypothesis as user_config + + +@dataclass +class hypothesis(user_config): + ''' + Uses [[https://github.com/karlicoss/hypexport][hypexport]] outputs + ''' + + # paths[s]/glob to the exported JSON data + export_path: Paths + + # path to a local clone of hypexport + # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/hypexport + hypexport : Optional[PathIsh] = None + + @property + def dal_module(self): + rpath = self.hypexport + if rpath is not None: + from .cfg import set_repo + set_repo('hypexport', rpath) + + import my.config.repos.hypexport.dal as dal + return dal + + +from .core.cfg import make_config +config = make_config(hypothesis) + + +from typing import TYPE_CHECKING +if TYPE_CHECKING: + import my.config.repos.hypexport.dal as dal +else: + dal = config.dal_module + +############################ from typing import List +from .core.error import Res, sort_res_by + +Highlight = dal.Highlight +Page = dal.Page -# TODO weird. not sure why e.g. from dal import Highlight doesn't work.. -Highlight = hypexport.Highlight -Page = hypexport.Page +def _dal() -> dal.DAL: + from .core import get_files + sources = get_files(config.export_path) + return dal.DAL(sources) -# TODO eh. not sure if I should rename everything to dao/DAO or not... -def _dal() -> hypexport.DAL: - sources = get_files(config.export_path, '*.json') - return hypexport.DAL(sources) def highlights() -> List[Res[Highlight]]: @@ -32,12 +69,6 @@ def pages() -> List[Res[Page]]: return sort_res_by(_dal().pages(), key=lambda h: h.created) -# TODO move to side tests? -def test(): - list(pages()) - list(highlights()) - - def _main(): for page in get_pages(): print(page) diff --git a/my/instapaper.py b/my/instapaper.py index 364c402..3ea064d 100644 --- a/my/instapaper.py +++ b/my/instapaper.py @@ -1,18 +1,58 @@ """ -Instapaper bookmarks, highlights and annotations +[[https://www.instapaper.com][Instapaper]] bookmarks, highlights and annotations """ -from .common import get_files +from dataclasses import dataclass +from typing import Optional + +from .core import Paths, PathIsh + +from my.config import instapaper as user_config -from my.config import instapaper as config -import my.config.repos.instapexport.dal as dal +@dataclass +class instapaper(user_config): + ''' + Uses [[https://github.com/karlicoss/instapexport][instapexport]] outputs. + ''' + # path[s]/glob to the exported JSON data + export_path : Paths + # path to a local clone of instapexport + # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/instapexport + instapexport: Optional[PathIsh] = None + + @property + def dal_module(self): + rpath = self.instapexport + if rpath is not None: + from .cfg import set_repo + set_repo('instapexport', rpath) + + import my.config.repos.instapexport.dal as dal + return dal + + +from .core.cfg import make_config +config = make_config(instapaper) + + +from typing import TYPE_CHECKING +if TYPE_CHECKING: + import my.config.repos.instapexport.dal as dal +else: + dal = config.dal_module + +############################ Highlight = dal.Highlight -Bookmark = dal.Bookmark +Bookmark = dal.Bookmark +Page = dal.Page -def inputs(): +from typing import Sequence, Iterable +from pathlib import Path +from .core import get_files +def inputs() -> Sequence[Path]: return get_files(config.export_path) @@ -20,9 +60,8 @@ def _dal() -> dal.DAL: return dal.DAL(inputs()) -def pages(): +def pages() -> Iterable[Page]: return _dal().pages() -get_pages = pages # todo also deprecate.. # TODO dunno, move this to private? @@ -30,3 +69,6 @@ def is_todo(hl: Highlight) -> bool: note = hl.note or '' note = note.lstrip().lower() return note.startswith('todo') + + +get_pages = pages # todo also deprecate.. diff --git a/my/reddit.py b/my/reddit.py index 6fab1df..7e9f908 100755 --- a/my/reddit.py +++ b/my/reddit.py @@ -1,7 +1,5 @@ """ Reddit data: saved items/comments/upvotes/etc. - -Uses [[https://github.com/karlicoss/rexport][rexport]] output. """ from typing import Optional @@ -13,20 +11,26 @@ from dataclasses import dataclass @dataclass class reddit(uconfig): - export_path: Paths # path[s]/glob to the exported data - rexport : Optional[PathIsh] = None # path to a local clone of rexport + ''' + Uses [[https://github.com/karlicoss/rexport][rexport]] output. + ''' + + # path[s]/glob to the exported JSON data + export_path: Paths + + # path to a local clone of rexport + # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/rexport + rexport : Optional[PathIsh] = None @property - def rexport_module(self) -> ModuleType: - # todo return Type[rexport]?? - # todo ModuleIsh? + def dal_module(self) -> ModuleType: rpath = self.rexport if rpath is not None: - from my.cfg import set_repo + from .cfg import set_repo set_repo('rexport', rpath) - import my.config.repos.rexport.dal as m - return m + import my.config.repos.rexport.dal as dal + return dal from .core.cfg import make_config, Attrs @@ -43,15 +47,16 @@ config = make_config(reddit, migration=migration) from typing import TYPE_CHECKING if TYPE_CHECKING: # TODO not sure what is the right way to handle this.. - import my.config.repos.rexport.dal as rexport + import my.config.repos.rexport.dal as dal else: # TODO ugh. this would import too early # but on the other hand we do want to bring the objects into the scope for easier imports, etc. ugh! # ok, fair enough I suppose. It makes sense to configure something before using it. can always figure it out later.. # maybe, the config could dynamically detect change and reimport itself? dunno. - rexport = config.rexport_module + dal = config.dal_module ### +############################ from typing import List, Sequence, Mapping, Iterator from .core.common import mcachew, get_files, LazyLogger, make_dict @@ -70,35 +75,35 @@ def inputs() -> Sequence[Path]: return tuple(res) -Sid = rexport.Sid -Save = rexport.Save -Comment = rexport.Comment -Submission = rexport.Submission -Upvote = rexport.Upvote +Sid = dal.Sid +Save = dal.Save +Comment = dal.Comment +Submission = dal.Submission +Upvote = dal.Upvote -def dal() -> rexport.DAL: - return rexport.DAL(inputs()) +def _dal() -> dal.DAL: + return dal.DAL(inputs()) @mcachew(hashf=lambda: inputs()) def saved() -> Iterator[Save]: - return dal().saved() + return _dal().saved() @mcachew(hashf=lambda: inputs()) def comments() -> Iterator[Comment]: - return dal().comments() + return _dal().comments() @mcachew(hashf=lambda: inputs()) def submissions() -> Iterator[Submission]: - return dal().submissions() + return _dal().submissions() @mcachew(hashf=lambda: inputs()) def upvoted() -> Iterator[Upvote]: - return dal().upvoted() + return _dal().upvoted() ### the rest of the file is some elaborate attempt of restoring favorite/unfavorite times @@ -151,7 +156,7 @@ def _get_state(bfile: Path) -> Dict[Sid, SaveWithDt]: bdt = _get_bdate(bfile) - saves = [SaveWithDt(save, bdt) for save in rexport.DAL([bfile]).saved()] + saves = [SaveWithDt(save, bdt) for save in dal.DAL([bfile]).saved()] return make_dict( sorted(saves, key=lambda p: p.save.created), key=lambda s: s.save.sid, diff --git a/setup.py b/setup.py index e6edf26..4acce99 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ def main(): author_email='karlicoss@gmail.com', description='A Python interface to my life', + python_requires='>=3.6', install_requires=INSTALL_REQUIRES, extras_require={ 'testing': [ diff --git a/tests/config.py b/tests/config.py index 5df0e04..65f6c36 100644 --- a/tests/config.py +++ b/tests/config.py @@ -1,21 +1,18 @@ from pathlib import Path -# TODO switch these from using SimpleNamespace - def setup_notes_path(notes: Path) -> None: # TODO reuse doc from my.cfg? from my.cfg import config - from types import SimpleNamespace - config.orgmode = SimpleNamespace( # type: ignore[misc,assignment] - roots=[notes], - ) + class user_config: + roots = [notes] + config.orgmode = user_config # type: ignore[misc,assignment] # TODO FIXME ugh. this belongs to tz provider or global config or someting import pytz - config.weight = SimpleNamespace( # type: ignore[misc,assignment] + class user_config_2: default_timezone = pytz.timezone('Europe/London') - ) + config.weight = user_config_2 # type: ignore[misc,assignment] def test_dynamic_configuration(notes: Path) -> None: @@ -38,10 +35,9 @@ import pytest # type: ignore def test_set_repo(tmp_path: Path) -> None: from my.cfg import config - from types import SimpleNamespace - config.hypothesis = SimpleNamespace( # type: ignore[misc,assignment] - export_path='whatever', - ) + class user_config: + export_path = 'whatever', + config.hypothesis = user_config # type: ignore[misc,assignment] # precondition: # should fail because can't find hypexport diff --git a/tests/hypothesis.py b/tests/hypothesis.py new file mode 100644 index 0000000..9ef8880 --- /dev/null +++ b/tests/hypothesis.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from my.hypothesis import pages, highlights + +def test(): + assert len(list(pages())) > 10 + assert len(list(highlights())) > 10