add demo; proper readme; remove kython dependencies

README.md

@@ -1,17 +1,39 @@
-Python interface into my life
+Python interface into my life.
 
+This package deals with abstracting away various data sources and providing nice Python interface for them, also lets you define covenience functions.
+
+This might not necessarily be convenient for you to use, perhaps it's more of a concept of how you can organize and access your personal data.
+But it works for me so hopefully that would help you if you're struggling!
 
 # Setting up
+First you need to tell the package where to look for your data and external repositories, which is done though a python file named `my_configuration.py`, e.g.:
+```
+class paths:
+    class stexport:
+        repo       = /path/repos/stackexchange_export_repo
+        export_dir = /path/to/backups/stackexchange
+
+    class ghexport:
+        repo       = /path/repos/github_export_repo
+        export_dir = /path/to/backups/github
+```
+
+and pass the filename to the package:
+
 ```
 cp with_my.example with_my
 
-# edit path to your private configuration now:
+# specify path to your my_configuration.py:
 vim with_my
 ```
-TODO add instructions to try it?
 
 
 # Usage example
+If you run your script with `with_my` wrapper, you'd have `my` in `PYTHONPATH` which gives you access to your data from within the script.
+
+
 ```
 with_my python3 -c 'import my.books.kobo as kobo; print(kobo.get_todos())' 
 ```
+
+Also read/run [demo.py](demo.py) for a full demonstration of setting up Hypothesis.

demo.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+from subprocess import check_call, DEVNULL
+from shutil import copy, copytree
+import os
+import tempfile
+from pathlib import Path
+
+my_repo = Path(__file__).absolute().parent
+
+
+def run():
+    # clone git@github.com:karlicoss/my.git
+    copytree(my_repo, 'my_repo')
+
+    # prepare repositories you'd be using. For this demo we only set up Hypothesis
+    hypothesis_repo = os.path.abspath('hypothesis_repo')
+    check_call(['git', 'clone', 'https://github.com/karlicoss/hypexport.git', hypothesis_repo])
+    #
+
+
+    # prepare some demo Hypothesis data
+    hypothesis_backups = os.path.abspath('backups/hypothesis')
+    Path(hypothesis_backups).mkdir(exist_ok=True, parents=True)
+    check_call([
+        'curl',
+        'https://raw.githubusercontent.com/taniki/netrights-dashboard-mockup/master/_data/annotations.json',
+        '-o', hypothesis_backups + '/annotations.json',
+    ], stderr=DEVNULL)
+    #
+
+
+    # create private configuration and set necessary paths
+    with_my = 'my_repo/with_my'
+    copy('my_repo/with_my.example', with_my)
+
+    private_config = os.path.abspath('my_configuration.py')
+    Path(private_config).write_text("""
+
+class paths:
+    class hypexport:
+        repo       = '{hypothesis_repo}'
+        export_dir = '{hypothesis_backups}'
+
+""".format(**locals()))
+    #
+
+    # edit the config and set path to private configuration
+    my = Path(with_my).read_text().replace('MY_CONFIGURATION_PATH=', 'MY_CONFIGURATION_PATH=' + private_config)
+    Path(with_my).write_text(my)
+    #
+
+
+    # now we can use it!
+
+    check_call(['my_repo/with_my', 'python3', '-c', '''
+import my.hypothesis
+
+from pprint import pprint
+
+for page in my.hypothesis.get_pages()[:8]:
+    print('URL:   ' + page.link)
+    print('Title: ' + page.title)
+    print('{} annotations'.format(len(page.annotations)))
+    print()
+'''])
+
+# that should result in something like this:
+
+# URL:   https://tacticaltech.org/
+# Title: Tactical Technology Collective
+# 1 annotations
+#
+# URL:   https://web.hypothes.is/blog/annotating-the-wild-west-of-information-flow/
+# Title: Annotating the wild west of information flow – Hypothesis
+# 1 annotations
+#
+# URL:   http://www.liberation.fr/futurs/2016/12/12/megafichier-beauvau-prie-de-revoir-sa-copie_1534720
+# Title: «Mégafichier» : Beauvau prié de revoir sa copie
+# 3 annotations
+#
+# URL:   https://www.wired.com/2016/12/7500-faceless-coders-paid-bitcoin-built-hedge-funds-brain/
+# Title: 7,500 Faceless Coders Paid in Bitcoin Built a Hedge Fund’s Brain
+# 4 annotations
+#
+# URL:   http://realscreen.com/2016/12/06/project-x-tough-among-sundance-17-doc-shorts/
+# Title: “Project X,” “Tough” among Sundance ’17 doc shorts
+# 1 annotations
+#
+# URL:   https://grehack.fr/2016/program
+# Title: GreHack | Security conference and hacking game 2016 | Grenoble
+# 1 annotations
+#
+# URL:   https://respectmynet.eu/
+# Title: [!] Respect My Net
+# 1 annotations
+#
+# URL:   https://www.youtube.com/watch?v=Xgp7BIBtPhk
+# Title: BBC Documentaries 2016: The Joy of Data [FULL BBC SCIENCE DOCUMENTARY]
+# 1 annotations
+
+
+
+def main():
+    with tempfile.TemporaryDirectory() as tdir:
+        os.chdir(tdir)
+        run()
+
+
+if __name__ == '__main__':
+    main()

my/books/kobo.py

@@ -4,7 +4,7 @@ from .. import paths
 
 @lru_cache()
 def kobuddy_module():
-    from kython import import_from
+    from ..common import import_from
     return import_from(paths.kobuddy.repo, 'kobuddy')
 
 kobuddy = kobuddy_module()

my/coding/github/__init__.py

@@ -4,7 +4,7 @@ from ... import paths
 
 @lru_cache()
 def ghexport():
-    from kython import import_file
+    from ...common import import_file
     return import_file(paths.ghexport.repo / 'model.py')
 
 

my/common.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+import functools
+from typing import Union, Callable, Dict, List, Iterable, TypeVar
+
+# some helper functions
+
+def import_file(p: Union[str, Path], name=None):
+    p = Path(p)
+    if name is None:
+        name = p.stem
+    import importlib.util
+    spec = importlib.util.spec_from_file_location(name, p) # type: ignore
+    foo = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(foo) # type: ignore
+    return foo
+
+def import_from(path, name):
+    path = str(path)
+    import sys
+    try:
+        sys.path.append(path)
+        import importlib
+        return importlib.import_module(name)
+    finally:
+        sys.path.remove(path)
+
+
+T = TypeVar('T')
+K = TypeVar('K')
+V = TypeVar('V')
+
+def the(l: Iterable[T]) -> T:
+    it = iter(l)
+    try:
+        first = next(it)
+    except StopIteration as ee:
+        raise RuntimeError('Empty iterator?')
+    assert all(e == first for e in it)
+    return first
+
+
+def group_by_key(l: Iterable[T], key: Callable[[T], K]) -> Dict[K, List[T]]:
+    res: Dict[K, List[T]] = {}
+    for i in l:
+        kk = key(i)
+        lst = res.get(kk, [])
+        lst.append(i)
+        res[kk] = lst
+    return res
+
+
+Cl = TypeVar('Cl')
+R = TypeVar('R')
+
+def cproperty(f: Callable[[Cl], R]) -> R:
+    return property(functools.lru_cache(maxsize=1)(f)) # type: ignore

my/hypothesis.py

@@ -1,26 +1,27 @@
 from functools import lru_cache
+from pathlib import Path
 
 from . import paths
 
 @lru_cache()
 def hypexport():
-    from kython import import_file
+    from .common import import_file
-    return import_file(paths.hypexport.repo / 'model.py')
+    return import_file(Path(paths.hypexport.repo) / 'model.py')
 
 Annotation = hypexport().Annotation
 
 def get_model():
-    sources = list(sorted(paths.hypexport.export_dir.glob('*.json')))
+    export_dir = Path(paths.hypexport.export_dir)
+    sources = list(sorted(export_dir.glob('*.json')))
     model = hypexport().Model(sources)
     return model
 
 
-from kython import listdir_abs
 from typing import Dict, List, NamedTuple, Optional, Sequence
 from pathlib import Path
 from datetime import datetime
 
-from kython import group_by_key, the, cproperty
+from .common import group_by_key, the, cproperty
 
 
 class Page(NamedTuple):

my/reading/goodreads.py

@@ -8,7 +8,7 @@ from .. import paths
 
 @lru_cache()
 def goodrexport():
-    from kython import import_file
+    from ..common import import_file
     return import_file(paths.goodrexport.repo / 'model.py')
 
 

my/stackexchange.py

@@ -4,7 +4,7 @@ from . import paths
 
 @lru_cache()
 def stexport():
-    from kython import import_file
+    from .common import import_file
     stexport_model = import_file(paths.stexport.repo / 'model.py')
     return stexport_model