diff --git a/README.md b/README.md index 20dd965..20941a8 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,39 @@ -Python interface into my life +Python interface into my life. +This package deals with abstracting away various data sources and providing nice Python interface for them, also lets you define covenience functions. + +This might not necessarily be convenient for you to use, perhaps it's more of a concept of how you can organize and access your personal data. +But it works for me so hopefully that would help you if you're struggling! # Setting up +First you need to tell the package where to look for your data and external repositories, which is done though a python file named `my_configuration.py`, e.g.: +``` +class paths: + class stexport: + repo = /path/repos/stackexchange_export_repo + export_dir = /path/to/backups/stackexchange + + class ghexport: + repo = /path/repos/github_export_repo + export_dir = /path/to/backups/github +``` + +and pass the filename to the package: + ``` cp with_my.example with_my -# edit path to your private configuration now: +# specify path to your my_configuration.py: vim with_my ``` -TODO add instructions to try it? # Usage example +If you run your script with `with_my` wrapper, you'd have `my` in `PYTHONPATH` which gives you access to your data from within the script. + + ``` with_my python3 -c 'import my.books.kobo as kobo; print(kobo.get_todos())' ``` + +Also read/run [demo.py](demo.py) for a full demonstration of setting up Hypothesis. diff --git a/demo.py b/demo.py new file mode 100755 index 0000000..70df862 --- /dev/null +++ b/demo.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +from subprocess import check_call, DEVNULL +from shutil import copy, copytree +import os +import tempfile +from pathlib import Path + +my_repo = Path(__file__).absolute().parent + + +def run(): + # clone git@github.com:karlicoss/my.git + copytree(my_repo, 'my_repo') + + # prepare repositories you'd be using. For this demo we only set up Hypothesis + hypothesis_repo = os.path.abspath('hypothesis_repo') + check_call(['git', 'clone', 'https://github.com/karlicoss/hypexport.git', hypothesis_repo]) + # + + + # prepare some demo Hypothesis data + hypothesis_backups = os.path.abspath('backups/hypothesis') + Path(hypothesis_backups).mkdir(exist_ok=True, parents=True) + check_call([ + 'curl', + 'https://raw.githubusercontent.com/taniki/netrights-dashboard-mockup/master/_data/annotations.json', + '-o', hypothesis_backups + '/annotations.json', + ], stderr=DEVNULL) + # + + + # create private configuration and set necessary paths + with_my = 'my_repo/with_my' + copy('my_repo/with_my.example', with_my) + + private_config = os.path.abspath('my_configuration.py') + Path(private_config).write_text(""" + +class paths: + class hypexport: + repo = '{hypothesis_repo}' + export_dir = '{hypothesis_backups}' + +""".format(**locals())) + # + + # edit the config and set path to private configuration + my = Path(with_my).read_text().replace('MY_CONFIGURATION_PATH=', 'MY_CONFIGURATION_PATH=' + private_config) + Path(with_my).write_text(my) + # + + + # now we can use it! + + check_call(['my_repo/with_my', 'python3', '-c', ''' +import my.hypothesis + +from pprint import pprint + +for page in my.hypothesis.get_pages()[:8]: + print('URL: ' + page.link) + print('Title: ' + page.title) + print('{} annotations'.format(len(page.annotations))) + print() +''']) + +# that should result in something like this: + +# URL: https://tacticaltech.org/ +# Title: Tactical Technology Collective +# 1 annotations +# +# URL: https://web.hypothes.is/blog/annotating-the-wild-west-of-information-flow/ +# Title: Annotating the wild west of information flow – Hypothesis +# 1 annotations +# +# URL: http://www.liberation.fr/futurs/2016/12/12/megafichier-beauvau-prie-de-revoir-sa-copie_1534720 +# Title: «Mégafichier» : Beauvau prié de revoir sa copie +# 3 annotations +# +# URL: https://www.wired.com/2016/12/7500-faceless-coders-paid-bitcoin-built-hedge-funds-brain/ +# Title: 7,500 Faceless Coders Paid in Bitcoin Built a Hedge Fund’s Brain +# 4 annotations +# +# URL: http://realscreen.com/2016/12/06/project-x-tough-among-sundance-17-doc-shorts/ +# Title: “Project X,” “Tough” among Sundance ’17 doc shorts +# 1 annotations +# +# URL: https://grehack.fr/2016/program +# Title: GreHack | Security conference and hacking game 2016 | Grenoble +# 1 annotations +# +# URL: https://respectmynet.eu/ +# Title: [!] Respect My Net +# 1 annotations +# +# URL: https://www.youtube.com/watch?v=Xgp7BIBtPhk +# Title: BBC Documentaries 2016: The Joy of Data [FULL BBC SCIENCE DOCUMENTARY] +# 1 annotations + + + +def main(): + with tempfile.TemporaryDirectory() as tdir: + os.chdir(tdir) + run() + + +if __name__ == '__main__': + main() diff --git a/my/books/kobo.py b/my/books/kobo.py index 006828f..5edc338 100644 --- a/my/books/kobo.py +++ b/my/books/kobo.py @@ -4,7 +4,7 @@ from .. import paths @lru_cache() def kobuddy_module(): - from kython import import_from + from ..common import import_from return import_from(paths.kobuddy.repo, 'kobuddy') kobuddy = kobuddy_module() diff --git a/my/coding/github/__init__.py b/my/coding/github/__init__.py index 320866f..aa58211 100644 --- a/my/coding/github/__init__.py +++ b/my/coding/github/__init__.py @@ -4,7 +4,7 @@ from ... import paths @lru_cache() def ghexport(): - from kython import import_file + from ...common import import_file return import_file(paths.ghexport.repo / 'model.py') diff --git a/my/common.py b/my/common.py new file mode 100644 index 0000000..0113563 --- /dev/null +++ b/my/common.py @@ -0,0 +1,56 @@ +from pathlib import Path +import functools +from typing import Union, Callable, Dict, List, Iterable, TypeVar + +# some helper functions + +def import_file(p: Union[str, Path], name=None): + p = Path(p) + if name is None: + name = p.stem + import importlib.util + spec = importlib.util.spec_from_file_location(name, p) # type: ignore + foo = importlib.util.module_from_spec(spec) + spec.loader.exec_module(foo) # type: ignore + return foo + +def import_from(path, name): + path = str(path) + import sys + try: + sys.path.append(path) + import importlib + return importlib.import_module(name) + finally: + sys.path.remove(path) + + +T = TypeVar('T') +K = TypeVar('K') +V = TypeVar('V') + +def the(l: Iterable[T]) -> T: + it = iter(l) + try: + first = next(it) + except StopIteration as ee: + raise RuntimeError('Empty iterator?') + assert all(e == first for e in it) + return first + + +def group_by_key(l: Iterable[T], key: Callable[[T], K]) -> Dict[K, List[T]]: + res: Dict[K, List[T]] = {} + for i in l: + kk = key(i) + lst = res.get(kk, []) + lst.append(i) + res[kk] = lst + return res + + +Cl = TypeVar('Cl') +R = TypeVar('R') + +def cproperty(f: Callable[[Cl], R]) -> R: + return property(functools.lru_cache(maxsize=1)(f)) # type: ignore diff --git a/my/hypothesis.py b/my/hypothesis.py index 308677e..ee15277 100644 --- a/my/hypothesis.py +++ b/my/hypothesis.py @@ -1,26 +1,27 @@ from functools import lru_cache +from pathlib import Path from . import paths @lru_cache() def hypexport(): - from kython import import_file - return import_file(paths.hypexport.repo / 'model.py') + from .common import import_file + return import_file(Path(paths.hypexport.repo) / 'model.py') Annotation = hypexport().Annotation def get_model(): - sources = list(sorted(paths.hypexport.export_dir.glob('*.json'))) + export_dir = Path(paths.hypexport.export_dir) + sources = list(sorted(export_dir.glob('*.json'))) model = hypexport().Model(sources) return model -from kython import listdir_abs from typing import Dict, List, NamedTuple, Optional, Sequence from pathlib import Path from datetime import datetime -from kython import group_by_key, the, cproperty +from .common import group_by_key, the, cproperty class Page(NamedTuple): diff --git a/my/reading/goodreads.py b/my/reading/goodreads.py index 087cb9f..c0297f2 100755 --- a/my/reading/goodreads.py +++ b/my/reading/goodreads.py @@ -8,7 +8,7 @@ from .. import paths @lru_cache() def goodrexport(): - from kython import import_file + from ..common import import_file return import_file(paths.goodrexport.repo / 'model.py') diff --git a/my/stackexchange.py b/my/stackexchange.py index 9a606bd..c68fc1a 100644 --- a/my/stackexchange.py +++ b/my/stackexchange.py @@ -4,7 +4,7 @@ from . import paths @lru_cache() def stexport(): - from kython import import_file + from .common import import_file stexport_model = import_file(paths.stexport.repo / 'model.py') return stexport_model