add proper my_configuration demo and use it in readme

.circleci/config.yml

@@ -8,11 +8,10 @@ jobs:
     steps:
       - checkout
 
-      # TODO FIXME switch to tox?
-      - run: pip3 install --user mypy pylint
-
       - run: ./demo.py
 
+      # TODO FIXME switch to tox?
+      # - run: pip3 install --user mypy pylint
       # - run: python3 -m mypy --namespace-packages my
       # - run: python3 -m pylint -E my
 

README.org

@@ -8,61 +8,66 @@ This might not necessarily be convenient for you to use, perhaps it's more of a
 But it works for me so hopefully that would help you if you're struggling!
 
 * Setting up
+** =my_configuration= package for private paths/repositores
 First you need to tell the package where to look for your data and external repositories, which is done though a separate (private) package named ~my_configuration~.
 
-In the simplest case it can be just a single file named ~my_configuration.py~, e.g.:
-# TODO eh, perhaps 
-# TODO prepare private my_configuration package???
+You can see example in ~my_configuration_template~. You can copy it somewhere else and modify to your needs.
 
-#+begin_src python
-class paths:
-    class stexport:
-        repo       = /path/repos/stackexchange_export_repo
-        export_dir = /path/to/backups/stackexchange
+Some explanations:
 
-    class ghexport:
-        repo       = /path/repos/github_export_repo
-        export_dir = /path/to/backups/github
-#+end_src
-
-
-
-
-#+begin_src bash :results drawer
-find my_configuration/ | grep -v -E 'mypy_cache|.git|__pycache__'
+#+begin_src bash :exports results :results output
+  for x in $(find my_configuration_template/ | grep -v -E 'mypy_cache|.git|__pycache__'); do
+    if   [[ -L "$x" ]]; then
+      echo "l $x -> $(readlink $x)"
+    elif [[ -d "$x" ]]; then
+      echo "d $x"
+    else
+      echo "f $x"
+      (echo "---"; cat "$x"; echo "---" ) | sed 's/^/  /'
+    fi
+  done
 #+end_src
 
 #+RESULTS:
-:results:
-my_configuration/
-my_configuration/my_configuration
-my_configuration/my_configuration/__init__.py
-my_configuration/my_configuration/repos
-my_configuration/my_configuration/repos/kobuddy
-my_configuration/my_configuration/repos/rexport
-my_configuration/my_configuration/repos/hypexport
-my_configuration/my_configuration/pdf_configuration.py
-:end:
+#+begin_example
+d my_configuration_template/
+d my_configuration_template/my_configuration
+f my_configuration_template/my_configuration/__init__.py
+  ---
+  class paths:
+      """
+      Feel free to remore hypexport if you don't need it/add your own custom settings and use them
+      """
+      class hypexport:
+          export_dir = '/tmp/my_demo/backups/hypothesis'
+  ---
+d my_configuration_template/my_configuration/repos
+l my_configuration_template/my_configuration/repos/hypexport -> /tmp/my_demo/hypothesis_repo
+#+end_example
 
-and pass the filename to the package:
-# TODO FIXME not filename?
+As you can see, generally you specify fixed paths (e.g. to backup directory) in ~__init__.py~.
+Feel free to add other files as well though to organize better, it's a real python package after all!
+
+Some things (e.g. links to external packages like [[https://github.com/karlicoss/hypexport][hypexport]]) are specified as normal symlinks in ~repos~ directory.
+That way you get easy imports (e.g. =import my_configuration.repos.hypexport.model=) and proper IDE integration.
+
+# TODO link to post about exports?
+** =with_my= helper script
+Next, point =with_my= script to your private configuration:
+   
 
 #+begin_src bash
 cp with_my.example with_my
-
-# specify path to your my_configuration:
-vim with_my
+vim with_my # specify path to your my_configuration:
 #+end_src
 
 ** Dependencies
-They depend on specific modules you're planning to use, so it's hard to specify.
+Dependencies are different for specific modules you're planning to use, so it's hard to specify.
 Generally you can just try and then install missing packages via ~pip install --user~, should be fairly straighforward.
 
-
 * Usage example
 If you run your script with ~with_my~ wrapper, you'd have ~my~ in ~PYTHONPATH~ which gives you access to your data from within the script.
 
-
 #+begin_src bash
   with_my python3 -c 'import my.books.kobo as kobo; print(kobo.get_todos())' 
 #+end_src
@@ -83,3 +88,6 @@ or, set up as ~mypy.ini~ file:
 [mypy]
 mypy_path=/path/to/my_configuration_dir
 #+end_src
+
+
+# TODO hmm, if package isn't using my_configuration then we don't really need it?

demo.py

@@ -2,24 +2,26 @@
 from subprocess import check_call, DEVNULL
 from shutil import copy, copytree
 import os
-import tempfile
+from os.path import abspath
 from pathlib import Path
 
 my_repo = Path(__file__).absolute().parent
 
 
 def run():
-    # clone git@github.com:karlicoss/my.git
-    copytree(my_repo, 'my_repo')
+    # uses fixed paths; worth it for the sake of demonstration
+    # assumes we're in /tmp/my_demo now
 
-    # prepare repositories you'd be using. For this demo we only set up Hypothesis
-    hypothesis_repo = os.path.abspath('hypothesis_repo')
+    # 1. clone git@github.com:karlicoss/my.git
+    copytree(my_repo, 'my_repo', symlinks=True)
+
+    # 2. prepare repositories you'd be using. For this demo we only set up Hypothesis
+    hypothesis_repo = abspath('hypothesis_repo')
     check_call(['git', 'clone', 'https://github.com/karlicoss/hypexport.git', hypothesis_repo])
     #
 
-
-    # prepare some demo Hypothesis data
-    hypothesis_backups = os.path.abspath('backups/hypothesis')
+    # 3. prepare some demo Hypothesis data
+    hypothesis_backups = abspath('backups/hypothesis')
     Path(hypothesis_backups).mkdir(exist_ok=True, parents=True)
     check_call([
         'curl',
@@ -29,36 +31,17 @@ def run():
     #
 
 
-    # create private configuration and set necessary paths
+    # 4. create private with_my file and set path to private configuration
     with_my = 'my_repo/with_my'
     copy('my_repo/with_my.example', with_my)
 
-    private_config = Path('my_configuration').absolute()
-    private_config.mkdir()
-
-    my_configuration = private_config / 'my_configuration'
-    my_configuration.mkdir()
-
-    repos = my_configuration / 'repos'
-    repos.mkdir()
-    (repos / 'hypexport').symlink_to(hypothesis_repo)
-
-    Path(my_configuration / '__init__.py').write_text("""
-
-class paths:
-    class hypexport:
-        export_dir = '{hypothesis_backups}'
-
-""".format(**locals()))
-    #
-
+    my_configuration_root = abspath('my_repo/my_configuration_template')
     # edit the config and set path to private configuration
-    my = Path(with_my).read_text().replace('MY_CONFIGURATION_DIR=', 'MY_CONFIGURATION_DIR=' + str(private_config))
+    my = Path(with_my).read_text().replace('MY_CONFIGURATION_DIR=', 'MY_CONFIGURATION_DIR=' + str(my_configuration_root))
     Path(with_my).write_text(my)
     #
 
-
-    # now we can use it!
+    # 5. now we can use it!
 
     check_call(['my_repo/with_my', 'python3', '-c', '''
 import my.hypothesis
@@ -106,10 +89,23 @@ for page in my.hypothesis.get_pages()[:8]:
 # Title: BBC Documentaries 2016: The Joy of Data [FULL BBC SCIENCE DOCUMENTARY]
 # 1 annotations
 
+from contextlib import contextmanager
+@contextmanager
+def named_temp_dir(name: str):
+    """
+    Fixed name tmp dir
+    """
+    td = (Path('/tmp') / name)
+    try:
+        td.mkdir(exist_ok=False)
+        yield td
+    finally:
+        import shutil
+        shutil.rmtree(str(td))
 
 
 def main():
-    with tempfile.TemporaryDirectory() as tdir:
+    with named_temp_dir('my_demo') as tdir:
         os.chdir(tdir)
         run()
 

my_configuration_template/my_configuration/__init__.py

@@ -0,0 +1,6 @@
+class paths:
+    """
+    Feel free to remore hypexport if you don't need it/add your own custom settings and use them
+    """
+    class hypexport:
+        export_dir = '/tmp/my_demo/backups/hypothesis'

my_configuration_template/my_configuration/repos/hypexport

@@ -0,0 +1 @@
+/tmp/my_demo/hypothesis_repo