documentation: generate tables of content, better navigation

doc/DEVELOPMENT.org

@@ -1,8 +1,23 @@
+* TOC
+:PROPERTIES:
+:TOC:      :include all :depth 3
+:END:
+
+:CONTENTS:
+- [[#toc][TOC]]
+- [[#running-tests][Running tests]]
+- [[#ide-setup][IDE setup]]
+- [[#linting][Linting]]
+- [[#modifyingadding-modules][Modifying/adding modules]]
+:END:
+
 * Running tests
 I'm using =tox= to run test/lint. You can check out [[file:../.github/workflows/main.yml][Github Actions]] config
 and [[file:../scripts/ci/run]] for the up to date info on the specifics.
 
-* IDE setup: make sure my.config is in your package search path
+* IDE setup
+To benefit from type hinting, make sure =my.config= is in your package search path.
+
 In runtime, ~my.config~ is imported from the user config directory dynamically.
 
 However, Pycharm/Emacs/whatever you use won't be able to figure that out, so you'd need to adjust your IDE configuration.
@@ -43,3 +58,5 @@ Now if you add =my_reddit_overlay= *in the front* of ~PYTHONPATH~, all the downs
 This could be useful to monkey patch some behaviours, or dynamically add some extra data sources -- anything that comes to your mind.
 
 I'll put up a better guide on this, in the meantime see [[https://packaging.python.org/guides/packaging-namespace-packages]["namespace packages"]] for more info.
+
+# TODO add example with overriding 'all'

doc/MODULES.org

@@ -1,11 +1,31 @@
 This file is an overview of *documented* modules.
 There are many more, see [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules, I'm progressively working on documenting them.
 
+* TOC
+:PROPERTIES:
+:TOC:      :include all
+:END:
+:CONTENTS:
+- [[#toc][TOC]]
+- [[#intro][Intro]]
+- [[#configs][Configs]]
+  - [[#mygoogletakeoutpaths][my.google.takeout.paths]]
+  - [[#myhypothesis][my.hypothesis]]
+  - [[#myreddit][my.reddit]]
+  - [[#mytwittertwint][my.twitter.twint]]
+  - [[#mytwitterarchive][my.twitter.archive]]
+  - [[#mylastfm][my.lastfm]]
+  - [[#myreadingpolar][my.reading.polar]]
+  - [[#myinstapaper][my.instapaper]]
+:END:
+
+* Intro
+
 See [[file:SETUP.org][SETUP]] to find out how to set up your own config.
 
 Some explanations:
 
-- =MY_CONFIG= is whereever you are keeping your private configuration (usually =~/.config/my/=)
+- =MY_CONFIG= is the path where you are keeping your private configuration (usually =~/.config/my/=)
 - [[https://docs.python.org/3/library/pathlib.html#pathlib.Path][Path]] is a standard Python object to represent paths
 - [[https://github.com/karlicoss/HPI/blob/5f4acfddeeeba18237e8b039c8f62bcaa62a4ac2/my/core/common.py#L9][PathIsh]] is a helper type to allow using either =str=, or a =Path=
 - [[https://github.com/karlicoss/HPI/blob/5f4acfddeeeba18237e8b039c8f62bcaa62a4ac2/my/core/common.py#L108][Paths]] is another helper type for paths.
@@ -21,12 +41,15 @@ Some explanations:
 
 - if the field has a default value, you can omit it from your private config altogether
 
+* Configs
 
 The config snippets below are meant to be modified accordingly and *pasted into your private configuration*, e.g =$MY_CONFIG/my/config.py=.
 
 You don't have to set them up all at once, it's recommended to do it gradually.
 
-#+begin_src python :dir .. :results output drawer :exports result
+# TODO hmm. drawer raw means it can output outlines, but then have to manually erase the generated results. ugh.
+
+#+begin_src python :dir .. :results output drawer raw :exports result
 # TODO ugh, pkgutil.walk_packages doesn't recurse and find packages like my.twitter.archive??
 import importlib
 # from lint import all_modules # meh
@@ -63,7 +86,7 @@ for cls, p in modules:
     for x in ['.py', '__init__.py']:
         if Path(mpath + x).exists():
             mpath = mpath + x
-    print(f'- [[file:../{mpath}][{p}]]')
+    print(f'** [[file:../{mpath}][{p}]]')
     mdoc = m.__doc__
     if mdoc is not None:
         print(indent(mdoc))
@@ -73,10 +96,9 @@ for cls, p in modules:
 #+end_src
 
 #+RESULTS:
-:results:
 
 
-- [[file:../my/google/takeout/paths.py][my.google.takeout.paths]]
+** [[file:../my/google/takeout/paths.py][my.google.takeout.paths]]
 
     Module for locating and accessing [[https://takeout.google.com][Google Takeout]] data
 
@@ -84,7 +106,7 @@ for cls, p in modules:
     class google:
         takeout_path: Paths # path/paths/glob for the takeout zips
     #+end_src
-- [[file:../my/hypothesis.py][my.hypothesis]]
+** [[file:../my/hypothesis.py][my.hypothesis]]
 
     [[https://hypothes.is][Hypothes.is]] highlights and annotations
 
@@ -101,7 +123,7 @@ for cls, p in modules:
         # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/hypexport
         hypexport  : Optional[PathIsh] = None
     #+end_src
-- [[file:../my/reddit.py][my.reddit]]
+** [[file:../my/reddit.py][my.reddit]]
 
     Reddit data: saved items/comments/upvotes/etc.
 
@@ -118,7 +140,7 @@ for cls, p in modules:
         # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/rexport
         rexport    : Optional[PathIsh] = None
     #+end_src
-- [[file:../my/twitter/twint.py][my.twitter.twint]]
+** [[file:../my/twitter/twint.py][my.twitter.twint]]
 
     Twitter data (tweets and favorites).
 
@@ -128,7 +150,7 @@ for cls, p in modules:
     class twint:
         export_path: Paths # path[s]/glob to the twint Sqlite database
     #+end_src
-- [[file:../my/twitter/archive.py][my.twitter.archive]]
+** [[file:../my/twitter/archive.py][my.twitter.archive]]
 
     Twitter data (uses [[https://help.twitter.com/en/managing-your-account/how-to-download-your-twitter-archive][official twitter archive export]])
 
@@ -136,7 +158,7 @@ for cls, p in modules:
     class twitter:
         export_path: Paths # path[s]/glob to the twitter archive takeout
     #+end_src
-- [[file:../my/lastfm][my.lastfm]]
+** [[file:../my/lastfm][my.lastfm]]
 
     Last.fm scrobbles
 
@@ -147,7 +169,7 @@ for cls, p in modules:
         """
         export_path: Paths
     #+end_src
-- [[file:../my/reading/polar.py][my.reading.polar]]
+** [[file:../my/reading/polar.py][my.reading.polar]]
 
     [[https://github.com/burtonator/polar-books][Polar]] articles and highlights
 
@@ -159,7 +181,7 @@ for cls, p in modules:
         polar_dir: PathIsh = Path('~/.polar').expanduser()
         defensive: bool = True # pass False if you want it to fail faster on errors (useful for debugging)
     #+end_src
-- [[file:../my/instapaper.py][my.instapaper]]
+** [[file:../my/instapaper.py][my.instapaper]]
 
     [[https://www.instapaper.com][Instapaper]] bookmarks, highlights and annotations
 
@@ -175,4 +197,3 @@ for cls, p in modules:
         # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/instapexport
         instapexport: Optional[PathIsh] = None
     #+end_src
-:end:

doc/SETUP.org

@@ -2,8 +2,36 @@
 Please don't be shy and raise issues if something in the instructions is unclear.
 You'd be really helping me, I want to make the setup as straightforward as possible!
 
+# update with org-make-toc
+* TOC
+:PROPERTIES:
+:TOC:      :include all
+:END:
+
+:CONTENTS:
+- [[#toc][TOC]]
+- [[#few-notes][Few notes]]
+- [[#setting-up-the-main-package][Setting up the main package]]
+  - [[#option-1-install-from-pip][option 1: install from PIP]]
+  - [[#option-2-local-install][option 2: local install]]
+  - [[#option-3-use-without-installing][option 3: use without installing]]
+- [[#optional-packages][Optional packages]]
+- [[#setting-up-the-modules][Setting up the modules]]
+  - [[#private-configuration-myconfig][private configuration (my.config)]]
+  - [[#module-dependencies][module dependencies]]
+- [[#usage-examples][Usage examples]]
+  - [[#end-to-end-roam-research-setup][End-to-end Roam Research setup]]
+  - [[#polar][Polar]]
+  - [[#google-takeout][Google Takeout]]
+  - [[#kobo-reader][Kobo reader]]
+  - [[#orger][Orger]]
+    - [[#orger--polar][Orger + Polar]]
+  - [[#demopy][demo.py]]
+:END:
+
+
 * Few notes
-I understand people may not super familiar with Python, PIP or generally unix, so here are some short notes:
+I understand people may not super familiar with Python, PIP or generally unix, so here are some useful notes:
 
 - only python3 is supported, and more specifically, ~python >= 3.6~.
 - I'm using ~pip3~ command, but on your system you might only have ~pip~.
@@ -13,7 +41,7 @@ I understand people may not super familiar with Python, PIP or generally unix, s
 - similarly, I'm using =python3= in the documentation, but if your =python --version= says python3, it's okay to use =python=
 
 - when you are using ~pip install~, [[https://stackoverflow.com/a/42989020/706389][always pass]] =--user=, and *never install third party packages with sudo* (unless you know what you are doing)
-- throughout the guide I'm assuming the config directory is =~/.config=, but it's different on Mac/Windows.
+- throughout the guide I'm assuming the user config directory is =~/.config=, but it's *different on Mac/Windows*.
 
   See [[https://github.com/ActiveState/appdirs/blob/3fe6a83776843a46f20c2e5587afcffe05e03b39/appdirs.py#L187-L190][this]] if you're not sure what's your user config dir.
 
@@ -22,12 +50,12 @@ This is a *required step*
 
 You can choose one of the following options:
 
-** install from [[https://pypi.org/project/HPI][PIP]]
+** option 1: install from [[https://pypi.org/project/HPI][PIP]]
-This is the easiest way:
+This is the *easiest way*:
 
 : pip3 install --user HPI
 
-** local install
+** option 2: local install
 This is convenient if you're planning to add new modules or change the existing ones.
 
 1. Clone the repository: =git clone git@github.com:karlicoss/HPI.git /path/to/hpi=
@@ -39,7 +67,7 @@ This is convenient if you're planning to add new modules or change the existing
 
    It's *extremely* convenient for developing and debugging.
   
-** use without installing
+** option 3: use without installing
 This is less convenient, but gives you more control.
 
 1. Clone the repository: =git clone git@github.com:karlicoss/HPI.git /path/to/hpi=
@@ -59,7 +87,7 @@ This is less convenient, but gives you more control.
 
 The benefit of this way is that you get a bit more control, explicitly allowing your scripts to use your data.
 
-** optional packages
+* Optional packages
 You can also install some opional packages
 
 : pip3 install 'HPI[optional]'
@@ -69,12 +97,14 @@ They aren't necessary, but improve your experience. At the moment these are:
 - [[https://github.com/karlicoss/cachew][cachew]]: automatic caching library, which can greatly speedup data access
 - [[https://github.com/metachris/logzero][logzero]]: a nice logging library, supporting colors
 
-* Setting up the modules
+* Setting up modules
-This is an *optional step* as some modules might work without extra setup.
+This is an *optional step* as few modules work without extra setup.
 But it depends on the specific module.
 
+See [[file:MODULES.org][MODULES]] to read documentation on specific modules that interest you.
+
 You might also find interesting to read [[file:CONFIGURING.org][CONFIGURING]], where I'm
-elaborating on some rationales behind the current configuration system.
+elaborating on some technical rationales behind the current configuration system.
 
 ** private configuration (=my.config=)
 # TODO write about dynamic configuration
@@ -87,7 +117,7 @@ The config is simply a *python package* (named =my.config=), expected to be in =
 
 Since it's a Python package, generally it's very *flexible* and there are many ways to set it up.
 
-- The simplest and very minimum you need is =~/.config/my/my/config.py=. For example:
+- *The simplest and the very minimum* you need is =~/.config/my/my/config.py=. For example:
 
   #+begin_src python
   import pytz # yes, you can use any Python stuff in the config
@@ -116,32 +146,6 @@ Since it's a Python package, generally it's very *flexible* and there are many w
 
   - or you can just try running them and fill in the attributes Python complains about!
 
-- My config layout is a bit more complicated:
-
-  #+begin_src python :exports results :results output
-  from pathlib import Path
-  home = Path("~").expanduser()
-  pp = home / '.config/my/my/config'
-  for p in sorted(pp.rglob('*')):
-    if '__pycache__' in p.parts:
-      continue
-    ps = str(p).replace(str(home), '~')
-    print(ps)
-  #+end_src
-
-  #+RESULTS:
-  #+begin_example
-  ~/.config/my/my/config/__init__.py
-  ~/.config/my/my/config/locations.py
-  ~/.config/my/my/config/repos
-  ~/.config/my/my/config/repos/endoexport
-  ~/.config/my/my/config/repos/fbmessengerexport
-  ~/.config/my/my/config/repos/kobuddy
-  ~/.config/my/my/config/repos/monzoexport
-  ~/.config/my/my/config/repos/pockexport
-  ~/.config/my/my/config/repos/rexport
-  #+end_example
-
 - Another example is in [[file:example_config][example_config]]:
 
   #+begin_src bash :exports results :results output
@@ -183,6 +187,32 @@ Feel free to add other files as well though to organize better, it's a real Pyth
 Some things (e.g. links to external packages like [[https://github.com/karlicoss/hypexport][hypexport]]) are specified as *ordinary symlinks* in ~repos~ directory.
 That way you get easy imports (e.g. =import my.config.repos.hypexport.model=) and proper IDE integration.
 
+- my own config layout is a bit more complicated:
+
+  #+begin_src python :exports results :results output
+  from pathlib import Path
+  home = Path("~").expanduser()
+  pp = home / '.config/my/my/config'
+  for p in sorted(pp.rglob('*')):
+    if '__pycache__' in p.parts:
+      continue
+    ps = str(p).replace(str(home), '~')
+    print(ps)
+  #+end_src
+
+  #+RESULTS:
+  #+begin_example
+  ~/.config/my/my/config/__init__.py
+  ~/.config/my/my/config/locations.py
+  ~/.config/my/my/config/repos
+  ~/.config/my/my/config/repos/endoexport
+  ~/.config/my/my/config/repos/fbmessengerexport
+  ~/.config/my/my/config/repos/kobuddy
+  ~/.config/my/my/config/repos/monzoexport
+  ~/.config/my/my/config/repos/pockexport
+  ~/.config/my/my/config/repos/rexport
+  #+end_example
+
 # TODO link to post about exports?
 ** module dependencies
 Dependencies are different for specific modules you're planning to use, so it's hard to specify.

my/core/init.py

@@ -41,7 +41,10 @@ def setup_config() -> None:
         mycfg_dir = Path(appdirs.user_config_dir('my'))
 
     if not mycfg_dir.exists():
-        warnings.warn(f"my.config package isn't found! (expected at {mycfg_dir}). This is likely to result in issues.")
+        warnings.warn(f"""
+'my.config' package isn't found! (expected at {mycfg_dir}). This is likely to result in issues.
+See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-modules for more info.
+""".strip())
         return
 
     mpath = str(mycfg_dir)
@@ -58,10 +61,12 @@ def setup_config() -> None:
         import my.config
     except ImportError as ex:
         # just in case... who knows what crazy setup users have in mind.
-        warnings.warn(f"Importing my.config failed! (error: {ex}). This is likely to result in issues.")
+        # todo log?
+        warnings.warn(f"""
+Importing 'my.config' failed! (error: {ex}). This is likely to result in issues.
+See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-modules for more info.
+""")
 
 
 setup_config()
 del setup_config
-
-# TODO move to my.core?

my/hypothesis.py

@@ -19,7 +19,7 @@ class hypothesis(user_config):
     export_path: Paths
 
     # path to a local clone of hypexport
-    # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/hypexport
+    # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/hypexport
     hypexport  : Optional[PathIsh] = None
 
     @property

my/instapaper.py

@@ -18,7 +18,7 @@ class instapaper(user_config):
     export_path : Paths
 
     # path to a local clone of instapexport
-    # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/instapexport
+    # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/instapexport
     instapexport: Optional[PathIsh] = None
 
     @property

my/reddit.py

@@ -19,7 +19,7 @@ class reddit(uconfig):
     export_path: Paths
 
     # path to a local clone of rexport
-    # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/repos/rexport
+    # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/rexport
     rexport    : Optional[PathIsh] = None
 
     @property