start autogenerating documentation on modules

doc/MODULES.org

@@ -0,0 +1,57 @@
+# TODO explain Paths/PathIsh
+
+
+#+begin_src python :dir .. :results output :exports result
+# TODO ugh, pkgutil.walk_packages doesn't recurse and find packages like my.twitter.archive??
+import importlib
+# from lint import all_modules # meh
+# TODO figure out how to discover configs automatically...
+modules = [
+    ('google' , 'my.google.takeout.paths'),
+    ('reddit' , 'my.reddit'              ),
+    ('twint'  , 'my.twitter.twint'       ),
+    ('twitter', 'my.twitter.archive'     ),
+]
+
+# TODO generate links?
+
+import inspect
+from dataclasses import fields
+# print(',#+begin_src python')
+for cls, p in modules:
+    m = importlib.import_module(p)
+    C = getattr(m, cls)
+    src = inspect.getsource(C)
+    i = src.find('@property')
+    if i != -1:
+        src = src[:i]
+    print(src)
+# print('#+end_src')
+#+end_src
+
+#+RESULTS:
+#+begin_example
+class google(user_config):
+    '''
+    Expects [[https://takeout.google.com][Google Takeout]] data.
+    '''
+    takeout_path: Paths # path/paths/glob for the takeout zips
+
+class reddit(uconfig):
+    '''
+    Reddit module uses [[https://github.com/karlicoss/rexport][rexport]] output
+    '''
+    export_path: Paths                     # path[s]/glob to the exported data
+    rexport    : Optional[PathIsh] = None  # path to a local clone of rexport
+
+
+class twint(user_config):
+    '''
+    Uses [[https://github.com/twintproject/twint][Twint]] data export.
+    '''
+    export_path: Paths # path[s]/glob to twint Sqlite database
+
+class twitter(user_config):
+    export_path: Paths # path[s]/glob to the twitter archive takeout
+
+#+end_example

doc/SETUP.org

@@ -73,6 +73,9 @@ They aren't necessary, but improve your experience. At the moment these are:
 This is an *optional step* as some modules might work without extra setup.
 But it depends on the specific module.
 
+You might also find interesting to read [[file:CONFIGURING.org][CONFIGURING]], where I'm
+elaborating on some rationales behind the current configuration system.
+
 ** private configuration (=my.config=)
 # TODO write about dynamic configuration
 # TODO add a command to edit config?? e.g. HPI config edit
@@ -103,12 +106,15 @@ Since it's a Python package, generally it's very *flexible* and there are many w
       username    = 'karlicoss'
 
   #+end_src
- 
-  I'm [[https://github.com/karlicoss/HPI/issues/12][working]] on improving the documentation for configuring the individual modules,
-  but in the meantime the easiest is perhaps to skim through the code of the module and see what config attributes it's using.
 
-  For example, if you search for =config.= in [[file:../my/emfit/__init__.py][emfit module]], you'll see that it's using =export_path=, =tz=, =excluded_sids= and =cache_path=.
-  Or you can just try running them and fill in the attributes Python complains about.
+  To find out which attributes you need to specify:
+
+  - check in [[file:MODULES.org][MODULES]]
+  - if there is nothing there, the easiest is perhaps to skim through the code of the module and to search for =config.= uses.
+   
+    For example, if you search for =config.= in [[file:../my/emfit/__init__.py][emfit module]], you'll see that it's using =export_path=, =tz=, =excluded_sids= and =cache_path=.
+
+  - or you can just try running them and fill in the attributes Python complains about!
 
 - My config layout is a bit more complicated:
 

my/reddit.py

@@ -12,7 +12,7 @@ from dataclasses import dataclass
 @dataclass
 class reddit(uconfig):
     '''
-    Reddit module uses [[rexport][https://github.com/karlicoss/rexport]] output
+    Reddit module uses [[https://github.com/karlicoss/rexport][rexport]] output
     '''
     export_path: Paths                     # path[s]/glob to the exported data
     rexport    : Optional[PathIsh] = None  # path to a local clone of rexport

my/twitter/twint.py

@@ -7,7 +7,7 @@ from dataclasses import dataclass
 from my.config import twint as user_config
 
 @dataclass
-class twitter(user_config):
+class twint(user_config):
     '''
     Uses [[https://github.com/twintproject/twint][Twint]] data export.
     '''
@@ -15,7 +15,7 @@ class twitter(user_config):
 
 
 from ..core.cfg import make_config
-config = make_config(twitter)
+config = make_config(twint)
 
 
 from datetime import datetime