core: add modules_ast for more robust module collection

my/body/exercise/cross_trainer.py

@@ -1,5 +1,5 @@
 '''
-My cross trainer exercise data, arbitrated between differen sources (mainly, Endomondo and various manual plaintext notes)
+My cross trainer exercise data, arbitrated from different sources (mainly, Endomondo and manual text notes)
 
 This is probably too specific to my needs, so later I will move it away to a personal 'layer'.
 For now it's worth keeping it here as an example and perhaps utility functions might be useful for other HPI modules.

my/core/util.py

@@ -11,6 +11,8 @@ from typing import List, Iterable, NamedTuple, Optional
 class HPIModule(NamedTuple):
     name: str
     skip_reason: Optional[str]
+    doc: Optional[str] = None
+    file: Optional[Path] = None
 
 
 def modules() -> Iterable[HPIModule]:
@@ -167,6 +169,42 @@ def _walk_packages(path=None, prefix='', onerror=None) -> Iterable[HPIModule]:
             path = [p for p in path if not seen(p)]
             yield from _walk_packages(path, mname+'.', onerror)
 
+
+def modules_via_ast() -> Iterable[HPIModule]:
+    '''
+    Experimental version, which isn't importing the modules, making it more robust and safe.
+    '''
+    import ast
+
+    my_root = Path(__file__).absolute().parent.parent
+    for f in sorted(my_root.rglob('*.py')):
+        if f.is_symlink():
+            continue # meh
+        mp = f.relative_to(my_root.parent)
+        if mp.name == '__init__.py':
+            mp = mp.parent
+        m = str(mp.with_suffix('')).replace('/', '.')
+        if ignored(m):
+            continue
+        a = ast.parse(f.read_text())
+        NM = '__NOT_HPI_MODULE__'
+        is_not_module = any(
+            getattr(node, 'name', None) == NM # direct definition
+            or
+            any(getattr(n, 'name', None) == NM for n in getattr(node, 'names', [])) # import from
+            for node in a.body)
+        if is_not_module:
+            continue
+        doc = ast.get_docstring(a, clean=False)
+
+        yield HPIModule(
+            name=m,
+            skip_reason=None,
+            doc=doc,
+            file=f.relative_to(my_root.parent), # todo not sure if should be relative
+        )
+
+
 # deprecate?
 def get_modules() -> List[HPIModule]:
     return list(modules())

my/github/common.py

@@ -47,6 +47,5 @@ def parse_dt(s: str) -> datetime:
     # TODO isoformat?
     return pytz.utc.localize(datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ'))
 
-# TODO not sure
+
-# def get_events() -> Iterable[Res[Event]]:
+from ..core import __NOT_HPI_MODULE__
-#     return sort_res_by(events(), key=lambda e: e.dt)

my/roamresearch.py

@@ -9,7 +9,7 @@ from typing import NamedTuple, Iterator, List, Optional
 
 import pytz
 
-from .common import get_files, LazyLogger, Json
+from .core import get_files, LazyLogger, Json
 
 from my.config import roamresearch as config
 

my/stackexchange/stexport.py

@@ -1,5 +1,5 @@
 '''
-Stackexchange data
+Stackexchange data (uses API via [[https://github.com/karlicoss/stexport][stexport]])
 '''
 REQUIRES = [
     'git+https://github.com/karlicoss/stexport',

my/time/tz/main.py

@@ -1,5 +1,5 @@
 '''
-Timezone data provider
+Timezone data provider, used to localize timezone-unaware timestamps for other modules
 '''
 from datetime import datetime
 from ...core.common import tzdatetime

my/time/tz/via_location.py

@@ -1,5 +1,5 @@
 '''
-Timezone data provider, useful for localizing UTC-only/timezone unaware dates.
+Timezone data provider, guesses timezone based on location data (e.g. GPS)
 '''
 REQUIRES = [
     # for determining timezone by coordinate

my/twitter/twint.py

@@ -1,7 +1,5 @@
 """
-Twitter data (tweets and favorites).
+Twitter data (tweets and favorites). Uses [[https://github.com/twintproject/twint][Twint]] data export.
-
-Uses [[https://github.com/twintproject/twint][Twint]] data export.
 
 Requirements: =pip3 install --user dataset=
 """

my/vk/favorites.py

@@ -1,7 +1,4 @@
-'''
+# todo: uses my private export script?
-todo: uses my private export script?
-'''
-
 from datetime import datetime
 import json
 from typing import NamedTuple, Iterable, Sequence, Optional

my/vk/vk_messages_backup.py

@@ -1,5 +1,5 @@
 '''
-Handles VK data exported by https://github.com/Totktonada/vk_messages_backup
+VK data (exported by [[https://github.com/Totktonada/vk_messages_backup][Totktonada/vk_messages_backup]])
 '''
 # note: could reuse the original repo, but little point I guess since VK closed their API