cli: add checks for importing modules

doc/MODULES.org

@@ -1,5 +1,10 @@
-This file is an overview of *documented* modules.
+This file is an overview of *documented* modules (which I'm progressively expanding).
-There are many more, see [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules, I'm progressively working on documenting them.
+
+There are many more, see:
+
+- [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules.
+- you can also run =hpi modules= to list what's available on your system
+- source code is always the primary source of truth
 
 If you have some issues with the setup, see [[file:SETUP.org::#troubleshooting]["Troubleshooting"]].
 
@@ -54,6 +59,7 @@ You don't have to set them up all at once, it's recommended to do it gradually.
 
 #+begin_src python :dir .. :results output drawer raw :exports result
 # TODO ugh, pkgutil.walk_packages doesn't recurse and find packages like my.twitter.archive??
+# yep.. https://stackoverflow.com/q/41203765/706389
 import importlib
 # from lint import all_modules # meh
 # TODO figure out how to discover configs automatically...

doc/SETUP.org

@@ -228,6 +228,10 @@ Generally you can just try using the module and then install missing packages vi
 HPI comes with a command line tool that can help you detect potential issues. Run:
 
 : hpi doctor
+: # alternatively, for more output:
+: hpi doctor --verbose
+
+If you only have few modules set up, lots of them will error for you, which is expected, so check the ones you expect to work.
 
 If you have any ideas on how to improve it, please let me know!
 

my/core/__main__.py

@@ -2,6 +2,7 @@ import os
 from pathlib import Path
 import sys
 from subprocess import check_call, run, PIPE
+import importlib
 import traceback
 
 from . import LazyLogger
@@ -9,9 +10,9 @@ from . import LazyLogger
 log = LazyLogger('HPI cli')
 
 class Modes:
-    HELLO  = 'hello'
+    CONFIG  = 'config'
-    CONFIG = 'config'
+    DOCTOR  = 'doctor'
-    DOCTOR = 'doctor'
+    MODULES = 'modules'
 
 
 def run_mypy(pkg):
@@ -39,28 +40,45 @@ def run_mypy(pkg):
     return mres
 
 
+def eprint(x: str):
+    print(x, file=sys.stderr)
+
+def indent(x: str) -> str:
+    return ''.join('   ' + l for l in x.splitlines(keepends=True))
+
+def info(x: str):
+    eprint('✅ ' + x)
+
+def error(x: str):
+    eprint('❌ ' + x)
+
+def warning(x: str):
+    eprint('❗ ' + x) # todo yellow?
+
+def tb(e):
+    tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__))
+    sys.stderr.write(indent(tb))
+
+
+class color:
+    BLACK = '\033[30m'
+    RED = '\033[31m'
+    GREEN = '\033[32m'
+    YELLOW = '\033[33m'
+    BLUE = '\033[34m'
+    MAGENTA = '\033[35m'
+    CYAN = '\033[36m'
+    WHITE = '\033[37m'
+    UNDERLINE = '\033[4m'
+    RESET = '\033[0m'
+
+
 def config_check(args):
-    def eprint(x: str):
-        print(x, file=sys.stderr)
-
-    def indent(x: str) -> str:
-        return ''.join('   ' + l for l in x.splitlines(keepends=True))
-
-    def info(x: str):
-        eprint('✅ ' + x)
-
-    def error(x: str):
-        eprint('❌ ' + x)
-
-    def warning(x: str):
-        eprint('❗ ' + x) # todo yellow?
-
     try:
         import my.config as cfg
     except Exception as e:
         error("failed to import the config")
-        tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__))
+        tb(e)
-        sys.stderr.write(indent(tb))
         sys.exit(1)
 
     info(f"config file: {cfg.__file__}")
@@ -80,12 +98,35 @@ def config_check(args):
         sys.stderr.write(indent(mres.stdout.decode('utf8')))
 
 
+def modules_check(args):
+    verbose = args.verbose
+
+    from .util import get_modules
+    for m in get_modules():
+        try:
+            importlib.import_module(m)
+        except Exception as e:
+            # todo more specific command?
+            vw = '' if verbose else '; pass --verbose to print more information'
+            warning(f'{color.RED}FAIL{color.RESET}: {m:<30} loading failed{vw}')
+            if verbose:
+                tb(e)
+
+        else:
+            info(f'{color.GREEN}OK{color.RESET}  : {m:<30}')
+
+
+def list_modules(args):
+    # todo with docs/etc?
+    from .util import get_modules
+    for m in get_modules():
+        print(f'- {m}')
+
+
+# todo check that it finds private modules too?
 def doctor(args):
     config_check(args)
-
+    modules_check(args)
-
-def hello(args):
-    print('Hello')
 
 
 def parser():
@@ -96,10 +137,8 @@ Tool for HPI.
 Work in progress, will be used for config management, troubleshooting & introspection
 ''')
     sp = p.add_subparsers(dest='mode')
-    hp = sp.add_parser(Modes.HELLO , help='TODO just a stub, remove later')
-    hp.set_defaults(func=hello)
-
     dp = sp.add_parser(Modes.DOCTOR, help='Run various checks')
+    dp.add_argument('--verbose', action='store_true', help='Print more diagnosic infomration')
     dp.set_defaults(func=doctor)
 
     cp = sp.add_parser(Modes.CONFIG, help='Work with configuration')
@@ -108,6 +147,9 @@ Work in progress, will be used for config management, troubleshooting & introspe
     #     ccp = scp.add_parser('check', help='Check config')
     #     ccp.set_defaults(func=config_check)
 
+    mp = sp.add_parser(Modes.MODULES, help='List available modules')
+    mp.set_defaults(func=list_modules)
+
     return p
 
 

my/core/util.py

@@ -0,0 +1,82 @@
+from pathlib import Path
+from itertools import chain
+import os
+import re
+import pkgutil
+from typing import List
+
+# TODO reuse in readme/blog post
+# borrowed from https://github.com/sanitizers/octomachinery/blob/24288774d6dcf977c5033ae11311dbff89394c89/tests/circular_imports_test.py#L22-L55
+def _find_all_importables(pkg):
+    """Find all importables in the project.
+    Return them in order.
+    """
+    return sorted(
+        set(
+            chain.from_iterable(
+                _discover_path_importables(Path(p), pkg.__name__)
+                for p in pkg.__path__
+            ),
+        ),
+    )
+
+
+def _discover_path_importables(pkg_pth, pkg_name):
+    """Yield all importables under a given path and package."""
+    for dir_path, _d, file_names in os.walk(pkg_pth):
+        pkg_dir_path = Path(dir_path)
+
+        if pkg_dir_path.parts[-1] == '__pycache__':
+            continue
+
+        if all(Path(_).suffix != '.py' for _ in file_names):
+            continue
+
+        rel_pt = pkg_dir_path.relative_to(pkg_pth)
+        pkg_pref = '.'.join((pkg_name, ) + rel_pt.parts)
+
+
+        yield from (
+            pkg_path
+            for _, pkg_path, _ in pkgutil.walk_packages(
+                (str(pkg_dir_path), ), prefix=f'{pkg_pref}.',
+            )
+        )
+
+
+# todo need a better way to mark module as 'interface'
+def ignored(m: str):
+    excluded = [
+        'kython.*',
+        'mycfg_stub',
+        'common',
+        'error',
+        'cfg',
+        'core.*',
+        'config.*',
+        'jawbone.plots',
+        'emfit.plot',
+
+        # todo think about these...
+        # 'google.takeout.paths',
+        'bluemaestro.check',
+        'location.__main__',
+        'photos.utils',
+        'books',
+        'coding',
+        'media',
+        'reading',
+        '_rss',
+        'twitter.common',
+        'rss.common',
+        'lastfm.fill_influxdb',
+    ]
+    exs = '|'.join(excluded)
+    return re.match(f'^my.({exs})$', m)
+
+
+def get_modules() -> List[str]:
+    import my as pkg # todo not sure?
+    importables = _find_all_importables(pkg)
+    public = [x for x in importables if not ignored(x)]
+    return public