cli: add checks for importing modules

my/core/__main__.py

@@ -2,6 +2,7 @@ import os
 from pathlib import Path
 import sys
 from subprocess import check_call, run, PIPE
+import importlib
 import traceback
 
 from . import LazyLogger
@@ -9,9 +10,9 @@ from . import LazyLogger
 log = LazyLogger('HPI cli')
 
 class Modes:
-    HELLO  = 'hello'
-    CONFIG = 'config'
-    DOCTOR = 'doctor'
+    CONFIG  = 'config'
+    DOCTOR  = 'doctor'
+    MODULES = 'modules'
 
 
 def run_mypy(pkg):
@@ -39,28 +40,45 @@ def run_mypy(pkg):
     return mres
 
 
+def eprint(x: str):
+    print(x, file=sys.stderr)
+
+def indent(x: str) -> str:
+    return ''.join('   ' + l for l in x.splitlines(keepends=True))
+
+def info(x: str):
+    eprint('✅ ' + x)
+
+def error(x: str):
+    eprint('❌ ' + x)
+
+def warning(x: str):
+    eprint('❗ ' + x) # todo yellow?
+
+def tb(e):
+    tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__))
+    sys.stderr.write(indent(tb))
+
+
+class color:
+    BLACK = '\033[30m'
+    RED = '\033[31m'
+    GREEN = '\033[32m'
+    YELLOW = '\033[33m'
+    BLUE = '\033[34m'
+    MAGENTA = '\033[35m'
+    CYAN = '\033[36m'
+    WHITE = '\033[37m'
+    UNDERLINE = '\033[4m'
+    RESET = '\033[0m'
+
+
 def config_check(args):
-    def eprint(x: str):
-        print(x, file=sys.stderr)
-
-    def indent(x: str) -> str:
-        return ''.join('   ' + l for l in x.splitlines(keepends=True))
-
-    def info(x: str):
-        eprint('✅ ' + x)
-
-    def error(x: str):
-        eprint('❌ ' + x)
-
-    def warning(x: str):
-        eprint('❗ ' + x) # todo yellow?
-
     try:
         import my.config as cfg
     except Exception as e:
         error("failed to import the config")
-        tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__))
-        sys.stderr.write(indent(tb))
+        tb(e)
         sys.exit(1)
 
     info(f"config file: {cfg.__file__}")
@@ -80,12 +98,35 @@ def config_check(args):
         sys.stderr.write(indent(mres.stdout.decode('utf8')))
 
 
+def modules_check(args):
+    verbose = args.verbose
+
+    from .util import get_modules
+    for m in get_modules():
+        try:
+            importlib.import_module(m)
+        except Exception as e:
+            # todo more specific command?
+            vw = '' if verbose else '; pass --verbose to print more information'
+            warning(f'{color.RED}FAIL{color.RESET}: {m:<30} loading failed{vw}')
+            if verbose:
+                tb(e)
+
+        else:
+            info(f'{color.GREEN}OK{color.RESET}  : {m:<30}')
+
+
+def list_modules(args):
+    # todo with docs/etc?
+    from .util import get_modules
+    for m in get_modules():
+        print(f'- {m}')
+
+
+# todo check that it finds private modules too?
 def doctor(args):
     config_check(args)
-
-
-def hello(args):
-    print('Hello')
+    modules_check(args)
 
 
 def parser():
@@ -96,10 +137,8 @@ Tool for HPI.
 Work in progress, will be used for config management, troubleshooting & introspection
 ''')
     sp = p.add_subparsers(dest='mode')
-    hp = sp.add_parser(Modes.HELLO , help='TODO just a stub, remove later')
-    hp.set_defaults(func=hello)
-
     dp = sp.add_parser(Modes.DOCTOR, help='Run various checks')
+    dp.add_argument('--verbose', action='store_true', help='Print more diagnosic infomration')
     dp.set_defaults(func=doctor)
 
     cp = sp.add_parser(Modes.CONFIG, help='Work with configuration')
@@ -108,6 +147,9 @@ Work in progress, will be used for config management, troubleshooting & introspe
     #     ccp = scp.add_parser('check', help='Check config')
     #     ccp.set_defaults(func=config_check)
 
+    mp = sp.add_parser(Modes.MODULES, help='List available modules')
+    mp.set_defaults(func=list_modules)
+
     return p
 
 

my/core/util.py

@@ -0,0 +1,82 @@
+from pathlib import Path
+from itertools import chain
+import os
+import re
+import pkgutil
+from typing import List
+
+# TODO reuse in readme/blog post
+# borrowed from https://github.com/sanitizers/octomachinery/blob/24288774d6dcf977c5033ae11311dbff89394c89/tests/circular_imports_test.py#L22-L55
+def _find_all_importables(pkg):
+    """Find all importables in the project.
+    Return them in order.
+    """
+    return sorted(
+        set(
+            chain.from_iterable(
+                _discover_path_importables(Path(p), pkg.__name__)
+                for p in pkg.__path__
+            ),
+        ),
+    )
+
+
+def _discover_path_importables(pkg_pth, pkg_name):
+    """Yield all importables under a given path and package."""
+    for dir_path, _d, file_names in os.walk(pkg_pth):
+        pkg_dir_path = Path(dir_path)
+
+        if pkg_dir_path.parts[-1] == '__pycache__':
+            continue
+
+        if all(Path(_).suffix != '.py' for _ in file_names):
+            continue
+
+        rel_pt = pkg_dir_path.relative_to(pkg_pth)
+        pkg_pref = '.'.join((pkg_name, ) + rel_pt.parts)
+
+
+        yield from (
+            pkg_path
+            for _, pkg_path, _ in pkgutil.walk_packages(
+                (str(pkg_dir_path), ), prefix=f'{pkg_pref}.',
+            )
+        )
+
+
+# todo need a better way to mark module as 'interface'
+def ignored(m: str):
+    excluded = [
+        'kython.*',
+        'mycfg_stub',
+        'common',
+        'error',
+        'cfg',
+        'core.*',
+        'config.*',
+        'jawbone.plots',
+        'emfit.plot',
+
+        # todo think about these...
+        # 'google.takeout.paths',
+        'bluemaestro.check',
+        'location.__main__',
+        'photos.utils',
+        'books',
+        'coding',
+        'media',
+        'reading',
+        '_rss',
+        'twitter.common',
+        'rss.common',
+        'lastfm.fill_influxdb',
+    ]
+    exs = '|'.join(excluded)
+    return re.match(f'^my.({exs})$', m)
+
+
+def get_modules() -> List[str]:
+    import my as pkg # todo not sure?
+    importables = _find_all_importables(pkg)
+    public = [x for x in importables if not ignored(x)]
+    return public