Merge pull request #55 from karlicoss/updates

cli updates: doctor mode

doc/MODULES.org

@@ -1,5 +1,12 @@
-This file is an overview of *documented* modules.
+This file is an overview of *documented* modules (which I'm progressively expanding).
-There are many more, see [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules, I'm progressively working on documenting them.
+
+There are many more, see:
+
+- [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules.
+- you can also run =hpi modules= to list what's available on your system
+- source code is always the primary source of truth
+
+If you have some issues with the setup, see [[file:SETUP.org::#troubleshooting]["Troubleshooting"]].
 
 * TOC
 :PROPERTIES:
@@ -52,6 +59,7 @@ You don't have to set them up all at once, it's recommended to do it gradually.
 
 #+begin_src python :dir .. :results output drawer raw :exports result
 # TODO ugh, pkgutil.walk_packages doesn't recurse and find packages like my.twitter.archive??
+# yep.. https://stackoverflow.com/q/41203765/706389
 import importlib
 # from lint import all_modules # meh
 # TODO figure out how to discover configs automatically...

doc/SETUP.org

@@ -19,6 +19,7 @@ You'd be really helping me, I want to make the setup as straightforward as possi
 - [[#setting-up-modules][Setting up modules]]
   - [[#private-configuration-myconfig][private configuration (my.config)]]
   - [[#module-dependencies][module dependencies]]
+- [[#troubleshooting][Troubleshooting]]
 - [[#usage-examples][Usage examples]]
   - [[#end-to-end-roam-research-setup][End-to-end Roam Research setup]]
   - [[#polar][Polar]]
@@ -97,6 +98,7 @@ They aren't necessary, but will improve your experience. At the moment these are
 
 - [[https://github.com/karlicoss/cachew][cachew]]: automatic caching library, which can greatly speedup data access
 - [[https://github.com/metachris/logzero][logzero]]: a nice logging library, supporting colors
+- [[https://github.com/python/mypy][mypy]]: mypy is used for checking configs and troubleshooting
 
 * Setting up modules
 This is an *optional step* as few modules work without extra setup.
@@ -110,7 +112,6 @@ elaborating on some technical rationales behind the current configuration system
 ** private configuration (=my.config=)
 # TODO write about dynamic configuration
 # TODO add a command to edit config?? e.g. HPI config edit
-# HPI doctor?
 If you're not planning to use private configuration (some modules don't need it) you can skip straight to the next step. Still, I'd recommend you to read anyway.
 
 The configuration contains paths to the data on your disks, links to external repositories, etc.
@@ -118,7 +119,11 @@ The config is simply a *python package* (named =my.config=), expected to be in =
 
 Since it's a Python package, generally it's very *flexible* and there are many ways to set it up.
 
-- *The simplest and the very minimum* you need is =~/.config/my/my/config.py=. For example:
+- *The simplest way*
+
+  After installing HPI, run =hpi config init=.
+
+  This will create an empty config file for you (usually, in =~/.config/my=), which you can edit. Example configuration:
 
   #+begin_src python
   import pytz # yes, you can use any Python stuff in the config
@@ -220,6 +225,20 @@ Dependencies are different for specific modules you're planning to use, so it's
 
 Generally you can just try using the module and then install missing packages via ~pip3 install --user~, should be fairly straightforward.
 
+
+* Troubleshooting
+# todo replace with_my with it??
+
+HPI comes with a command line tool that can help you detect potential issues. Run:
+
+: hpi doctor
+: # alternatively, for more output:
+: hpi doctor --verbose
+
+If you only have few modules set up, lots of them will error for you, which is expected, so check the ones you expect to work.
+
+If you have any ideas on how to improve it, please let me know!
+
 * Usage examples
 If you run your script with ~with_my~ wrapper, you'd have ~my~ in ~PYTHONPATH~ which gives you access to your data from within the script.
 

my/__main__.py

@@ -1,30 +0,0 @@
-class Modes:
-    HELLO = 'hello'
-
-
-def parser():
-    from argparse import ArgumentParser
-    p = ArgumentParser('Human Programming Interface', epilog='''
-Tool for HPI.
-
-Work in progress, will be used for config management, troubleshooting & introspection
-''')
-    sp = p.add_subparsers(dest='mode')
-    sp.add_parser(Modes.HELLO, help='TODO just a stub, remove later')
-    return p
-
-
-def main():
-    p = parser()
-    args = p.parse_args()
-    mode = args.mode
-    if mode == Modes.HELLO:
-        print('hi')
-    else:
-        import sys
-        p.print_usage()
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    main()

my/core/__main__.py

@@ -0,0 +1,204 @@
+import os
+from pathlib import Path
+import sys
+from subprocess import check_call, run, PIPE
+import importlib
+import traceback
+
+from . import LazyLogger
+
+log = LazyLogger('HPI cli')
+
+
+def run_mypy(pkg):
+    from .preinit import get_mycfg_dir
+    mycfg_dir = get_mycfg_dir()
+    # todo ugh. not sure how to extract it from pkg?
+
+    # todo dunno maybe use the same mypy config in repository?
+    # I'd need to install mypy.ini then??
+    env = {**os.environ}
+    mpath = env.get('MYPYPATH')
+    mpath = str(mycfg_dir) + ('' if mpath is None else f':{mpath}')
+    env['MYPYPATH'] = mpath
+
+    mres = run([
+        'python3', '-m', 'mypy',
+        '--namespace-packages',
+        '--color-output', # not sure if works??
+        '--pretty',
+        '--show-error-codes',
+        '--show-error-context',
+        '--check-untyped-defs',
+        '-p', pkg.__name__,
+    ], stderr=PIPE, stdout=PIPE, env=env)
+    return mres
+
+
+def eprint(x: str):
+    print(x, file=sys.stderr)
+
+def indent(x: str) -> str:
+    return ''.join('   ' + l for l in x.splitlines(keepends=True))
+
+def info(x: str):
+    eprint('✅ ' + x)
+
+def error(x: str):
+    eprint('❌ ' + x)
+
+def warning(x: str):
+    eprint('❗ ' + x) # todo yellow?
+
+def tb(e):
+    tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__))
+    sys.stderr.write(indent(tb))
+
+
+class color:
+    BLACK = '\033[30m'
+    RED = '\033[31m'
+    GREEN = '\033[32m'
+    YELLOW = '\033[33m'
+    BLUE = '\033[34m'
+    MAGENTA = '\033[35m'
+    CYAN = '\033[36m'
+    WHITE = '\033[37m'
+    UNDERLINE = '\033[4m'
+    RESET = '\033[0m'
+
+
+def config_create(args):
+    from .preinit import get_mycfg_dir
+    mycfg_dir = get_mycfg_dir()
+
+    created = False
+    if not mycfg_dir.exists():
+        # todo not sure about the layout... should I use my/config.py instead?
+        my_config = mycfg_dir / 'my' / 'config' / '__init__.py'
+
+        my_config.parent.mkdir(parents=True)
+        my_config.touch()
+        info(f'created empty config: {my_config}')
+        created = True
+    else:
+        error(f"config directory '{mycfg_dir}' already exists, skipping creation")
+
+    config_check(args)
+    if not created:
+        sys.exit(1)
+
+
+def config_check(args):
+    try:
+        import my.config as cfg
+    except Exception as e:
+        error("failed to import the config")
+        tb(e)
+        sys.exit(1)
+
+    info(f"config file: {cfg.__file__}")
+
+    try:
+        import mypy
+    except ImportError:
+        warning("mypy not found, can't check config with it")
+    else:
+        mres = run_mypy(cfg)
+        rc = mres.returncode
+        if rc == 0:
+            info('mypy check: success')
+        else:
+            error('mypy check: failed')
+            sys.stderr.write(indent(mres.stderr.decode('utf8')))
+        sys.stderr.write(indent(mres.stdout.decode('utf8')))
+
+
+def modules_check(args):
+    verbose = args.verbose
+    vw = '' if verbose else '; pass --verbose to print more information'
+
+    from .util import get_modules
+    for m in get_modules():
+        try:
+            mod = importlib.import_module(m)
+        except Exception as e:
+            # todo more specific command?
+            warning(f'{color.RED}FAIL{color.RESET}: {m:<30} loading failed{vw}')
+            if verbose:
+                tb(e)
+            continue
+
+        info(f'{color.GREEN}OK{color.RESET}  : {m:<30}')
+        stats = getattr(mod, 'stats', None)
+        if stats is None:
+            continue
+        try:
+            res = stats()
+        except Exception as ee:
+            warning(f'     - {color.RED}stats:{color.RESET}                      computing failed{vw}')
+            if verbose:
+                tb(ee)
+        else:
+            info(f'    - stats: {res}')
+
+
+def list_modules(args):
+    # todo with docs/etc?
+    from .util import get_modules
+    for m in get_modules():
+        print(f'- {m}')
+
+
+# todo check that it finds private modules too?
+def doctor(args):
+    config_check(args)
+    modules_check(args)
+
+
+def parser():
+    from argparse import ArgumentParser
+    p = ArgumentParser('Human Programming Interface', epilog='''
+Tool for HPI.
+
+Work in progress, will be used for config management, troubleshooting & introspection
+''')
+    sp = p.add_subparsers(dest='mode')
+    dp = sp.add_parser('doctor', help='Run various checks')
+    dp.add_argument('--verbose', action='store_true', help='Print more diagnosic infomration')
+    dp.set_defaults(func=doctor)
+
+    cp = sp.add_parser('config', help='Work with configuration')
+    scp = cp.add_subparsers(dest='mode')
+    if True:
+        ccp = scp.add_parser('check', help='Check config')
+        ccp.set_defaults(func=config_check)
+
+        icp = scp.add_parser('create', help='Create user config')
+        icp.set_defaults(func=config_create)
+
+    mp = sp.add_parser('modules', help='List available modules')
+    mp.set_defaults(func=list_modules)
+
+    return p
+
+
+def main():
+    p = parser()
+    args = p.parse_args()
+
+    func = args.func
+    if func is None:
+        # shouldn't happen.. but just in case
+        p.print_usage()
+        sys.exit(1)
+
+    import tempfile
+    with tempfile.TemporaryDirectory() as td:
+        # cd into tmp dir to prevent accidental imports..
+        os.chdir(str(td))
+        args.func(args)
+
+
+if __name__ == '__main__':
+    main()

my/core/init.py

@@ -23,22 +23,15 @@ def assign_module(parent: str, name: str, module: ModuleType) -> None:
 
 del ModuleType
 
+
 # separate function to present namespace pollution
 def setup_config() -> None:
-    from pathlib import Path
     import sys
-    import os
     import warnings
     from typing import Optional
-    import appdirs # type: ignore[import]
 
-    # not sure if that's necessary, i.e. could rely on PYTHONPATH instead
+    from .preinit import get_mycfg_dir
-    # on the other hand, by using MY_CONFIG we are guaranteed to load it from the desired path?
+    mycfg_dir = get_mycfg_dir()
-    mvar = os.environ.get('MY_CONFIG')
-    if mvar is not None:
-        mycfg_dir = Path(mvar)
-    else:
-        mycfg_dir = Path(appdirs.user_config_dir('my'))
 
     if not mycfg_dir.exists():
         warnings.warn(f"""

my/core/preinit.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+
+def get_mycfg_dir() -> Path:
+    import appdirs # type: ignore[import]
+    import os
+    # not sure if that's necessary, i.e. could rely on PYTHONPATH instead
+    # on the other hand, by using MY_CONFIG we are guaranteed to load it from the desired path?
+    mvar = os.environ.get('MY_CONFIG')
+    if mvar is not None:
+        mycfg_dir = Path(mvar)
+    else:
+        mycfg_dir = Path(appdirs.user_config_dir('my'))
+    return mycfg_dir

my/core/util.py

@@ -0,0 +1,82 @@
+from pathlib import Path
+from itertools import chain
+import os
+import re
+import pkgutil
+from typing import List
+
+# TODO reuse in readme/blog post
+# borrowed from https://github.com/sanitizers/octomachinery/blob/24288774d6dcf977c5033ae11311dbff89394c89/tests/circular_imports_test.py#L22-L55
+def _find_all_importables(pkg):
+    """Find all importables in the project.
+    Return them in order.
+    """
+    return sorted(
+        set(
+            chain.from_iterable(
+                _discover_path_importables(Path(p), pkg.__name__)
+                for p in pkg.__path__
+            ),
+        ),
+    )
+
+
+def _discover_path_importables(pkg_pth, pkg_name):
+    """Yield all importables under a given path and package."""
+    for dir_path, _d, file_names in os.walk(pkg_pth):
+        pkg_dir_path = Path(dir_path)
+
+        if pkg_dir_path.parts[-1] == '__pycache__':
+            continue
+
+        if all(Path(_).suffix != '.py' for _ in file_names):
+            continue
+
+        rel_pt = pkg_dir_path.relative_to(pkg_pth)
+        pkg_pref = '.'.join((pkg_name, ) + rel_pt.parts)
+
+
+        yield from (
+            pkg_path
+            for _, pkg_path, _ in pkgutil.walk_packages(
+                (str(pkg_dir_path), ), prefix=f'{pkg_pref}.',
+            )
+        )
+
+
+# todo need a better way to mark module as 'interface'
+def ignored(m: str):
+    excluded = [
+        'kython.*',
+        'mycfg_stub',
+        'common',
+        'error',
+        'cfg',
+        'core.*',
+        'config.*',
+        'jawbone.plots',
+        'emfit.plot',
+
+        # todo think about these...
+        # 'google.takeout.paths',
+        'bluemaestro.check',
+        'location.__main__',
+        'photos.utils',
+        'books',
+        'coding',
+        'media',
+        'reading',
+        '_rss',
+        'twitter.common',
+        'rss.common',
+        'lastfm.fill_influxdb',
+    ]
+    exs = '|'.join(excluded)
+    return re.match(f'^my.({exs})$', m)
+
+
+def get_modules() -> List[str]:
+    import my as pkg # todo not sure?
+    importables = _find_all_importables(pkg)
+    public = [x for x in importables if not ignored(x)]
+    return public

my/emfit/__init__.py

@@ -322,6 +322,12 @@ def by_night() -> Dict[date, Emfit]:
     return res
 
 
+def stats():
+    return {
+        'nights': len(by_night()),
+    }
+
+
 def main():
     for k, v in by_night().items():
         print(k, v.start, v.end)

my/foursquare.py

@@ -90,3 +90,10 @@ def get_cid_map(bfile: str):
 
 def print_checkins():
     print(get_checkins())
+
+
+def stats():
+    from more_itertools import ilen
+    return {
+        'checkins': ilen(get_checkins()),
+    }

my/hypothesis.py

@@ -69,6 +69,16 @@ def pages() -> List[Res[Page]]:
     return sort_res_by(_dal().pages(), key=lambda h: h.created)
 
 
+# todo not public api yet
+def stats():
+    # todo add 'last date' checks et
+    return {
+        # todo ilen
+        'highlights': len(highlights()),
+        'pages'     : len(pages()),
+    }
+
+
 def _main():
     for page in get_pages():
         print(page)

my/rss/feedbin.py

@@ -40,3 +40,10 @@ def states() -> Iterable[SubscriptionState]:
         dt = isoparse(dts)
         subs = parse_file(f)
         yield dt, subs
+
+
+def stats():
+    from more_itertools import ilen, last
+    return {
+        'subscriptions': ilen(last(states())[1])
+    }

setup.py

@@ -61,13 +61,14 @@ def main():
                 # TODO document these?
                 'logzero',
                 'cachew',
+                'mypy', # used for config checks
             ],
             ':python_version<"3.7"': [
                 # used for some modules... hopefully reasonable to have it as a default
                 'dataclasses',
             ],
         },
-        entry_points={'console_scripts': ['hpi=my.__main__:main']},
+        entry_points={'console_scripts': ['hpi=my.core.__main__:main']},
     )
 
 

tests/foursquare.py

@@ -1,6 +1,7 @@
 from my.foursquare import get_checkins
 
 def test_checkins():
+    # todo reuse stats?
     checkins = get_checkins()
     assert len(checkins) > 100
     assert any('Victoria Park' in c.summary for c in checkins)

tox.ini

@@ -17,7 +17,7 @@ commands =
     # TODO add; once I figure out porg depdencency?? tests/config.py
     # TODO run demo.py? just make sure with_my is a bit cleverer?
     # TODO e.g. under CI, rely on installing
-    hpi hello
+    hpi modules
 
 
 [testenv:demo]