From d890599c7c6fce15fee89010a011f1c137a32e20 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 25 May 2020 11:12:27 +0100 Subject: [PATCH] cli: add checks for importing modules --- doc/MODULES.org | 10 ++++- doc/SETUP.org | 4 ++ my/core/__main__.py | 96 ++++++++++++++++++++++++++++++++------------- my/core/util.py | 82 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 29 deletions(-) create mode 100644 my/core/util.py diff --git a/doc/MODULES.org b/doc/MODULES.org index 8632543..d45f8a1 100644 --- a/doc/MODULES.org +++ b/doc/MODULES.org @@ -1,5 +1,10 @@ -This file is an overview of *documented* modules. -There are many more, see [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules, I'm progressively working on documenting them. +This file is an overview of *documented* modules (which I'm progressively expanding). + +There are many more, see: + +- [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules. +- you can also run =hpi modules= to list what's available on your system +- source code is always the primary source of truth If you have some issues with the setup, see [[file:SETUP.org::#troubleshooting]["Troubleshooting"]]. @@ -54,6 +59,7 @@ You don't have to set them up all at once, it's recommended to do it gradually. #+begin_src python :dir .. :results output drawer raw :exports result # TODO ugh, pkgutil.walk_packages doesn't recurse and find packages like my.twitter.archive?? +# yep.. https://stackoverflow.com/q/41203765/706389 import importlib # from lint import all_modules # meh # TODO figure out how to discover configs automatically... diff --git a/doc/SETUP.org b/doc/SETUP.org index afc9b30..6eb26c3 100644 --- a/doc/SETUP.org +++ b/doc/SETUP.org @@ -228,6 +228,10 @@ Generally you can just try using the module and then install missing packages vi HPI comes with a command line tool that can help you detect potential issues. Run: : hpi doctor +: # alternatively, for more output: +: hpi doctor --verbose + +If you only have few modules set up, lots of them will error for you, which is expected, so check the ones you expect to work. If you have any ideas on how to improve it, please let me know! diff --git a/my/core/__main__.py b/my/core/__main__.py index c973afe..2de9537 100644 --- a/my/core/__main__.py +++ b/my/core/__main__.py @@ -2,6 +2,7 @@ import os from pathlib import Path import sys from subprocess import check_call, run, PIPE +import importlib import traceback from . import LazyLogger @@ -9,9 +10,9 @@ from . import LazyLogger log = LazyLogger('HPI cli') class Modes: - HELLO = 'hello' - CONFIG = 'config' - DOCTOR = 'doctor' + CONFIG = 'config' + DOCTOR = 'doctor' + MODULES = 'modules' def run_mypy(pkg): @@ -39,28 +40,45 @@ def run_mypy(pkg): return mres +def eprint(x: str): + print(x, file=sys.stderr) + +def indent(x: str) -> str: + return ''.join(' ' + l for l in x.splitlines(keepends=True)) + +def info(x: str): + eprint('✅ ' + x) + +def error(x: str): + eprint('❌ ' + x) + +def warning(x: str): + eprint('❗ ' + x) # todo yellow? + +def tb(e): + tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__)) + sys.stderr.write(indent(tb)) + + +class color: + BLACK = '\033[30m' + RED = '\033[31m' + GREEN = '\033[32m' + YELLOW = '\033[33m' + BLUE = '\033[34m' + MAGENTA = '\033[35m' + CYAN = '\033[36m' + WHITE = '\033[37m' + UNDERLINE = '\033[4m' + RESET = '\033[0m' + + def config_check(args): - def eprint(x: str): - print(x, file=sys.stderr) - - def indent(x: str) -> str: - return ''.join(' ' + l for l in x.splitlines(keepends=True)) - - def info(x: str): - eprint('✅ ' + x) - - def error(x: str): - eprint('❌ ' + x) - - def warning(x: str): - eprint('❗ ' + x) # todo yellow? - try: import my.config as cfg except Exception as e: error("failed to import the config") - tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__)) - sys.stderr.write(indent(tb)) + tb(e) sys.exit(1) info(f"config file: {cfg.__file__}") @@ -80,12 +98,35 @@ def config_check(args): sys.stderr.write(indent(mres.stdout.decode('utf8'))) +def modules_check(args): + verbose = args.verbose + + from .util import get_modules + for m in get_modules(): + try: + importlib.import_module(m) + except Exception as e: + # todo more specific command? + vw = '' if verbose else '; pass --verbose to print more information' + warning(f'{color.RED}FAIL{color.RESET}: {m:<30} loading failed{vw}') + if verbose: + tb(e) + + else: + info(f'{color.GREEN}OK{color.RESET} : {m:<30}') + + +def list_modules(args): + # todo with docs/etc? + from .util import get_modules + for m in get_modules(): + print(f'- {m}') + + +# todo check that it finds private modules too? def doctor(args): config_check(args) - - -def hello(args): - print('Hello') + modules_check(args) def parser(): @@ -96,10 +137,8 @@ Tool for HPI. Work in progress, will be used for config management, troubleshooting & introspection ''') sp = p.add_subparsers(dest='mode') - hp = sp.add_parser(Modes.HELLO , help='TODO just a stub, remove later') - hp.set_defaults(func=hello) - dp = sp.add_parser(Modes.DOCTOR, help='Run various checks') + dp.add_argument('--verbose', action='store_true', help='Print more diagnosic infomration') dp.set_defaults(func=doctor) cp = sp.add_parser(Modes.CONFIG, help='Work with configuration') @@ -108,6 +147,9 @@ Work in progress, will be used for config management, troubleshooting & introspe # ccp = scp.add_parser('check', help='Check config') # ccp.set_defaults(func=config_check) + mp = sp.add_parser(Modes.MODULES, help='List available modules') + mp.set_defaults(func=list_modules) + return p diff --git a/my/core/util.py b/my/core/util.py new file mode 100644 index 0000000..f76b3be --- /dev/null +++ b/my/core/util.py @@ -0,0 +1,82 @@ +from pathlib import Path +from itertools import chain +import os +import re +import pkgutil +from typing import List + +# TODO reuse in readme/blog post +# borrowed from https://github.com/sanitizers/octomachinery/blob/24288774d6dcf977c5033ae11311dbff89394c89/tests/circular_imports_test.py#L22-L55 +def _find_all_importables(pkg): + """Find all importables in the project. + Return them in order. + """ + return sorted( + set( + chain.from_iterable( + _discover_path_importables(Path(p), pkg.__name__) + for p in pkg.__path__ + ), + ), + ) + + +def _discover_path_importables(pkg_pth, pkg_name): + """Yield all importables under a given path and package.""" + for dir_path, _d, file_names in os.walk(pkg_pth): + pkg_dir_path = Path(dir_path) + + if pkg_dir_path.parts[-1] == '__pycache__': + continue + + if all(Path(_).suffix != '.py' for _ in file_names): + continue + + rel_pt = pkg_dir_path.relative_to(pkg_pth) + pkg_pref = '.'.join((pkg_name, ) + rel_pt.parts) + + + yield from ( + pkg_path + for _, pkg_path, _ in pkgutil.walk_packages( + (str(pkg_dir_path), ), prefix=f'{pkg_pref}.', + ) + ) + + +# todo need a better way to mark module as 'interface' +def ignored(m: str): + excluded = [ + 'kython.*', + 'mycfg_stub', + 'common', + 'error', + 'cfg', + 'core.*', + 'config.*', + 'jawbone.plots', + 'emfit.plot', + + # todo think about these... + # 'google.takeout.paths', + 'bluemaestro.check', + 'location.__main__', + 'photos.utils', + 'books', + 'coding', + 'media', + 'reading', + '_rss', + 'twitter.common', + 'rss.common', + 'lastfm.fill_influxdb', + ] + exs = '|'.join(excluded) + return re.match(f'^my.({exs})$', m) + + +def get_modules() -> List[str]: + import my as pkg # todo not sure? + importables = _find_all_importables(pkg) + public = [x for x in importables if not ignored(x)] + return public