From 2ede5b3a5c05532cd2d111cd7289abf7ba5f4a0d Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 25 May 2020 09:46:25 +0100 Subject: [PATCH 1/6] cli: add config check command --- my/__main__.py | 94 ++++++++++++++++++++++++++++++++++++++++++++++---- setup.py | 1 + 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/my/__main__.py b/my/__main__.py index 6ac0aad..4c6ac87 100644 --- a/my/__main__.py +++ b/my/__main__.py @@ -1,5 +1,72 @@ +import os +import sys +from subprocess import check_call, run, PIPE +import traceback + +from my.core import LazyLogger + +log = LazyLogger('HPI cli') + class Modes: - HELLO = 'hello' + HELLO = 'hello' + CONFIG = 'config' + + +def config_check(args): + def eprint(x: str): + print(x, file=sys.stderr) + + def indent(x: str) -> str: + return ''.join(' ' + l for l in x.splitlines(keepends=True)) + + def info(x: str): + eprint('✅ ' + x) + + def error(x: str): + eprint('❌ ' + x) + + def warning(x: str): + eprint('❗ ' + x) # todo yellow? + + try: + import my.config as cfg + except Exception as e: + error("failed to import the config") + tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__)) + sys.stderr.write(indent(tb)) + sys.exit(1) + + info(f"config file: {cfg.__file__}") + + try: + import mypy + except ImportError: + warning("mypy not found, can't check config with it") + else: + # todo dunno maybe use the same mypy config in repository? + # I'd need to install mypy.ini then?? + # todo how to bring it into mypypath? cooperate with core, maybe? + mres = run([ + 'python3', '-m', 'mypy', + '--namespace-packages', + '--color-output', # not sure if works?? + '--pretty', + '--show-error-codes', + '--show-error-context', + '--check-untyped-defs', + '-p', 'my.config' + ], stderr=PIPE, stdout=PIPE) + rc = mres.returncode + if rc == 0: + info('mypy check: success') + else: + error('mypy check: failed') + sys.stderr.write(indent(mres.stderr.decode('utf8'))) + sys.stderr.write(indent(mres.stdout.decode('utf8'))) + + +def hello(args): + print('Hello') def parser(): @@ -10,21 +77,34 @@ Tool for HPI. Work in progress, will be used for config management, troubleshooting & introspection ''') sp = p.add_subparsers(dest='mode') - sp.add_parser(Modes.HELLO, help='TODO just a stub, remove later') + hp = sp.add_parser(Modes.HELLO , help='TODO just a stub, remove later') + hp.set_defaults(func=hello) + + cp = sp.add_parser(Modes.CONFIG, help='Work with configuration') + scp = cp.add_subparsers(dest='mode') + if True: + ccp = scp.add_parser('check', help='Check config') + ccp.set_defaults(func=config_check) + return p def main(): p = parser() args = p.parse_args() - mode = args.mode - if mode == Modes.HELLO: - print('hi') - else: - import sys + + func = args.func + if func is None: + # shouldn't happen.. but just in case p.print_usage() sys.exit(1) + import tempfile + with tempfile.TemporaryDirectory() as td: + # cd into tmp dir to prevent accidental imports.. + os.chdir(str(td)) + args.func(args) + if __name__ == '__main__': main() diff --git a/setup.py b/setup.py index 4acce99..b68d77f 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ def main(): # TODO document these? 'logzero', 'cachew', + 'mypy', # used for config checks ], ':python_version<"3.7"': [ # used for some modules... hopefully reasonable to have it as a default From dab29a44b590e0d445115e7700747438d400fe0d Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 25 May 2020 10:04:58 +0100 Subject: [PATCH 2/6] cli: detect config properly in mypy check --- my/__main__.py | 40 +++++++++++++++++++++++++++------------- my/core/init.py | 22 ++++++++++++++-------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/my/__main__.py b/my/__main__.py index 4c6ac87..b465669 100644 --- a/my/__main__.py +++ b/my/__main__.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import sys from subprocess import check_call, run, PIPE import traceback @@ -12,6 +13,31 @@ class Modes: CONFIG = 'config' +def run_mypy(pkg): + from my.core.init import get_mycfg_dir + mycfg_dir = get_mycfg_dir() + # todo ugh. not sure how to extract it from pkg? + + # todo dunno maybe use the same mypy config in repository? + # I'd need to install mypy.ini then?? + env = {**os.environ} + mpath = env.get('MYPYPATH') + mpath = str(mycfg_dir) + ('' if mpath is None else f':{mpath}') + env['MYPYPATH'] = mpath + + mres = run([ + 'python3', '-m', 'mypy', + '--namespace-packages', + '--color-output', # not sure if works?? + '--pretty', + '--show-error-codes', + '--show-error-context', + '--check-untyped-defs', + '-p', pkg.__name__, + ], stderr=PIPE, stdout=PIPE, env=env) + return mres + + def config_check(args): def eprint(x: str): print(x, file=sys.stderr) @@ -43,19 +69,7 @@ def config_check(args): except ImportError: warning("mypy not found, can't check config with it") else: - # todo dunno maybe use the same mypy config in repository? - # I'd need to install mypy.ini then?? - # todo how to bring it into mypypath? cooperate with core, maybe? - mres = run([ - 'python3', '-m', 'mypy', - '--namespace-packages', - '--color-output', # not sure if works?? - '--pretty', - '--show-error-codes', - '--show-error-context', - '--check-untyped-defs', - '-p', 'my.config' - ], stderr=PIPE, stdout=PIPE) + mres = run_mypy(cfg) rc = mres.returncode if rc == 0: info('mypy check: success') diff --git a/my/core/init.py b/my/core/init.py index dd21a0a..d2c7195 100644 --- a/my/core/init.py +++ b/my/core/init.py @@ -23,15 +23,11 @@ def assign_module(parent: str, name: str, module: ModuleType) -> None: del ModuleType -# separate function to present namespace pollution -def setup_config() -> None: - from pathlib import Path - import sys - import os - import warnings - from typing import Optional - import appdirs # type: ignore[import] +def get_mycfg_dir(): + import appdirs # type: ignore[import] + from pathlib import Path + import os # not sure if that's necessary, i.e. could rely on PYTHONPATH instead # on the other hand, by using MY_CONFIG we are guaranteed to load it from the desired path? mvar = os.environ.get('MY_CONFIG') @@ -39,6 +35,16 @@ def setup_config() -> None: mycfg_dir = Path(mvar) else: mycfg_dir = Path(appdirs.user_config_dir('my')) + return mycfg_dir + + +# separate function to present namespace pollution +def setup_config() -> None: + import sys + import warnings + from typing import Optional + + mycfg_dir = get_mycfg_dir() if not mycfg_dir.exists(): warnings.warn(f""" From 8019389ccb369e9c71987ad6a7c2295d6e9b312e Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 25 May 2020 10:17:40 +0100 Subject: [PATCH 3/6] cli: move doctor to core, add doc --- doc/MODULES.org | 2 ++ doc/SETUP.org | 11 +++++++++++ my/{ => core}/__main__.py | 18 +++++++++++++----- setup.py | 2 +- 4 files changed, 27 insertions(+), 6 deletions(-) rename my/{ => core}/__main__.py (89%) diff --git a/doc/MODULES.org b/doc/MODULES.org index e26d439..8632543 100644 --- a/doc/MODULES.org +++ b/doc/MODULES.org @@ -1,6 +1,8 @@ This file is an overview of *documented* modules. There are many more, see [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules, I'm progressively working on documenting them. +If you have some issues with the setup, see [[file:SETUP.org::#troubleshooting]["Troubleshooting"]]. + * TOC :PROPERTIES: :TOC: :include all diff --git a/doc/SETUP.org b/doc/SETUP.org index 73e2a82..afc9b30 100644 --- a/doc/SETUP.org +++ b/doc/SETUP.org @@ -19,6 +19,7 @@ You'd be really helping me, I want to make the setup as straightforward as possi - [[#setting-up-modules][Setting up modules]] - [[#private-configuration-myconfig][private configuration (my.config)]] - [[#module-dependencies][module dependencies]] +- [[#troubleshooting][Troubleshooting]] - [[#usage-examples][Usage examples]] - [[#end-to-end-roam-research-setup][End-to-end Roam Research setup]] - [[#polar][Polar]] @@ -220,6 +221,16 @@ Dependencies are different for specific modules you're planning to use, so it's Generally you can just try using the module and then install missing packages via ~pip3 install --user~, should be fairly straightforward. + +* Troubleshooting +# todo replace with_my with it?? + +HPI comes with a command line tool that can help you detect potential issues. Run: + +: hpi doctor + +If you have any ideas on how to improve it, please let me know! + * Usage examples If you run your script with ~with_my~ wrapper, you'd have ~my~ in ~PYTHONPATH~ which gives you access to your data from within the script. diff --git a/my/__main__.py b/my/core/__main__.py similarity index 89% rename from my/__main__.py rename to my/core/__main__.py index b465669..c973afe 100644 --- a/my/__main__.py +++ b/my/core/__main__.py @@ -4,17 +4,18 @@ import sys from subprocess import check_call, run, PIPE import traceback -from my.core import LazyLogger +from . import LazyLogger log = LazyLogger('HPI cli') class Modes: HELLO = 'hello' CONFIG = 'config' + DOCTOR = 'doctor' def run_mypy(pkg): - from my.core.init import get_mycfg_dir + from .init import get_mycfg_dir mycfg_dir = get_mycfg_dir() # todo ugh. not sure how to extract it from pkg? @@ -79,6 +80,10 @@ def config_check(args): sys.stderr.write(indent(mres.stdout.decode('utf8'))) +def doctor(args): + config_check(args) + + def hello(args): print('Hello') @@ -94,11 +99,14 @@ Work in progress, will be used for config management, troubleshooting & introspe hp = sp.add_parser(Modes.HELLO , help='TODO just a stub, remove later') hp.set_defaults(func=hello) + dp = sp.add_parser(Modes.DOCTOR, help='Run various checks') + dp.set_defaults(func=doctor) + cp = sp.add_parser(Modes.CONFIG, help='Work with configuration') scp = cp.add_subparsers(dest='mode') - if True: - ccp = scp.add_parser('check', help='Check config') - ccp.set_defaults(func=config_check) + # if True: + # ccp = scp.add_parser('check', help='Check config') + # ccp.set_defaults(func=config_check) return p diff --git a/setup.py b/setup.py index b68d77f..f2c2b03 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def main(): 'dataclasses', ], }, - entry_points={'console_scripts': ['hpi=my.__main__:main']}, + entry_points={'console_scripts': ['hpi=my.core.__main__:main']}, ) From d890599c7c6fce15fee89010a011f1c137a32e20 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 25 May 2020 11:12:27 +0100 Subject: [PATCH 4/6] cli: add checks for importing modules --- doc/MODULES.org | 10 ++++- doc/SETUP.org | 4 ++ my/core/__main__.py | 96 ++++++++++++++++++++++++++++++++------------- my/core/util.py | 82 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 29 deletions(-) create mode 100644 my/core/util.py diff --git a/doc/MODULES.org b/doc/MODULES.org index 8632543..d45f8a1 100644 --- a/doc/MODULES.org +++ b/doc/MODULES.org @@ -1,5 +1,10 @@ -This file is an overview of *documented* modules. -There are many more, see [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules, I'm progressively working on documenting them. +This file is an overview of *documented* modules (which I'm progressively expanding). + +There are many more, see: + +- [[file:../README.org::#whats-inside]["What's inside"]] for the full list of modules. +- you can also run =hpi modules= to list what's available on your system +- source code is always the primary source of truth If you have some issues with the setup, see [[file:SETUP.org::#troubleshooting]["Troubleshooting"]]. @@ -54,6 +59,7 @@ You don't have to set them up all at once, it's recommended to do it gradually. #+begin_src python :dir .. :results output drawer raw :exports result # TODO ugh, pkgutil.walk_packages doesn't recurse and find packages like my.twitter.archive?? +# yep.. https://stackoverflow.com/q/41203765/706389 import importlib # from lint import all_modules # meh # TODO figure out how to discover configs automatically... diff --git a/doc/SETUP.org b/doc/SETUP.org index afc9b30..6eb26c3 100644 --- a/doc/SETUP.org +++ b/doc/SETUP.org @@ -228,6 +228,10 @@ Generally you can just try using the module and then install missing packages vi HPI comes with a command line tool that can help you detect potential issues. Run: : hpi doctor +: # alternatively, for more output: +: hpi doctor --verbose + +If you only have few modules set up, lots of them will error for you, which is expected, so check the ones you expect to work. If you have any ideas on how to improve it, please let me know! diff --git a/my/core/__main__.py b/my/core/__main__.py index c973afe..2de9537 100644 --- a/my/core/__main__.py +++ b/my/core/__main__.py @@ -2,6 +2,7 @@ import os from pathlib import Path import sys from subprocess import check_call, run, PIPE +import importlib import traceback from . import LazyLogger @@ -9,9 +10,9 @@ from . import LazyLogger log = LazyLogger('HPI cli') class Modes: - HELLO = 'hello' - CONFIG = 'config' - DOCTOR = 'doctor' + CONFIG = 'config' + DOCTOR = 'doctor' + MODULES = 'modules' def run_mypy(pkg): @@ -39,28 +40,45 @@ def run_mypy(pkg): return mres +def eprint(x: str): + print(x, file=sys.stderr) + +def indent(x: str) -> str: + return ''.join(' ' + l for l in x.splitlines(keepends=True)) + +def info(x: str): + eprint('✅ ' + x) + +def error(x: str): + eprint('❌ ' + x) + +def warning(x: str): + eprint('❗ ' + x) # todo yellow? + +def tb(e): + tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__)) + sys.stderr.write(indent(tb)) + + +class color: + BLACK = '\033[30m' + RED = '\033[31m' + GREEN = '\033[32m' + YELLOW = '\033[33m' + BLUE = '\033[34m' + MAGENTA = '\033[35m' + CYAN = '\033[36m' + WHITE = '\033[37m' + UNDERLINE = '\033[4m' + RESET = '\033[0m' + + def config_check(args): - def eprint(x: str): - print(x, file=sys.stderr) - - def indent(x: str) -> str: - return ''.join(' ' + l for l in x.splitlines(keepends=True)) - - def info(x: str): - eprint('✅ ' + x) - - def error(x: str): - eprint('❌ ' + x) - - def warning(x: str): - eprint('❗ ' + x) # todo yellow? - try: import my.config as cfg except Exception as e: error("failed to import the config") - tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__)) - sys.stderr.write(indent(tb)) + tb(e) sys.exit(1) info(f"config file: {cfg.__file__}") @@ -80,12 +98,35 @@ def config_check(args): sys.stderr.write(indent(mres.stdout.decode('utf8'))) +def modules_check(args): + verbose = args.verbose + + from .util import get_modules + for m in get_modules(): + try: + importlib.import_module(m) + except Exception as e: + # todo more specific command? + vw = '' if verbose else '; pass --verbose to print more information' + warning(f'{color.RED}FAIL{color.RESET}: {m:<30} loading failed{vw}') + if verbose: + tb(e) + + else: + info(f'{color.GREEN}OK{color.RESET} : {m:<30}') + + +def list_modules(args): + # todo with docs/etc? + from .util import get_modules + for m in get_modules(): + print(f'- {m}') + + +# todo check that it finds private modules too? def doctor(args): config_check(args) - - -def hello(args): - print('Hello') + modules_check(args) def parser(): @@ -96,10 +137,8 @@ Tool for HPI. Work in progress, will be used for config management, troubleshooting & introspection ''') sp = p.add_subparsers(dest='mode') - hp = sp.add_parser(Modes.HELLO , help='TODO just a stub, remove later') - hp.set_defaults(func=hello) - dp = sp.add_parser(Modes.DOCTOR, help='Run various checks') + dp.add_argument('--verbose', action='store_true', help='Print more diagnosic infomration') dp.set_defaults(func=doctor) cp = sp.add_parser(Modes.CONFIG, help='Work with configuration') @@ -108,6 +147,9 @@ Work in progress, will be used for config management, troubleshooting & introspe # ccp = scp.add_parser('check', help='Check config') # ccp.set_defaults(func=config_check) + mp = sp.add_parser(Modes.MODULES, help='List available modules') + mp.set_defaults(func=list_modules) + return p diff --git a/my/core/util.py b/my/core/util.py new file mode 100644 index 0000000..f76b3be --- /dev/null +++ b/my/core/util.py @@ -0,0 +1,82 @@ +from pathlib import Path +from itertools import chain +import os +import re +import pkgutil +from typing import List + +# TODO reuse in readme/blog post +# borrowed from https://github.com/sanitizers/octomachinery/blob/24288774d6dcf977c5033ae11311dbff89394c89/tests/circular_imports_test.py#L22-L55 +def _find_all_importables(pkg): + """Find all importables in the project. + Return them in order. + """ + return sorted( + set( + chain.from_iterable( + _discover_path_importables(Path(p), pkg.__name__) + for p in pkg.__path__ + ), + ), + ) + + +def _discover_path_importables(pkg_pth, pkg_name): + """Yield all importables under a given path and package.""" + for dir_path, _d, file_names in os.walk(pkg_pth): + pkg_dir_path = Path(dir_path) + + if pkg_dir_path.parts[-1] == '__pycache__': + continue + + if all(Path(_).suffix != '.py' for _ in file_names): + continue + + rel_pt = pkg_dir_path.relative_to(pkg_pth) + pkg_pref = '.'.join((pkg_name, ) + rel_pt.parts) + + + yield from ( + pkg_path + for _, pkg_path, _ in pkgutil.walk_packages( + (str(pkg_dir_path), ), prefix=f'{pkg_pref}.', + ) + ) + + +# todo need a better way to mark module as 'interface' +def ignored(m: str): + excluded = [ + 'kython.*', + 'mycfg_stub', + 'common', + 'error', + 'cfg', + 'core.*', + 'config.*', + 'jawbone.plots', + 'emfit.plot', + + # todo think about these... + # 'google.takeout.paths', + 'bluemaestro.check', + 'location.__main__', + 'photos.utils', + 'books', + 'coding', + 'media', + 'reading', + '_rss', + 'twitter.common', + 'rss.common', + 'lastfm.fill_influxdb', + ] + exs = '|'.join(excluded) + return re.match(f'^my.({exs})$', m) + + +def get_modules() -> List[str]: + import my as pkg # todo not sure? + importables = _find_all_importables(pkg) + public = [x for x in importables if not ignored(x)] + return public From 7bd7cc922876c458544ccd57e1f64e19e75cfadd Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 25 May 2020 11:27:54 +0100 Subject: [PATCH 5/6] cli: integrate with stats reported by the modules --- my/core/__main__.py | 20 +++++++++++++++++--- my/emfit/__init__.py | 6 ++++++ my/foursquare.py | 7 +++++++ my/hypothesis.py | 10 ++++++++++ my/rss/feedbin.py | 7 +++++++ tests/foursquare.py | 1 + tox.ini | 2 +- 7 files changed, 49 insertions(+), 4 deletions(-) diff --git a/my/core/__main__.py b/my/core/__main__.py index 2de9537..c80bae3 100644 --- a/my/core/__main__.py +++ b/my/core/__main__.py @@ -100,20 +100,34 @@ def config_check(args): def modules_check(args): verbose = args.verbose + vw = '' if verbose else '; pass --verbose to print more information' from .util import get_modules for m in get_modules(): try: - importlib.import_module(m) + mod = importlib.import_module(m) except Exception as e: # todo more specific command? - vw = '' if verbose else '; pass --verbose to print more information' warning(f'{color.RED}FAIL{color.RESET}: {m:<30} loading failed{vw}') if verbose: tb(e) + continue + info(f'{color.GREEN}OK{color.RESET} : {m:<30}') + stats = getattr(mod, 'stats', None) + if stats is None: + continue + try: + res = stats() + except Exception as ee: + warning(f' - {color.RED}stats:{color.RESET} computing failed{vw}') + if verbose: + tb(ee) else: - info(f'{color.GREEN}OK{color.RESET} : {m:<30}') + info(f' - stats: {res}') + + + def list_modules(args): diff --git a/my/emfit/__init__.py b/my/emfit/__init__.py index 2b8f5a8..93ea337 100755 --- a/my/emfit/__init__.py +++ b/my/emfit/__init__.py @@ -322,6 +322,12 @@ def by_night() -> Dict[date, Emfit]: return res +def stats(): + return { + 'nights': len(by_night()), + } + + def main(): for k, v in by_night().items(): print(k, v.start, v.end) diff --git a/my/foursquare.py b/my/foursquare.py index ed55a24..a722b3a 100755 --- a/my/foursquare.py +++ b/my/foursquare.py @@ -90,3 +90,10 @@ def get_cid_map(bfile: str): def print_checkins(): print(get_checkins()) + + +def stats(): + from more_itertools import ilen + return { + 'checkins': ilen(get_checkins()), + } diff --git a/my/hypothesis.py b/my/hypothesis.py index bacc51e..d3d95c2 100644 --- a/my/hypothesis.py +++ b/my/hypothesis.py @@ -69,6 +69,16 @@ def pages() -> List[Res[Page]]: return sort_res_by(_dal().pages(), key=lambda h: h.created) +# todo not public api yet +def stats(): + # todo add 'last date' checks et + return { + # todo ilen + 'highlights': len(highlights()), + 'pages' : len(pages()), + } + + def _main(): for page in get_pages(): print(page) diff --git a/my/rss/feedbin.py b/my/rss/feedbin.py index 5a2f117..4cd1b8d 100644 --- a/my/rss/feedbin.py +++ b/my/rss/feedbin.py @@ -40,3 +40,10 @@ def states() -> Iterable[SubscriptionState]: dt = isoparse(dts) subs = parse_file(f) yield dt, subs + + +def stats(): + from more_itertools import ilen, last + return { + 'subscriptions': ilen(last(states())[1]) + } diff --git a/tests/foursquare.py b/tests/foursquare.py index 040f56c..ae9612e 100644 --- a/tests/foursquare.py +++ b/tests/foursquare.py @@ -1,6 +1,7 @@ from my.foursquare import get_checkins def test_checkins(): + # todo reuse stats? checkins = get_checkins() assert len(checkins) > 100 assert any('Victoria Park' in c.summary for c in checkins) diff --git a/tox.ini b/tox.ini index ba5ac70..9a724a3 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ commands = # TODO add; once I figure out porg depdencency?? tests/config.py # TODO run demo.py? just make sure with_my is a bit cleverer? # TODO e.g. under CI, rely on installing - hpi hello + hpi modules [testenv:demo] From e351c8ba49c9932c0b6edf9b88f38d9ba8c6c6cd Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 25 May 2020 12:21:31 +0100 Subject: [PATCH 6/6] cli: add 'config init' command --- doc/SETUP.org | 8 ++++++-- my/core/__main__.py | 46 ++++++++++++++++++++++++++++++--------------- my/core/init.py | 15 +-------------- my/core/preinit.py | 13 +++++++++++++ 4 files changed, 51 insertions(+), 31 deletions(-) create mode 100644 my/core/preinit.py diff --git a/doc/SETUP.org b/doc/SETUP.org index 6eb26c3..5d9f0f2 100644 --- a/doc/SETUP.org +++ b/doc/SETUP.org @@ -98,6 +98,7 @@ They aren't necessary, but will improve your experience. At the moment these are - [[https://github.com/karlicoss/cachew][cachew]]: automatic caching library, which can greatly speedup data access - [[https://github.com/metachris/logzero][logzero]]: a nice logging library, supporting colors +- [[https://github.com/python/mypy][mypy]]: mypy is used for checking configs and troubleshooting * Setting up modules This is an *optional step* as few modules work without extra setup. @@ -111,7 +112,6 @@ elaborating on some technical rationales behind the current configuration system ** private configuration (=my.config=) # TODO write about dynamic configuration # TODO add a command to edit config?? e.g. HPI config edit -# HPI doctor? If you're not planning to use private configuration (some modules don't need it) you can skip straight to the next step. Still, I'd recommend you to read anyway. The configuration contains paths to the data on your disks, links to external repositories, etc. @@ -119,7 +119,11 @@ The config is simply a *python package* (named =my.config=), expected to be in = Since it's a Python package, generally it's very *flexible* and there are many ways to set it up. -- *The simplest and the very minimum* you need is =~/.config/my/my/config.py=. For example: +- *The simplest way* + + After installing HPI, run =hpi config init=. + + This will create an empty config file for you (usually, in =~/.config/my=), which you can edit. Example configuration: #+begin_src python import pytz # yes, you can use any Python stuff in the config diff --git a/my/core/__main__.py b/my/core/__main__.py index c80bae3..1159907 100644 --- a/my/core/__main__.py +++ b/my/core/__main__.py @@ -9,14 +9,9 @@ from . import LazyLogger log = LazyLogger('HPI cli') -class Modes: - CONFIG = 'config' - DOCTOR = 'doctor' - MODULES = 'modules' - def run_mypy(pkg): - from .init import get_mycfg_dir + from .preinit import get_mycfg_dir mycfg_dir = get_mycfg_dir() # todo ugh. not sure how to extract it from pkg? @@ -73,6 +68,27 @@ class color: RESET = '\033[0m' +def config_create(args): + from .preinit import get_mycfg_dir + mycfg_dir = get_mycfg_dir() + + created = False + if not mycfg_dir.exists(): + # todo not sure about the layout... should I use my/config.py instead? + my_config = mycfg_dir / 'my' / 'config' / '__init__.py' + + my_config.parent.mkdir(parents=True) + my_config.touch() + info(f'created empty config: {my_config}') + created = True + else: + error(f"config directory '{mycfg_dir}' already exists, skipping creation") + + config_check(args) + if not created: + sys.exit(1) + + def config_check(args): try: import my.config as cfg @@ -127,9 +143,6 @@ def modules_check(args): info(f' - stats: {res}') - - - def list_modules(args): # todo with docs/etc? from .util import get_modules @@ -151,17 +164,20 @@ Tool for HPI. Work in progress, will be used for config management, troubleshooting & introspection ''') sp = p.add_subparsers(dest='mode') - dp = sp.add_parser(Modes.DOCTOR, help='Run various checks') + dp = sp.add_parser('doctor', help='Run various checks') dp.add_argument('--verbose', action='store_true', help='Print more diagnosic infomration') dp.set_defaults(func=doctor) - cp = sp.add_parser(Modes.CONFIG, help='Work with configuration') + cp = sp.add_parser('config', help='Work with configuration') scp = cp.add_subparsers(dest='mode') - # if True: - # ccp = scp.add_parser('check', help='Check config') - # ccp.set_defaults(func=config_check) + if True: + ccp = scp.add_parser('check', help='Check config') + ccp.set_defaults(func=config_check) - mp = sp.add_parser(Modes.MODULES, help='List available modules') + icp = scp.add_parser('create', help='Create user config') + icp.set_defaults(func=config_create) + + mp = sp.add_parser('modules', help='List available modules') mp.set_defaults(func=list_modules) return p diff --git a/my/core/init.py b/my/core/init.py index d2c7195..158a311 100644 --- a/my/core/init.py +++ b/my/core/init.py @@ -24,26 +24,13 @@ def assign_module(parent: str, name: str, module: ModuleType) -> None: del ModuleType -def get_mycfg_dir(): - import appdirs # type: ignore[import] - from pathlib import Path - import os - # not sure if that's necessary, i.e. could rely on PYTHONPATH instead - # on the other hand, by using MY_CONFIG we are guaranteed to load it from the desired path? - mvar = os.environ.get('MY_CONFIG') - if mvar is not None: - mycfg_dir = Path(mvar) - else: - mycfg_dir = Path(appdirs.user_config_dir('my')) - return mycfg_dir - - # separate function to present namespace pollution def setup_config() -> None: import sys import warnings from typing import Optional + from .preinit import get_mycfg_dir mycfg_dir = get_mycfg_dir() if not mycfg_dir.exists(): diff --git a/my/core/preinit.py b/my/core/preinit.py new file mode 100644 index 0000000..c05ee40 --- /dev/null +++ b/my/core/preinit.py @@ -0,0 +1,13 @@ +from pathlib import Path + +def get_mycfg_dir() -> Path: + import appdirs # type: ignore[import] + import os + # not sure if that's necessary, i.e. could rely on PYTHONPATH instead + # on the other hand, by using MY_CONFIG we are guaranteed to load it from the desired path? + mvar = os.environ.get('MY_CONFIG') + if mvar is not None: + mycfg_dir = Path(mvar) + else: + mycfg_dir = Path(appdirs.user_config_dir('my')) + return mycfg_dir