From 70c801f692f01f85f0eb6fc08d8a35563589e0e2 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 28 Sep 2020 18:53:58 +0200 Subject: [PATCH] core: add 'core' config section, add disabled_modules/enabled_modules configs, use them for hpi modules and hpi doctor --- my/core/__main__.py | 39 +++++++++++++++++------- my/core/cachew.py | 9 +++--- my/core/cfg.py | 1 + my/core/core_config.py | 67 ++++++++++++++++++++++++++++++++++++++++++ my/core/init.py | 2 +- my/core/util.py | 38 ++++++++++++------------ 6 files changed, 121 insertions(+), 35 deletions(-) create mode 100644 my/core/core_config.py diff --git a/my/core/__main__.py b/my/core/__main__.py index a7ba6d4..e399c11 100644 --- a/my/core/__main__.py +++ b/my/core/__main__.py @@ -2,7 +2,7 @@ import os from pathlib import Path import sys from subprocess import check_call, run, PIPE -from typing import Optional, Sequence +from typing import Optional, Sequence, Iterable import importlib import traceback @@ -77,6 +77,7 @@ def tb(e): sys.stderr.write(indent(tb)) +# todo not gonna work on Windows... perhaps make it optional and use colorama/termcolor? (similar to core.warnings) class color: BLACK = '\033[30m' RED = '\033[31m' @@ -111,6 +112,7 @@ def config_create(args): sys.exit(1) +# TODO return the config as a result? def config_check(args): try: import my.config as cfg @@ -138,17 +140,26 @@ def modules_check(args): module: Optional[str] = args.module vw = '' if verbose else '; pass --verbose to print more information' + mods: Iterable[str] if module is None: - from .util import get_modules - modules = get_modules() + from .util import modules + mods = modules() else: - modules = [module] - for m in modules: + mods = [module] + + from .core_config import config + # todo add a --all argument to disregard is_active check? + for m in mods: + active = config.is_module_active(m) + if not active: + eprint(f'🔲 {color.YELLOW}SKIP{color.RESET}: {m:<30} module disabled in config') + continue + try: mod = importlib.import_module(m) except Exception as e: # todo more specific command? - warning(f'{color.RED}FAIL{color.RESET}: {m:<30} loading failed{vw}') + error(f'{color.RED}FAIL{color.RESET}: {m:<30} loading failed{vw}') if verbose: tb(e) continue @@ -171,11 +182,17 @@ def modules_check(args): info(f' - stats: {res}') -def list_modules(args): - # todo with docs/etc? - from .util import get_modules - for m in get_modules(): - print(f'- {m}') +def list_modules(args) -> None: + # todo add a --sort argument? + from .core_config import config + + # todo add an active_modules() method? would be useful for doctor? + from .util import modules + for m in modules(): + active = config.is_module_active(m) + # todo maybe reorder? (e.g. enabled first/last)? or/and color? + # todo maybe use [off] / [ON] so it's easier to distinguish visually? + print(f'- {m:50}' + ('' if active else f' {color.YELLOW}[disabled]{color.RESET}')) # todo check that it finds private modules too? diff --git a/my/core/cachew.py b/my/core/cachew.py index b8eecf4..27a35fb 100644 --- a/my/core/cachew.py +++ b/my/core/cachew.py @@ -40,12 +40,11 @@ def cache_dir() -> Path: class config: cache_dir = '/your/custom/cache/path' ''' - import my.config as C - common_config = getattr(C, 'common', object()) - # TODO if attr is set _and_ it's none, disable cache? - cdir = getattr(common_config, 'cache_dir', None) + from .core_config import config + cdir = config.cache_dir if cdir is None: - # TODO fallback to default cachew dir instead? + # TODO handle this in core_config.py + # TODO fallback to default cachew dir instead? or appdirs cache return Path('/var/tmp/cachew') else: return Path(cdir) diff --git a/my/core/cfg.py b/my/core/cfg.py index 344cfa9..c20ed7d 100644 --- a/my/core/cfg.py +++ b/my/core/cfg.py @@ -6,6 +6,7 @@ C = TypeVar('C') # todo not sure about it, could be overthinking... # but short enough to change later +# TODO document why it's necessary? def make_config(cls: Type[C], migration: Callable[[Attrs], Attrs]=lambda x: x) -> C: props = dict(vars(cls.__base__)) props = migration(props) diff --git a/my/core/core_config.py b/my/core/core_config.py new file mode 100644 index 0000000..5c7843d --- /dev/null +++ b/my/core/core_config.py @@ -0,0 +1,67 @@ +''' +Bindings for the 'core' HPI configuration +''' +import re +from typing import Sequence, Optional + +from .common import PathIsh + +try: + # FIXME support legacy 'common'? + from my.config import core as user_config # type: ignore[attr-defined] +except Exception as e: + # make it defensive, because it's pretty commonly used and would be annoying if it breaks hpi doctor etc. + # this way it'll at least use the defaults + # TODO add high warning + user_config = object # type: ignore[assignment, misc] + + +from dataclasses import dataclass +@dataclass +class Config(user_config): + # TODO if attr is set _and_ it's none, disable cache? + # todo or empty string? + # I guess flip the switch at some point when I'm confident in cachew + cache_dir: Optional[PathIsh] = None # FIXME use appdirs cache dir or something + + # list of regexes/globs + # None means 'rely on disabled_modules' + enabled_modules : Optional[Sequence[str]] = None + + # list of regexes/globs + # None means 'rely on enabled_modules' + disabled_modules: Optional[Sequence[str]] = None + + + def is_module_active(self, module: str) -> bool: + # todo might be nice to return the 'reason' too? e.g. which option has matched + should_enable = None + should_disable = None + def matches(specs: Sequence[str]) -> bool: + for spec in specs: + # not sure because . (packages separate) matches anything, but I guess unlikely to clash + if re.match(spec, module): + return True + return False + + enabled = self.enabled_modules + disabled = self.disabled_modules + if enabled is None: + if disabled is None: + # by default, enable everything? not sure + return True + else: + # only disable the specified modules + return not matches(disabled) + else: + if disabled is None: + # only enabled the specifid modules + return matches(enabled) + else: + # ok, this means the config is inconsistent. better fallback onto the 'enable everything', then the user will notice? + # todo add medium warning? + return True + + +from .cfg import make_config +config = make_config(Config) diff --git a/my/core/init.py b/my/core/init.py index 158a311..55224a8 100644 --- a/my/core/init.py +++ b/my/core/init.py @@ -35,7 +35,7 @@ def setup_config() -> None: if not mycfg_dir.exists(): warnings.warn(f""" -'my.config' package isn't found! (expected at {mycfg_dir}). This is likely to result in issues. +'my.config' package isn't found! (expected at '{mycfg_dir}'). This is likely to result in issues. See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-modules for more info. """.strip()) return diff --git a/my/core/util.py b/my/core/util.py index f76b3be..e1a9e8b 100644 --- a/my/core/util.py +++ b/my/core/util.py @@ -3,27 +3,24 @@ from itertools import chain import os import re import pkgutil -from typing import List +from typing import List, Iterable # TODO reuse in readme/blog post # borrowed from https://github.com/sanitizers/octomachinery/blob/24288774d6dcf977c5033ae11311dbff89394c89/tests/circular_imports_test.py#L22-L55 -def _find_all_importables(pkg): - """Find all importables in the project. - Return them in order. - """ - return sorted( - set( - chain.from_iterable( - _discover_path_importables(Path(p), pkg.__name__) - for p in pkg.__path__ - ), - ), +def _iter_all_importables(pkg): + yield from chain.from_iterable( + _discover_path_importables(Path(p), pkg.__name__) + for p in pkg.__path__ ) def _discover_path_importables(pkg_pth, pkg_name): """Yield all importables under a given path and package.""" - for dir_path, _d, file_names in os.walk(pkg_pth): + for dir_path, dirs, file_names in os.walk(pkg_pth): + file_names.sort() + # NOTE: sorting dirs in place is intended, it's the way you're supposed to do it with os.walk + dirs.sort() + pkg_dir_path = Path(dir_path) if pkg_dir_path.parts[-1] == '__pycache__': @@ -36,6 +33,7 @@ def _discover_path_importables(pkg_pth, pkg_name): pkg_pref = '.'.join((pkg_name, ) + rel_pt.parts) + # TODO might need to make it defensive and yield Exception (otherwise hpi doctor might fail for no good reason) yield from ( pkg_path for _, pkg_path, _ in pkgutil.walk_packages( @@ -44,7 +42,7 @@ def _discover_path_importables(pkg_pth, pkg_name): ) -# todo need a better way to mark module as 'interface' +# TODO marking hpi modules or unmarking non-modules? not sure what's worse def ignored(m: str): excluded = [ 'kython.*', @@ -75,8 +73,12 @@ def ignored(m: str): return re.match(f'^my.({exs})$', m) -def get_modules() -> List[str]: +def modules() -> Iterable[str]: import my as pkg # todo not sure? - importables = _find_all_importables(pkg) - public = [x for x in importables if not ignored(x)] - return public + for x in _iter_all_importables(pkg): + if not ignored(x): + yield x + + +def get_modules() -> List[str]: + return list(modules())