core: add 'core' config section, add disabled_modules/enabled_modules configs, use them for hpi modules and hpi doctor

my/core/__main__.py

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 import sys
 from subprocess import check_call, run, PIPE
-from typing import Optional, Sequence
+from typing import Optional, Sequence, Iterable
 import importlib
 import traceback
 
@@ -77,6 +77,7 @@ def tb(e):
     sys.stderr.write(indent(tb))
 
 
+# todo not gonna work on Windows... perhaps make it optional and use colorama/termcolor? (similar to core.warnings)
 class color:
     BLACK = '\033[30m'
     RED = '\033[31m'
@@ -111,6 +112,7 @@ def config_create(args):
         sys.exit(1)
 
 
+# TODO return the config as a result?
 def config_check(args):
     try:
         import my.config as cfg
@@ -138,17 +140,26 @@ def modules_check(args):
     module: Optional[str] = args.module
     vw = '' if verbose else '; pass --verbose to print more information'
 
+    mods: Iterable[str]
     if module is None:
-        from .util import get_modules
+        from .util import modules
-        modules = get_modules()
+        mods = modules()
     else:
-        modules = [module]
+        mods = [module]
-    for m in modules:
+
+    from .core_config import config
+    # todo add a --all argument to disregard is_active check?
+    for m in mods:
+        active = config.is_module_active(m)
+        if not active:
+            eprint(f'🔲 {color.YELLOW}SKIP{color.RESET}: {m:<30} module disabled in config')
+            continue
+
         try:
             mod = importlib.import_module(m)
         except Exception as e:
             # todo more specific command?
-            warning(f'{color.RED}FAIL{color.RESET}: {m:<30} loading failed{vw}')
+            error(f'{color.RED}FAIL{color.RESET}: {m:<30} loading failed{vw}')
             if verbose:
                 tb(e)
             continue
@@ -171,11 +182,17 @@ def modules_check(args):
             info(f'    - stats: {res}')
 
 
-def list_modules(args):
+def list_modules(args) -> None:
-    # todo with docs/etc?
+    # todo add a --sort argument?
-    from .util import get_modules
+    from .core_config import config
-    for m in get_modules():
+
-        print(f'- {m}')
+    # todo add an active_modules() method? would be useful for doctor?
+    from .util import modules
+    for m in modules():
+        active = config.is_module_active(m)
+        # todo maybe reorder? (e.g. enabled first/last)? or/and color?
+        # todo maybe use [off] / [ON] so it's easier to distinguish visually?
+        print(f'- {m:50}' + ('' if active else f' {color.YELLOW}[disabled]{color.RESET}'))
 
 
 # todo check that it finds private modules too?

my/core/cachew.py

@@ -40,12 +40,11 @@ def cache_dir() -> Path:
     class config:
         cache_dir = '/your/custom/cache/path'
     '''
-    import my.config as C
+    from .core_config import config
-    common_config = getattr(C, 'common', object())
+    cdir = config.cache_dir
-    # TODO if attr is set _and_ it's none, disable cache?
-    cdir = getattr(common_config, 'cache_dir', None)
     if cdir is None:
-        # TODO fallback to default cachew dir instead?
+        # TODO handle this in core_config.py
+        # TODO fallback to default cachew dir instead? or appdirs cache
         return Path('/var/tmp/cachew')
     else:
         return Path(cdir)

my/core/cfg.py

@@ -6,6 +6,7 @@ C = TypeVar('C')
 
 # todo not sure about it, could be overthinking...
 # but short enough to change later
+# TODO document why it's necessary?
 def make_config(cls: Type[C], migration: Callable[[Attrs], Attrs]=lambda x: x) -> C:
     props = dict(vars(cls.__base__))
     props = migration(props)

my/core/core_config.py

@@ -0,0 +1,67 @@
+'''
+Bindings for the 'core' HPI configuration
+'''
+import re
+from typing import Sequence, Optional
+
+from .common import PathIsh
+
+try:
+    # FIXME support legacy 'common'?
+    from my.config import core as user_config # type: ignore[attr-defined]
+except Exception as e:
+    # make it defensive, because it's pretty commonly used and would be annoying if it breaks hpi doctor etc.
+    # this way it'll at least use the defaults
+    # TODO add high warning
+    user_config = object # type: ignore[assignment, misc]
+
+
+from dataclasses import dataclass
+@dataclass
+class Config(user_config):
+    # TODO if attr is set _and_ it's none, disable cache?
+    # todo or empty string?
+    # I guess flip the switch at some point when I'm confident in cachew
+    cache_dir: Optional[PathIsh] = None # FIXME use appdirs cache dir or something
+
+    # list of regexes/globs
+    # None means 'rely on disabled_modules'
+    enabled_modules : Optional[Sequence[str]] = None
+
+    # list of regexes/globs
+    # None means 'rely on enabled_modules'
+    disabled_modules: Optional[Sequence[str]] = None
+
+
+    def is_module_active(self, module: str) -> bool:
+        # todo might be nice to return the 'reason' too? e.g. which option has matched
+        should_enable  = None
+        should_disable = None
+        def matches(specs: Sequence[str]) -> bool:
+            for spec in specs:
+                # not sure because . (packages separate) matches anything, but I guess unlikely to clash
+                if re.match(spec, module):
+                    return True
+            return False
+
+        enabled  = self.enabled_modules
+        disabled = self.disabled_modules
+        if enabled is None:
+            if disabled is None:
+                # by default, enable everything? not sure
+                return True
+            else:
+                # only disable the specified modules
+                return not matches(disabled)
+        else:
+            if disabled is None:
+                # only enabled the specifid modules
+                return matches(enabled)
+            else:
+                # ok, this means the config is inconsistent. better fallback onto the 'enable everything', then the user will notice?
+                # todo add medium warning?
+                return True
+
+
+from .cfg import make_config
+config = make_config(Config)

my/core/init.py

@@ -35,7 +35,7 @@ def setup_config() -> None:
 
     if not mycfg_dir.exists():
         warnings.warn(f"""
-'my.config' package isn't found! (expected at {mycfg_dir}). This is likely to result in issues.
+'my.config' package isn't found! (expected at '{mycfg_dir}'). This is likely to result in issues.
 See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-modules for more info.
 """.strip())
         return

my/core/util.py

@@ -3,27 +3,24 @@ from itertools import chain
 import os
 import re
 import pkgutil
-from typing import List
+from typing import List, Iterable
 
 # TODO reuse in readme/blog post
 # borrowed from https://github.com/sanitizers/octomachinery/blob/24288774d6dcf977c5033ae11311dbff89394c89/tests/circular_imports_test.py#L22-L55
-def _find_all_importables(pkg):
+def _iter_all_importables(pkg):
-    """Find all importables in the project.
+    yield from chain.from_iterable(
-    Return them in order.
-    """
-    return sorted(
-        set(
-            chain.from_iterable(
         _discover_path_importables(Path(p), pkg.__name__)
         for p in pkg.__path__
-            ),
-        ),
     )
 
 
 def _discover_path_importables(pkg_pth, pkg_name):
     """Yield all importables under a given path and package."""
-    for dir_path, _d, file_names in os.walk(pkg_pth):
+    for dir_path, dirs, file_names in os.walk(pkg_pth):
+        file_names.sort()
+        # NOTE: sorting dirs in place is intended, it's the way you're supposed to do it with os.walk
+        dirs.sort()
+
         pkg_dir_path = Path(dir_path)
 
         if pkg_dir_path.parts[-1] == '__pycache__':
@@ -36,6 +33,7 @@ def _discover_path_importables(pkg_pth, pkg_name):
         pkg_pref = '.'.join((pkg_name, ) + rel_pt.parts)
 
 
+        # TODO might need to make it defensive and yield Exception (otherwise hpi doctor might fail for no good reason)
         yield from (
             pkg_path
             for _, pkg_path, _ in pkgutil.walk_packages(
@@ -44,7 +42,7 @@ def _discover_path_importables(pkg_pth, pkg_name):
         )
 
 
-# todo need a better way to mark module as 'interface'
+# TODO marking hpi modules or unmarking non-modules? not sure what's worse
 def ignored(m: str):
     excluded = [
         'kython.*',
@@ -75,8 +73,12 @@ def ignored(m: str):
     return re.match(f'^my.({exs})$', m)
 
 
-def get_modules() -> List[str]:
+def modules() -> Iterable[str]:
     import my as pkg # todo not sure?
-    importables = _find_all_importables(pkg)
+    for x in _iter_all_importables(pkg):
-    public = [x for x in importables if not ignored(x)]
+        if not ignored(x):
-    return public
+            yield x
+
+
+def get_modules() -> List[str]:
+    return list(modules())