core/cli: some enhacements for frendlier errors/default config

see https://github.com/karlicoss/HPI/issues/110

my/config.py

@@ -1,4 +1,7 @@
 '''
+NOTE: you shouldn't modify this file.
+You probably want to edit your personal config (check via 'hpi config check' or create with 'hpi config create').
+
 See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules for info on creating your own config
 
 This file is used for:

my/core/__main__.py

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 import sys
 from subprocess import check_call, run, PIPE
-from typing import Optional, Sequence, Iterable
+from typing import Optional, Sequence, Iterable, List
 import importlib
 import traceback
 
@@ -94,7 +94,7 @@ class color:
     RESET = '\033[0m'
 
 
-def config_create(args):
+def config_create(args) -> None:
     from .preinit import get_mycfg_dir
     mycfg_dir = get_mycfg_dir()
 
@@ -104,23 +104,54 @@ def config_create(args):
         my_config = mycfg_dir / 'my' / 'config' / '__init__.py'
 
         my_config.parent.mkdir(parents=True)
-        my_config.touch()
+        my_config.write_text('''
+### HPI personal config
+## see
+# https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules
+# https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org
+## for some help on writing your own config
+
+# to quickly check your config, run:
+# hpi config check
+
+# to quickly check a specific module setup, run hpi doctor <module>, e.g.:
+# hpi doctor my.reddit
+
+### useful default imports
+from my.core import Paths, PathIsh, get_files
+###
+
+# most of your configs will look like this:
+class example:
+    export_path: Paths = '/home/user/data/example_data_dir/'
+
+### you can insert your own configuration below
+### but feel free to delete the stuff above if you don't need ti
+'''.lstrip())
         info(f'created empty config: {my_config}')
         created = True
     else:
         error(f"config directory '{mycfg_dir}' already exists, skipping creation")
 
-    config_check(args)
+    check_passed = config_ok(args)
-    if not created:
+    if not created or not check_passed:
         sys.exit(1)
 
 
+def config_check_cli(args) -> None:
+    ok = config_ok(args)
+    sys.exit(0 if ok else False)
+
+
 # TODO return the config as a result?
-def config_check(args):
+def config_ok(args) -> bool:
+    errors: List[Exception] = []
+
     import my
     try:
         paths = my.__path__._path # type: ignore[attr-defined]
     except Exception as e:
+        errors.append(e)
         error('failed to determine module import path')
         tb(e)
     else:
@@ -129,9 +160,12 @@ def config_check(args):
     try:
         import my.config as cfg
     except Exception as e:
+        errors.append(e)
         error("failed to import the config")
         tb(e)
-        sys.exit(1) # todo yield exception here? so it doesn't fail immediately..
+        # todo yield exception here? so it doesn't fail immediately..
+        # I guess it's fairly critical and worth exiting immediately
+        sys.exit(1)
 
     cfg_path = cfg.__file__# todo might be better to use __path__?
     info(f"config file: {cfg_path}")
@@ -141,22 +175,40 @@ def config_check(args):
         core_pkg_path = str(Path(core.__path__[0]).parent) # type: ignore[attr-defined]
         if cfg_path.startswith(core_pkg_path):
             error(f'''
-            Seems that the default config is used ({cfg_path}).
+Seems that the stub config is used ({cfg_path}). This is likely not going to work.
-            See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules for more information
+See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules for more information
-            '''.strip())
+'''.strip())
+            errors.append(RuntimeError('bad config path'))
     except Exception as e:
+        errors.append(e)
         tb(e)
 
+    # todo for some reason compileall.compile_file always returns true??
+    try:
+        cmd = [sys.executable, '-m', 'compileall', str(cfg_path)]
+        check_call(cmd)
+        info('syntax check: ' + ' '.join(cmd))
+    except Exception as e:
+        errors.append(e)
+
     mres = run_mypy(cfg)
-    if mres is None: # no mypy
+    if mres is not None: # has mypy
-        return
+        rc = mres.returncode
-    rc = mres.returncode
+        if rc == 0:
-    if rc == 0:
+            info('mypy check: success')
-        info('mypy config check: success')
+        else:
+            error('mypy check: failed')
+            errors.append(RuntimeError('mypy failed'))
+            sys.stderr.write(indent(mres.stderr.decode('utf8')))
+        sys.stderr.write(indent(mres.stdout.decode('utf8')))
+
+    if len(errors) > 0:
+        error(f'config check: {len(errors)} errors')
+        return False
     else:
-        error('mypy config check: failed')
+        # note: shouldn't exit here, might run something else
-        sys.stderr.write(indent(mres.stderr.decode('utf8')))
+        info('config check: success!')
-    sys.stderr.write(indent(mres.stdout.decode('utf8')))
+        return True
 
 
 def _modules(all=False):
@@ -171,7 +223,7 @@ def _modules(all=False):
         warning(f'Skipped {len(skipped)} modules: {skipped}. Pass --all if you want to see them.')
 
 
-def modules_check(args):
+def modules_check(args) -> None:
     verbose: bool         = args.verbose
     quick:   bool         = args.quick
     module: Optional[str] = args.module
@@ -244,7 +296,7 @@ def list_modules(args) -> None:
         print(f'{pre} {m:50}{suf}')
 
 
-def tabulate_warnings():
+def tabulate_warnings() -> None:
     '''
     Helper to avoid visual noise in hpi modules/doctor
     '''
@@ -258,8 +310,9 @@ def tabulate_warnings():
 
 
 # todo check that it finds private modules too?
-def doctor(args):
+def doctor(args) -> None:
-    config_check(args)
+    ok = config_ok(args)
+    # TODO propagate ok status up?
     modules_check(args)
 
 
@@ -282,7 +335,7 @@ Work in progress, will be used for config management, troubleshooting & introspe
     scp = cp.add_subparsers(dest='mode')
     if True:
         ccp = scp.add_parser('check', help='Check config')
-        ccp.set_defaults(func=config_check)
+        ccp.set_defaults(func=config_check_cli)
 
         icp = scp.add_parser('create', help='Create user config')
         icp.set_defaults(func=config_create)

my/core/common.py

@@ -5,6 +5,7 @@ import functools
 import types
 from typing import Union, Callable, Dict, Iterable, TypeVar, Sequence, List, Optional, Any, cast, Tuple, TYPE_CHECKING
 import warnings
+from . import warnings as core_warnings
 
 # some helper functions
 PathIsh = Union[Path, str]
@@ -123,7 +124,6 @@ def _is_compressed(p: Path) -> bool:
     return p.suffix in {'.xz', '.lz4', '.zstd'}
 
 
-# TODO support '' for empty path
 DEFAULT_GLOB = '*'
 def get_files(
         pp: Paths,
@@ -168,6 +168,7 @@ def get_files(
                 paths.extend(map(Path, do_glob(ss)))
             else:
                 if not src.is_file():
+                    # todo not sure, might be race condition?
                     raise RuntimeError(f"Expected '{src}' to exist")
                 # todo assert matches glob??
                 paths.append(src)
@@ -177,9 +178,11 @@ def get_files(
 
     if len(paths) == 0:
         # todo make it conditionally defensive based on some global settings
-        # TODO not sure about using warnings module for this
+        core_warnings.high(f'''
+{caller()}: no paths were matched against {pp}. This might result in missing data. Likely, the directory you passed is empty.
+'''.strip())
+        # traceback is useful to figure out what config caused it?
         import traceback
-        warnings.warn(f'{caller()}: no paths were matched against {paths}. This might result in missing data.')
         traceback.print_stack()
 
     if guess_compression: