fz0x1/HPI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

core: some cleanup for core/init and doctor; fix issue with compileall

Browse source
This commit is contained in:
Dima Gerasimov 2022-06-02 10:11:00 +01:00 • committed by karlicoss
parent 9461df6aa5
commit 186f561018
3 changed files with 71 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
my/config.py
View file

@ -9,7 +9,7 @@ This file is used for:
- mypy: this file provides some type annotations - mypy: this file provides some type annotations
- for loading the actual user config - for loading the actual user config
''' '''
#### vvvv you won't need this VVV in your personal config #### NOTE: you won't need this line VVVV in your personal config
from my.core import init from my.core import init
### ###

98
my/core/__main__.py
View file

@ -2,7 +2,9 @@ import functools
import importlib import importlib
import inspect import inspect
import os import os
import shutil
import sys import sys
import tempfile
import traceback import traceback
from typing import Optional, Sequence, Iterable, List, Type, Any, Callable from typing import Optional, Sequence, Iterable, List, Type, Any, Callable
from pathlib import Path from pathlib import Path
@ -16,31 +18,25 @@ def mypy_cmd() -> Optional[Sequence[str]]:
try: try:
# preferably, use mypy from current python env # preferably, use mypy from current python env
import mypy import mypy
return [sys.executable, '-m', 'mypy']
except ImportError: except ImportError:
pass pass
else:
return [sys.executable, '-m', 'mypy']
# ok, not ideal but try from PATH # ok, not ideal but try from PATH
import shutil
if shutil.which('mypy'): if shutil.which('mypy'):
return ['mypy'] return ['mypy']
warning("mypy not found, so can't check config with it. See https://github.com/python/mypy#readme if you want to install it and retry") warning("mypy not found, so can't check config with it. See https://github.com/python/mypy#readme if you want to install it and retry")
return None return None
from types import ModuleType def run_mypy(cfg_path: Path) -> Optional[CompletedProcess]:
def run_mypy(pkg: ModuleType) -> Optional[CompletedProcess]:
from .preinit import get_mycfg_dir
mycfg_dir = get_mycfg_dir()
# todo ugh. not sure how to extract it from pkg?
# todo dunno maybe use the same mypy config in repository? # todo dunno maybe use the same mypy config in repository?
# I'd need to install mypy.ini then?? # I'd need to install mypy.ini then??
env = {**os.environ} env = {**os.environ}
mpath = env.get('MYPYPATH') mpath = env.get('MYPYPATH')
mpath = str(mycfg_dir) + ('' if mpath is None else f':{mpath}') mpath = str(cfg_path) + ('' if mpath is None else f':{mpath}')
env['MYPYPATH'] = mpath env['MYPYPATH'] = mpath
cmd = mypy_cmd() cmd = mypy_cmd()
if cmd is None: if cmd is None:
return None return None
@ -52,7 +48,7 @@ def run_mypy(pkg: ModuleType) -> Optional[CompletedProcess]:
'--show-error-codes', '--show-error-codes',
'--show-error-context', '--show-error-context',
'--check-untyped-defs', '--check-untyped-defs',
'-p', pkg.__name__, '-p', 'my.config',
], stderr=PIPE, stdout=PIPE, env=env) ], stderr=PIPE, stdout=PIPE, env=env)
return mres return mres
@ -128,10 +124,11 @@ class example:
sys.exit(1) sys.exit(1)
# TODO return the config as a result? # todo return the config as a result?
def config_ok() -> bool: def config_ok() -> bool:
errors: List[Exception] = [] errors: List[Exception] = []
# at this point 'my' should already be imported, so doesn't hurt to extract paths from it
import my import my
try: try:
paths: List[str] = list(my.__path__) # type: ignore[attr-defined] paths: List[str] = list(my.__path__) # type: ignore[attr-defined]
@ -142,23 +139,17 @@ def config_ok() -> bool:
else: else:
info(f'import order: {paths}') info(f'import order: {paths}')
try: # first try doing as much as possible without actually imporing my.config
import my.config as cfg from .preinit import get_mycfg_dir
except Exception as e: cfg_path = get_mycfg_dir()
errors.append(e) # alternative is importing my.config and then getting cfg_path from its __file__/__path__
error("failed to import the config") # not sure which is better tbh
tb(e)
# todo yield exception here? so it doesn't fail immediately..
# I guess it's fairly critical and worth exiting immediately
sys.exit(1)
cfg_path = cfg.__file__# todo might be better to use __path__? ## check we're not using stub config
info(f"config file : {cfg_path}") import my.core
import my.core as core
try: try:
core_pkg_path = str(Path(core.__path__[0]).parent) # type: ignore[attr-defined] core_pkg_path = str(Path(my.core.__path__[0]).parent) # type: ignore[attr-defined]
if cfg_path.startswith(core_pkg_path): if str(cfg_path).startswith(core_pkg_path):
error(f''' error(f'''
Seems that the stub config is used ({cfg_path}). This is likely not going to work. Seems that the stub config is used ({cfg_path}). This is likely not going to work.
See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules for more information See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules for more information
@ -167,25 +158,53 @@ See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-module
except Exception as e: except Exception as e:
errors.append(e) errors.append(e)
tb(e) tb(e)
else:
info(f"config path : {cfg_path}")
##
# todo for some reason compileall.compile_file always returns true?? ## check syntax
try: with tempfile.TemporaryDirectory() as td:
cmd = [sys.executable, '-m', 'compileall', str(cfg_path)] # use a temporary directory, useful because
check_call(cmd) # - compileall ignores -B, so always craps with .pyc files (annoyng on RO filesystems)
info('syntax check: ' + ' '.join(cmd)) # - compileall isn't following symlinks, just silently ignores them
except Exception as e: # note: ugh, annoying that copytree requires a non-existing dir before 3.8.
errors.append(e) # once we have min version 3.8, can use dirs_exist_ok=True param
tdir = Path(td) / 'cfg'
# this will resolve symlinks when copying
shutil.copytree(cfg_path, tdir)
# NOTE: compileall still returns code 0 if the path doesn't exist..
# but in our case hopefully it's not an issue
cmd = [sys.executable, '-m', 'compileall', '-q', str(tdir)]
mres = run_mypy(cfg) try:
if mres is not None: # has mypy check_call(cmd)
rc = mres.returncode info('syntax check: ' + ' '.join( cmd))
except Exception as e:
errors.append(e)
tb(e)
##
## check types
mypy_res = run_mypy(cfg_path)
if mypy_res is not None: # has mypy
rc = mypy_res.returncode
if rc == 0: if rc == 0:
info('mypy check : success') info('mypy check : success')
else: else:
error('mypy check: failed') error('mypy check: failed')
errors.append(RuntimeError('mypy failed')) errors.append(RuntimeError('mypy failed'))
sys.stderr.write(indent(mres.stderr.decode('utf8'))) sys.stderr.write(indent(mypy_res.stderr.decode('utf8')))
sys.stderr.write(indent(mres.stdout.decode('utf8'))) sys.stderr.write(indent(mypy_res.stdout.decode('utf8')))
##
## finally, try actually importing the config (it should use same cfg_path)
try:
import my.config
except Exception as e:
errors.append(e)
error("failed to import the config")
tb(e)
##
if len(errors) > 0: if len(errors) > 0:
error(f'config check: {len(errors)} errors') error(f'config check: {len(errors)} errors')
@ -512,7 +531,6 @@ def main(debug: bool) -> None:
# to avoid importing relative modules by accident during development # to avoid importing relative modules by accident during development
# maybe can be removed later if theres more test coverage/confidence that nothing # maybe can be removed later if theres more test coverage/confidence that nothing
# would happen? # would happen?
import tempfile
# use a particular directory instead of a random one, since # use a particular directory instead of a random one, since
# click being decorator based means its more complicated # click being decorator based means its more complicated

37
my/core/init.py
View file

@ -1,29 +1,15 @@
''' '''
A hook to insert user's config directory into Python's search path. A hook to insert user's config directory into Python's search path.
- Ideally that would be in __init__.py (so it's executed without having to import explicityly) Ideally that would be in __init__.py (so it's executed without having to import explicityly)
But, with namespace packages, we can't have __init__.py in the parent subpackage But, with namespace packages, we can't have __init__.py in the parent subpackage
(see http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html#the-init-py-trap) (see http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html#the-init-py-trap)
Please let me know if you are aware of a better way of dealing with this! Instead, this is imported in the stub config (in this repository), so if the stub config is used, it triggers import of the 'real' config.
Please let me know if you are aware of a better way of dealing with this!
''' '''
from types import ModuleType
# TODO not ideal to keep it here, but this should really be a leaf in the import tree
# TODO maybe I don't even need it anymore?
def assign_module(parent: str, name: str, module: ModuleType) -> None:
import sys
import importlib
parent_module = importlib.import_module(parent)
sys.modules[parent + '.' + name] = module
if sys.version_info.minor == 6:
# ugh. not sure why it's necessary in py36...
# TODO that crap should be tested... I guess will get it for free when I run rest of tests in the matrix
setattr(parent_module, name, module)
del ModuleType
# separate function to present namespace pollution # separate function to present namespace pollution
def setup_config() -> None: def setup_config() -> None:
@ -45,16 +31,17 @@ See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-mo
# hopefully it doesn't cause any issues # hopefully it doesn't cause any issues
sys.path.insert(0, mpath) sys.path.insert(0, mpath)
# remove the stub and insert reimport hte 'real' config # remove the stub and reimport the 'real' config
# likely my.config will always be in sys.modules, but defensive just in case
if 'my.config' in sys.modules: if 'my.config' in sys.modules:
# TODO FIXME make sure this method isn't called twice...
del sys.modules['my.config'] del sys.modules['my.config']
# this should import from mpath now
try: try:
# todo import_from instead?? dunno
import my.config import my.config
except ImportError as ex: except ImportError as ex:
# just in case... who knows what crazy setup users have in mind. # just in case... who knows what crazy setup users have
# todo log? import logging
logging.exception(ex)
warnings.warn(f""" warnings.warn(f"""
Importing 'my.config' failed! (error: {ex}). This is likely to result in issues. Importing 'my.config' failed! (error: {ex}). This is likely to result in issues.
See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-modules for more info. See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-modules for more info.