core: move stuff from tests/demo.py to my/core/tests/test_config.py
also clean all this up a bit
This commit is contained in:
Dima Gerasimov
karlicoss
parent
5a67f0bafe
commit
1215181af5
4 changed files with 164 additions and 143 deletions
|
|
@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
|
|||
from .cfg import make_config
|
||||
from .common import PathIsh, Paths, get_files
|
||||
from .compat import assert_never
|
||||
from .error import Res, unwrap
|
||||
from .error import Res, unwrap, notnone
|
||||
from .logging import (
|
||||
make_logger,
|
||||
)
|
||||
|
|
@ -42,7 +42,7 @@ __all__ = [
|
|||
|
||||
'__NOT_HPI_MODULE__',
|
||||
|
||||
'Res', 'unwrap',
|
||||
'Res', 'unwrap', 'notnone',
|
||||
|
||||
'dataclass', 'Path',
|
||||
]
|
||||
|
|
|
|||
132
my/core/tests/test_config.py
Normal file
132
my/core/tests/test_config.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
"""
|
||||
Various tests that are checking behaviour of user config wrt to various things
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import pytz
|
||||
from more_itertools import ilen
|
||||
|
||||
import my.config
|
||||
from my.core import notnone
|
||||
from my.demo import items, make_config
|
||||
|
||||
|
||||
# run the same test multiple times to make sure there are not issues with import order etc
|
||||
@pytest.mark.parametrize('run_id', ['1', '2'])
|
||||
def test_override_config(tmp_path: Path, run_id: str) -> None:
|
||||
class user_config:
|
||||
username = f'user_{run_id}'
|
||||
data_path = f'{tmp_path}/*.json'
|
||||
|
||||
my.config.demo = user_config # type: ignore[misc, assignment]
|
||||
|
||||
[item1, item2] = items()
|
||||
assert item1.username == f'user_{run_id}'
|
||||
assert item2.username == f'user_{run_id}'
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="won't work at the moment because of inheritance")
|
||||
def test_dynamic_config_simplenamespace(tmp_path: Path) -> None:
|
||||
from types import SimpleNamespace
|
||||
|
||||
user_config = SimpleNamespace(
|
||||
username='user3',
|
||||
data_path=f'{tmp_path}/*.json',
|
||||
)
|
||||
my.config.demo = user_config # type: ignore[misc, assignment]
|
||||
|
||||
cfg = make_config()
|
||||
|
||||
assert cfg.username == 'user3'
|
||||
|
||||
|
||||
def test_mixin_attribute_handling(tmp_path: Path) -> None:
|
||||
"""
|
||||
Tests that arbitrary mixin attributes work with our config handling pattern
|
||||
"""
|
||||
|
||||
nytz = pytz.timezone('America/New_York')
|
||||
|
||||
class user_config:
|
||||
# check that override is taken into the account
|
||||
timezone = nytz
|
||||
|
||||
irrelevant = 'hello'
|
||||
|
||||
username = 'UUU'
|
||||
data_path = f'{tmp_path}/*.json'
|
||||
|
||||
my.config.demo = user_config # type: ignore[misc, assignment]
|
||||
|
||||
cfg = make_config()
|
||||
|
||||
assert cfg.username == 'UUU'
|
||||
|
||||
# mypy doesn't know about it, but the attribute is there
|
||||
assert getattr(cfg, 'irrelevant') == 'hello'
|
||||
|
||||
# check that overridden default attribute is actually getting overridden
|
||||
assert cfg.timezone == nytz
|
||||
|
||||
[item1, item2] = items()
|
||||
assert item1.username == 'UUU'
|
||||
assert notnone(item1.dt.tzinfo).zone == nytz.zone # type: ignore[attr-defined]
|
||||
assert item2.username == 'UUU'
|
||||
assert notnone(item2.dt.tzinfo).zone == nytz.zone # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# use multiple identical tests to make sure there are no issues with cached imports etc
|
||||
@pytest.mark.parametrize('run_id', ['1', '2'])
|
||||
def test_dynamic_module_import(tmp_path: Path, run_id: str) -> None:
|
||||
"""
|
||||
Test for dynamic hackery in config properties
|
||||
e.g. importing some external modules
|
||||
"""
|
||||
|
||||
ext = tmp_path / 'external'
|
||||
ext.mkdir()
|
||||
(ext / '__init__.py').write_text(
|
||||
'''
|
||||
def transform(x):
|
||||
from .submodule import do_transform
|
||||
return do_transform(x)
|
||||
|
||||
'''
|
||||
)
|
||||
(ext / 'submodule.py').write_text(
|
||||
f'''
|
||||
def do_transform(x):
|
||||
return {{"total_{run_id}": sum(x.values())}}
|
||||
'''
|
||||
)
|
||||
|
||||
class user_config:
|
||||
username = 'someuser'
|
||||
data_path = f'{tmp_path}/*.json'
|
||||
external = f'{ext}'
|
||||
|
||||
my.config.demo = user_config # type: ignore[misc, assignment]
|
||||
|
||||
[item1, item2] = items()
|
||||
assert item1.raw == {f'total_{run_id}': 1 + 123}, item1
|
||||
assert item2.raw == {f'total_{run_id}': 2 + 456}, item2
|
||||
|
||||
# need to reset these modules, otherwise they get cached
|
||||
# kind of relevant to my.core.cfg.tmp_config
|
||||
sys.modules.pop('external', None)
|
||||
sys.modules.pop('external.submodule', None)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def prepare_data(tmp_path: Path):
|
||||
(tmp_path / 'data.json').write_text(
|
||||
'''
|
||||
[
|
||||
{"key": 1, "value": 123},
|
||||
{"key": 2, "value": 456}
|
||||
]
|
||||
'''
|
||||
)
|
||||
51
my/demo.py
51
my/demo.py
|
|
@ -2,19 +2,23 @@
|
|||
Just a demo module for testing and documentation purposes
|
||||
'''
|
||||
|
||||
from .core import Paths, PathIsh
|
||||
|
||||
from typing import Optional
|
||||
from datetime import tzinfo, timezone
|
||||
|
||||
from my.config import demo as user_config
|
||||
import json
|
||||
from abc import abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone, tzinfo
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Optional, Protocol, Sequence
|
||||
|
||||
from my.core import Json, PathIsh, Paths, get_files
|
||||
|
||||
|
||||
@dataclass
|
||||
class demo(user_config):
|
||||
class config(Protocol):
|
||||
data_path: Paths
|
||||
|
||||
# this is to check required attribute handling
|
||||
username: str
|
||||
|
||||
# this is to check optional attribute handling
|
||||
timezone: tzinfo = timezone.utc
|
||||
|
||||
external: Optional[PathIsh] = None
|
||||
|
|
@ -23,47 +27,50 @@ class demo(user_config):
|
|||
def external_module(self):
|
||||
rpath = self.external
|
||||
if rpath is not None:
|
||||
from .core.utils.imports import import_dir
|
||||
from my.core.utils.imports import import_dir
|
||||
|
||||
return import_dir(rpath)
|
||||
|
||||
import my.config.repos.external as m # type: ignore
|
||||
|
||||
return m
|
||||
|
||||
|
||||
from .core import make_config
|
||||
config = make_config(demo)
|
||||
def make_config() -> config:
|
||||
from my.config import demo as user_config
|
||||
|
||||
# TODO not sure about type checking?
|
||||
external = config.external_module
|
||||
class combined_config(user_config, config): ...
|
||||
|
||||
return combined_config()
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Sequence, Iterable
|
||||
from datetime import datetime
|
||||
from .core import Json, get_files
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
'''
|
||||
Some completely arbitrary artificial stuff, just for testing
|
||||
'''
|
||||
|
||||
username: str
|
||||
raw: Json
|
||||
dt: datetime
|
||||
|
||||
|
||||
def inputs() -> Sequence[Path]:
|
||||
return get_files(config.data_path)
|
||||
cfg = make_config()
|
||||
return get_files(cfg.data_path)
|
||||
|
||||
|
||||
import json
|
||||
def items() -> Iterable[Item]:
|
||||
cfg = make_config()
|
||||
|
||||
transform = (lambda i: i) if cfg.external is None else cfg.external_module.transform
|
||||
|
||||
for f in inputs():
|
||||
dt = datetime.fromtimestamp(f.stat().st_mtime, tz=config.timezone)
|
||||
dt = datetime.fromtimestamp(f.stat().st_mtime, tz=cfg.timezone)
|
||||
j = json.loads(f.read_text())
|
||||
for raw in j:
|
||||
yield Item(
|
||||
username=config.username,
|
||||
raw=external.identity(raw),
|
||||
username=cfg.username,
|
||||
raw=transform(raw),
|
||||
dt=dt,
|
||||
)
|
||||
|
|
|
|||
118
tests/demo.py
118
tests/demo.py
|
|
@ -1,118 +0,0 @@
|
|||
import sys
|
||||
from pathlib import Path
|
||||
from more_itertools import ilen
|
||||
|
||||
# TODO NOTE: this wouldn't work because of an early my.config.demo import
|
||||
# from my.demo import items
|
||||
|
||||
def test_dynamic_config_1(tmp_path: Path) -> None:
|
||||
import my.config
|
||||
|
||||
class user_config:
|
||||
username = 'user'
|
||||
data_path = f'{tmp_path}/*.json'
|
||||
external = f'{tmp_path}/external'
|
||||
my.config.demo = user_config # type: ignore[misc, assignment]
|
||||
|
||||
from my.demo import items
|
||||
[item1, item2] = items()
|
||||
assert item1.username == 'user'
|
||||
|
||||
|
||||
# exactly the same test, but using a different config, to test out the behaviour w.r.t. import order
|
||||
def test_dynamic_config_2(tmp_path: Path) -> None:
|
||||
# doesn't work without it!
|
||||
# because the config from test_dybamic_config_1 is cached in my.demo.demo
|
||||
del sys.modules['my.demo']
|
||||
|
||||
import my.config
|
||||
|
||||
class user_config:
|
||||
username = 'user2'
|
||||
data_path = f'{tmp_path}/*.json'
|
||||
external = f'{tmp_path}/external'
|
||||
my.config.demo = user_config # type: ignore[misc, assignment]
|
||||
|
||||
from my.demo import items
|
||||
[item1, item2] = items()
|
||||
assert item1.username == 'user2'
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
@pytest.mark.skip(reason="won't work at the moment because of inheritance")
|
||||
def test_dynamic_config_simplenamespace(tmp_path: Path) -> None:
|
||||
# doesn't work without it!
|
||||
# because the config from test_dybamic_config_1 is cached in my.demo.demo
|
||||
del sys.modules['my.demo']
|
||||
|
||||
import my.config
|
||||
from types import SimpleNamespace
|
||||
|
||||
user_config = SimpleNamespace(
|
||||
username='user3',
|
||||
data_path=f'{tmp_path}/*.json',
|
||||
)
|
||||
my.config.demo = user_config # type: ignore[misc, assignment]
|
||||
|
||||
from my.demo import config
|
||||
assert config.username == 'user3'
|
||||
|
||||
|
||||
# make sure our config handling pattern does it as expected
|
||||
def test_attribute_handling(tmp_path: Path) -> None:
|
||||
# doesn't work without it!
|
||||
# because the config from test_dybamic_config_1 is cached in my.demo.demo
|
||||
del sys.modules['my.demo']
|
||||
|
||||
import pytz
|
||||
nytz = pytz.timezone('America/New_York')
|
||||
|
||||
import my.config
|
||||
class user_config:
|
||||
# check that override is taken into the account
|
||||
timezone = nytz
|
||||
|
||||
irrelevant = 'hello'
|
||||
|
||||
username = 'UUU'
|
||||
data_path = f'{tmp_path}/*.json'
|
||||
external = f'{tmp_path}/external'
|
||||
|
||||
|
||||
my.config.demo = user_config # type: ignore[misc, assignment]
|
||||
|
||||
from my.demo import config
|
||||
|
||||
assert config.username == 'UUU'
|
||||
|
||||
# mypy doesn't know about it, but the attribute is there
|
||||
assert getattr(config, 'irrelevant') == 'hello'
|
||||
|
||||
# check that overridden default attribute is actually getting overridden
|
||||
assert config.timezone == nytz
|
||||
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def prepare(tmp_path: Path):
|
||||
(tmp_path / 'data.json').write_text('''
|
||||
[
|
||||
{"key1": 1},
|
||||
{"key2": 2}
|
||||
]
|
||||
''')
|
||||
ext = tmp_path / 'external'
|
||||
ext.mkdir()
|
||||
(ext / '__init__.py').write_text('''
|
||||
def identity(x):
|
||||
from .submodule import hello
|
||||
hello(x)
|
||||
return x
|
||||
|
||||
''')
|
||||
(ext / 'submodule.py').write_text('hello = lambda x: print("hello " + str(x))')
|
||||
yield
|
||||
ex = 'my.config.repos.external'
|
||||
if ex in sys.modules:
|
||||
del sys.modules[ex]
|
||||
Loading…
Add table
Reference in a new issue