From 1215181af533ba49bbb7658de66b259333e90310 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 25 Aug 2024 16:36:34 +0100 Subject: [PATCH] core: move stuff from tests/demo.py to my/core/tests/test_config.py also clean all this up a bit --- my/core/__init__.py | 4 +- my/core/tests/test_config.py | 132 +++++++++++++++++++++++++++++++++++ my/demo.py | 53 ++++++++------ tests/demo.py | 118 ------------------------------- 4 files changed, 164 insertions(+), 143 deletions(-) create mode 100644 my/core/tests/test_config.py delete mode 100644 tests/demo.py diff --git a/my/core/__init__.py b/my/core/__init__.py index 19be7fe..ba633f6 100644 --- a/my/core/__init__.py +++ b/my/core/__init__.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING from .cfg import make_config from .common import PathIsh, Paths, get_files from .compat import assert_never -from .error import Res, unwrap +from .error import Res, unwrap, notnone from .logging import ( make_logger, ) @@ -42,7 +42,7 @@ __all__ = [ '__NOT_HPI_MODULE__', - 'Res', 'unwrap', + 'Res', 'unwrap', 'notnone', 'dataclass', 'Path', ] diff --git a/my/core/tests/test_config.py b/my/core/tests/test_config.py new file mode 100644 index 0000000..c76f5d9 --- /dev/null +++ b/my/core/tests/test_config.py @@ -0,0 +1,132 @@ +""" +Various tests that are checking behaviour of user config wrt to various things +""" + +import sys +from pathlib import Path + +import pytest +import pytz +from more_itertools import ilen + +import my.config +from my.core import notnone +from my.demo import items, make_config + + +# run the same test multiple times to make sure there are not issues with import order etc +@pytest.mark.parametrize('run_id', ['1', '2']) +def test_override_config(tmp_path: Path, run_id: str) -> None: + class user_config: + username = f'user_{run_id}' + data_path = f'{tmp_path}/*.json' + + my.config.demo = user_config # type: ignore[misc, assignment] + + [item1, item2] = items() + assert item1.username == f'user_{run_id}' + assert item2.username == f'user_{run_id}' + + +@pytest.mark.skip(reason="won't work at the moment because of inheritance") +def test_dynamic_config_simplenamespace(tmp_path: Path) -> None: + from types import SimpleNamespace + + user_config = SimpleNamespace( + username='user3', + data_path=f'{tmp_path}/*.json', + ) + my.config.demo = user_config # type: ignore[misc, assignment] + + cfg = make_config() + + assert cfg.username == 'user3' + + +def test_mixin_attribute_handling(tmp_path: Path) -> None: + """ + Tests that arbitrary mixin attributes work with our config handling pattern + """ + + nytz = pytz.timezone('America/New_York') + + class user_config: + # check that override is taken into the account + timezone = nytz + + irrelevant = 'hello' + + username = 'UUU' + data_path = f'{tmp_path}/*.json' + + my.config.demo = user_config # type: ignore[misc, assignment] + + cfg = make_config() + + assert cfg.username == 'UUU' + + # mypy doesn't know about it, but the attribute is there + assert getattr(cfg, 'irrelevant') == 'hello' + + # check that overridden default attribute is actually getting overridden + assert cfg.timezone == nytz + + [item1, item2] = items() + assert item1.username == 'UUU' + assert notnone(item1.dt.tzinfo).zone == nytz.zone # type: ignore[attr-defined] + assert item2.username == 'UUU' + assert notnone(item2.dt.tzinfo).zone == nytz.zone # type: ignore[attr-defined] + + +# use multiple identical tests to make sure there are no issues with cached imports etc +@pytest.mark.parametrize('run_id', ['1', '2']) +def test_dynamic_module_import(tmp_path: Path, run_id: str) -> None: + """ + Test for dynamic hackery in config properties + e.g. importing some external modules + """ + + ext = tmp_path / 'external' + ext.mkdir() + (ext / '__init__.py').write_text( + ''' +def transform(x): + from .submodule import do_transform + return do_transform(x) + +''' + ) + (ext / 'submodule.py').write_text( + f''' +def do_transform(x): + return {{"total_{run_id}": sum(x.values())}} +''' + ) + + class user_config: + username = 'someuser' + data_path = f'{tmp_path}/*.json' + external = f'{ext}' + + my.config.demo = user_config # type: ignore[misc, assignment] + + [item1, item2] = items() + assert item1.raw == {f'total_{run_id}': 1 + 123}, item1 + assert item2.raw == {f'total_{run_id}': 2 + 456}, item2 + + # need to reset these modules, otherwise they get cached + # kind of relevant to my.core.cfg.tmp_config + sys.modules.pop('external', None) + sys.modules.pop('external.submodule', None) + + +@pytest.fixture(autouse=True) +def prepare_data(tmp_path: Path): + (tmp_path / 'data.json').write_text( + ''' +[ + {"key": 1, "value": 123}, + {"key": 2, "value": 456} +] +''' + ) diff --git a/my/demo.py b/my/demo.py index 645be4f..e27b5dd 100644 --- a/my/demo.py +++ b/my/demo.py @@ -2,19 +2,23 @@ Just a demo module for testing and documentation purposes ''' -from .core import Paths, PathIsh - -from typing import Optional -from datetime import tzinfo, timezone - -from my.config import demo as user_config +import json +from abc import abstractmethod from dataclasses import dataclass +from datetime import datetime, timezone, tzinfo +from pathlib import Path +from typing import Iterable, Optional, Protocol, Sequence + +from my.core import Json, PathIsh, Paths, get_files -@dataclass -class demo(user_config): +class config(Protocol): data_path: Paths + + # this is to check required attribute handling username: str + + # this is to check optional attribute handling timezone: tzinfo = timezone.utc external: Optional[PathIsh] = None @@ -23,47 +27,50 @@ class demo(user_config): def external_module(self): rpath = self.external if rpath is not None: - from .core.utils.imports import import_dir + from my.core.utils.imports import import_dir + return import_dir(rpath) - import my.config.repos.external as m # type: ignore + import my.config.repos.external as m # type: ignore + return m -from .core import make_config -config = make_config(demo) +def make_config() -> config: + from my.config import demo as user_config -# TODO not sure about type checking? -external = config.external_module + class combined_config(user_config, config): ... + return combined_config() -from pathlib import Path -from typing import Sequence, Iterable -from datetime import datetime -from .core import Json, get_files @dataclass class Item: ''' Some completely arbitrary artificial stuff, just for testing ''' + username: str raw: Json dt: datetime def inputs() -> Sequence[Path]: - return get_files(config.data_path) + cfg = make_config() + return get_files(cfg.data_path) -import json def items() -> Iterable[Item]: + cfg = make_config() + + transform = (lambda i: i) if cfg.external is None else cfg.external_module.transform + for f in inputs(): - dt = datetime.fromtimestamp(f.stat().st_mtime, tz=config.timezone) + dt = datetime.fromtimestamp(f.stat().st_mtime, tz=cfg.timezone) j = json.loads(f.read_text()) for raw in j: yield Item( - username=config.username, - raw=external.identity(raw), + username=cfg.username, + raw=transform(raw), dt=dt, ) diff --git a/tests/demo.py b/tests/demo.py deleted file mode 100644 index 73a6c65..0000000 --- a/tests/demo.py +++ /dev/null @@ -1,118 +0,0 @@ -import sys -from pathlib import Path -from more_itertools import ilen - -# TODO NOTE: this wouldn't work because of an early my.config.demo import -# from my.demo import items - -def test_dynamic_config_1(tmp_path: Path) -> None: - import my.config - - class user_config: - username = 'user' - data_path = f'{tmp_path}/*.json' - external = f'{tmp_path}/external' - my.config.demo = user_config # type: ignore[misc, assignment] - - from my.demo import items - [item1, item2] = items() - assert item1.username == 'user' - - -# exactly the same test, but using a different config, to test out the behaviour w.r.t. import order -def test_dynamic_config_2(tmp_path: Path) -> None: - # doesn't work without it! - # because the config from test_dybamic_config_1 is cached in my.demo.demo - del sys.modules['my.demo'] - - import my.config - - class user_config: - username = 'user2' - data_path = f'{tmp_path}/*.json' - external = f'{tmp_path}/external' - my.config.demo = user_config # type: ignore[misc, assignment] - - from my.demo import items - [item1, item2] = items() - assert item1.username == 'user2' - - -import pytest - -@pytest.mark.skip(reason="won't work at the moment because of inheritance") -def test_dynamic_config_simplenamespace(tmp_path: Path) -> None: - # doesn't work without it! - # because the config from test_dybamic_config_1 is cached in my.demo.demo - del sys.modules['my.demo'] - - import my.config - from types import SimpleNamespace - - user_config = SimpleNamespace( - username='user3', - data_path=f'{tmp_path}/*.json', - ) - my.config.demo = user_config # type: ignore[misc, assignment] - - from my.demo import config - assert config.username == 'user3' - - -# make sure our config handling pattern does it as expected -def test_attribute_handling(tmp_path: Path) -> None: - # doesn't work without it! - # because the config from test_dybamic_config_1 is cached in my.demo.demo - del sys.modules['my.demo'] - - import pytz - nytz = pytz.timezone('America/New_York') - - import my.config - class user_config: - # check that override is taken into the account - timezone = nytz - - irrelevant = 'hello' - - username = 'UUU' - data_path = f'{tmp_path}/*.json' - external = f'{tmp_path}/external' - - - my.config.demo = user_config # type: ignore[misc, assignment] - - from my.demo import config - - assert config.username == 'UUU' - - # mypy doesn't know about it, but the attribute is there - assert getattr(config, 'irrelevant') == 'hello' - - # check that overridden default attribute is actually getting overridden - assert config.timezone == nytz - - - -@pytest.fixture(autouse=True) -def prepare(tmp_path: Path): - (tmp_path / 'data.json').write_text(''' -[ - {"key1": 1}, - {"key2": 2} -] -''') - ext = tmp_path / 'external' - ext.mkdir() - (ext / '__init__.py').write_text(''' -def identity(x): - from .submodule import hello - hello(x) - return x - -''') - (ext / 'submodule.py').write_text('hello = lambda x: print("hello " + str(x))') - yield - ex = 'my.config.repos.external' - if ex in sys.modules: - del sys.modules[ex]