core: move stuff from tests/demo.py to my/core/tests/test_config.py

also clean all this up a bit

my/core/__init__.py

@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
 from .cfg import make_config
 from .common import PathIsh, Paths, get_files
 from .compat import assert_never
-from .error import Res, unwrap
+from .error import Res, unwrap, notnone
 from .logging import (
     make_logger,
 )
@@ -42,7 +42,7 @@ __all__ = [
 
     '__NOT_HPI_MODULE__',
 
-    'Res', 'unwrap',
+    'Res', 'unwrap', 'notnone',
 
     'dataclass', 'Path',
 ]

my/core/tests/test_config.py

@@ -0,0 +1,132 @@
+"""
+Various tests that are checking behaviour of user config wrt to various things
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+import pytz
+from more_itertools import ilen
+
+import my.config
+from my.core import notnone
+from my.demo import items, make_config
+
+
+# run the same test multiple times to make sure there are not issues with import order etc
+@pytest.mark.parametrize('run_id', ['1', '2'])
+def test_override_config(tmp_path: Path, run_id: str) -> None:
+    class user_config:
+        username = f'user_{run_id}'
+        data_path = f'{tmp_path}/*.json'
+
+    my.config.demo = user_config  # type: ignore[misc, assignment]
+
+    [item1, item2] = items()
+    assert item1.username == f'user_{run_id}'
+    assert item2.username == f'user_{run_id}'
+
+
+@pytest.mark.skip(reason="won't work at the moment because of inheritance")
+def test_dynamic_config_simplenamespace(tmp_path: Path) -> None:
+    from types import SimpleNamespace
+
+    user_config = SimpleNamespace(
+        username='user3',
+        data_path=f'{tmp_path}/*.json',
+    )
+    my.config.demo = user_config  # type: ignore[misc, assignment]
+
+    cfg = make_config()
+
+    assert cfg.username == 'user3'
+
+
+def test_mixin_attribute_handling(tmp_path: Path) -> None:
+    """
+    Tests that arbitrary mixin attributes work with our config handling pattern
+    """
+
+    nytz = pytz.timezone('America/New_York')
+
+    class user_config:
+        # check that override is taken into the account
+        timezone = nytz
+
+        irrelevant = 'hello'
+
+        username = 'UUU'
+        data_path = f'{tmp_path}/*.json'
+
+    my.config.demo = user_config  # type: ignore[misc, assignment]
+
+    cfg = make_config()
+
+    assert cfg.username == 'UUU'
+
+    # mypy doesn't know about it, but the attribute is there
+    assert getattr(cfg, 'irrelevant') == 'hello'
+
+    # check that overridden default attribute is actually getting overridden
+    assert cfg.timezone == nytz
+
+    [item1, item2] = items()
+    assert item1.username == 'UUU'
+    assert notnone(item1.dt.tzinfo).zone == nytz.zone  # type: ignore[attr-defined]
+    assert item2.username == 'UUU'
+    assert notnone(item2.dt.tzinfo).zone == nytz.zone  # type: ignore[attr-defined]
+
+
+# use multiple identical tests to make sure there are no issues with cached imports etc
+@pytest.mark.parametrize('run_id', ['1', '2'])
+def test_dynamic_module_import(tmp_path: Path, run_id: str) -> None:
+    """
+    Test for dynamic hackery in config properties
+     e.g. importing some external modules
+    """
+
+    ext = tmp_path / 'external'
+    ext.mkdir()
+    (ext / '__init__.py').write_text(
+        '''
+def transform(x):
+    from .submodule import do_transform
+    return do_transform(x)
+
+'''
+    )
+    (ext / 'submodule.py').write_text(
+        f'''
+def do_transform(x):
+    return {{"total_{run_id}": sum(x.values())}}
+'''
+    )
+
+    class user_config:
+        username = 'someuser'
+        data_path = f'{tmp_path}/*.json'
+        external = f'{ext}'
+
+    my.config.demo = user_config  # type: ignore[misc, assignment]
+
+    [item1, item2] = items()
+    assert item1.raw == {f'total_{run_id}': 1 + 123}, item1
+    assert item2.raw == {f'total_{run_id}': 2 + 456}, item2
+
+    # need to reset these modules, otherwise they get cached
+    # kind of relevant to my.core.cfg.tmp_config
+    sys.modules.pop('external', None)
+    sys.modules.pop('external.submodule', None)
+
+
+@pytest.fixture(autouse=True)
+def prepare_data(tmp_path: Path):
+    (tmp_path / 'data.json').write_text(
+        '''
+[
+    {"key": 1, "value": 123},
+    {"key": 2, "value": 456}
+]
+'''
+    )

my/demo.py

@@ -2,19 +2,23 @@
 Just a demo module for testing and documentation purposes
 '''
 
-from .core import Paths, PathIsh
+import json
-
+from abc import abstractmethod
-from typing import Optional
-from datetime import tzinfo, timezone
-
-from my.config import demo as user_config
 from dataclasses import dataclass
+from datetime import datetime, timezone, tzinfo
+from pathlib import Path
+from typing import Iterable, Optional, Protocol, Sequence
+
+from my.core import Json, PathIsh, Paths, get_files
 
 
-@dataclass
+class config(Protocol):
-class demo(user_config):
     data_path: Paths
+
+    # this is to check required attribute handling
     username: str
+
+    # this is to check optional attribute handling
     timezone: tzinfo = timezone.utc
 
     external: Optional[PathIsh] = None
@@ -23,47 +27,50 @@ class demo(user_config):
     def external_module(self):
         rpath = self.external
         if rpath is not None:
-            from .core.utils.imports import import_dir
+            from my.core.utils.imports import import_dir
+
             return import_dir(rpath)
 
         import my.config.repos.external as m  # type: ignore
+
         return m
 
 
-from .core import make_config
+def make_config() -> config:
-config = make_config(demo)
+    from my.config import demo as user_config
 
-# TODO not sure about type checking?
+    class combined_config(user_config, config): ...
-external = config.external_module
 
+    return combined_config()
 
-from pathlib import Path
-from typing import Sequence, Iterable
-from datetime import datetime
-from .core import Json, get_files
 
 @dataclass
 class Item:
     '''
     Some completely arbitrary artificial stuff, just for testing
     '''
+
     username: str
     raw: Json
     dt: datetime
 
 
 def inputs() -> Sequence[Path]:
-    return get_files(config.data_path)
+    cfg = make_config()
+    return get_files(cfg.data_path)
 
 
-import json
 def items() -> Iterable[Item]:
+    cfg = make_config()
+
+    transform = (lambda i: i) if cfg.external is None else cfg.external_module.transform
+
     for f in inputs():
-        dt = datetime.fromtimestamp(f.stat().st_mtime, tz=config.timezone)
+        dt = datetime.fromtimestamp(f.stat().st_mtime, tz=cfg.timezone)
         j = json.loads(f.read_text())
         for raw in j:
             yield Item(
-                username=config.username,
+                username=cfg.username,
-                raw=external.identity(raw),
+                raw=transform(raw),
                 dt=dt,
             )

tests/demo.py

@@ -1,118 +0,0 @@
-import sys
-from pathlib import Path
-from more_itertools import ilen
-
-# TODO NOTE: this wouldn't work because of an early my.config.demo import
-# from my.demo import items
-
-def test_dynamic_config_1(tmp_path: Path) -> None:
-    import my.config
-
-    class user_config:
-        username  = 'user'
-        data_path = f'{tmp_path}/*.json'
-        external  = f'{tmp_path}/external'
-    my.config.demo = user_config # type: ignore[misc, assignment]
-
-    from my.demo import items
-    [item1, item2] = items()
-    assert item1.username == 'user'
-
-
-# exactly the same test, but using a different config, to test out the behaviour w.r.t. import order
-def test_dynamic_config_2(tmp_path: Path) -> None:
-    # doesn't work without it!
-    # because the config from test_dybamic_config_1 is cached in my.demo.demo
-    del sys.modules['my.demo']
-
-    import my.config
-
-    class user_config:
-        username  = 'user2'
-        data_path = f'{tmp_path}/*.json'
-        external  = f'{tmp_path}/external'
-    my.config.demo = user_config # type: ignore[misc, assignment]
-
-    from my.demo import items
-    [item1, item2] = items()
-    assert item1.username == 'user2'
-
-
-import pytest
-
-@pytest.mark.skip(reason="won't work at the moment because of inheritance")
-def test_dynamic_config_simplenamespace(tmp_path: Path) -> None:
-    # doesn't work without it!
-    # because the config from test_dybamic_config_1 is cached in my.demo.demo
-    del sys.modules['my.demo']
-
-    import my.config
-    from types import SimpleNamespace
-
-    user_config = SimpleNamespace(
-        username='user3',
-        data_path=f'{tmp_path}/*.json',
-    )
-    my.config.demo = user_config # type: ignore[misc, assignment]
-
-    from my.demo import config
-    assert config.username == 'user3'
-
-
-# make sure our config handling pattern does it as expected
-def test_attribute_handling(tmp_path: Path) -> None:
-    # doesn't work without it!
-    # because the config from test_dybamic_config_1 is cached in my.demo.demo
-    del sys.modules['my.demo']
-
-    import pytz
-    nytz = pytz.timezone('America/New_York')
-
-    import my.config
-    class user_config:
-        # check that override is taken into the account
-        timezone = nytz
-
-        irrelevant = 'hello'
-
-        username = 'UUU'
-        data_path = f'{tmp_path}/*.json'
-        external  = f'{tmp_path}/external'
-
-
-    my.config.demo = user_config # type: ignore[misc, assignment]
-
-    from my.demo import config
-
-    assert config.username == 'UUU'
-
-    # mypy doesn't know about it, but the attribute is there
-    assert getattr(config, 'irrelevant') == 'hello'
-
-    # check that overridden default attribute is actually getting overridden
-    assert config.timezone == nytz
-
-
-
-@pytest.fixture(autouse=True)
-def prepare(tmp_path: Path):
-    (tmp_path / 'data.json').write_text('''
-[
-    {"key1": 1},
-    {"key2": 2}
-]
-''')
-    ext = tmp_path / 'external'
-    ext.mkdir()
-    (ext / '__init__.py').write_text('''
-def identity(x):
-    from .submodule import hello
-    hello(x)
-    return x
-
-''')
-    (ext / 'submodule.py').write_text('hello = lambda x: print("hello " + str(x))')
-    yield
-    ex = 'my.config.repos.external'
-    if ex in sys.modules:
-        del sys.modules[ex]