tests: simplify tests for my.core.serialize a bit and simplify tox file

my/core/pytest.py

@@ -0,0 +1,22 @@
+"""
+Helpers to prevent depending on pytest in runtime
+"""
+
+from .common import assert_subpackage; assert_subpackage(__name__)
+
+import sys
+import typing
+
+under_pytest = 'pytest' in sys.modules
+
+if typing.TYPE_CHECKING or under_pytest:
+    import pytest
+
+    parametrize = pytest.mark.parametrize
+else:
+
+    def parametrize(*args, **kwargs):
+        def wrapper(f):
+            return f
+
+        return wrapper

my/core/serialize.py

@@ -2,11 +2,12 @@ import datetime
 from dataclasses import is_dataclass, asdict
 from pathlib import Path
 from decimal import Decimal
-from typing import Any, Optional, Callable, NamedTuple
+from typing import Any, Optional, Callable, NamedTuple, Protocol
 from functools import lru_cache
 
 from .common import is_namedtuple
 from .error import error_to_json
+from .pytest import parametrize
 
 # note: it would be nice to combine the 'asdict' and _default_encode to some function
 # that takes a complex python object and returns JSON-compatible fields, while still
@@ -16,6 +17,8 @@ from .error import error_to_json
 
 DefaultEncoder = Callable[[Any], Any]
 
+Dumps = Callable[[Any], str]
+
 
 def _default_encode(obj: Any) -> Any:
     """
@@ -75,22 +78,29 @@ def _dumps_factory(**kwargs) -> Callable[[Any], str]:
 
     kwargs["default"] = use_default
 
+    prefer_factory: Optional[str] = kwargs.pop('_prefer_factory', None)
+
+    def orjson_factory() -> Optional[Dumps]:
         try:
             import orjson
+        except ModuleNotFoundError:
+            return None
 
         # todo: add orjson.OPT_NON_STR_KEYS? would require some bitwise ops
         # most keys are typically attributes from a NT/Dataclass,
         # so most seem to work: https://github.com/ijl/orjson#opt_non_str_keys
-        def _orjson_dumps(obj: Any) -> str:
+        def _orjson_dumps(obj: Any) -> str:  # TODO rename?
             # orjson returns json as bytes, encode to string
             return orjson.dumps(obj, **kwargs).decode('utf-8')
 
         return _orjson_dumps
-    except ModuleNotFoundError:
-        pass
 
+    def simplejson_factory() -> Optional[Dumps]:
         try:
             from simplejson import dumps as simplejson_dumps
+        except ModuleNotFoundError:
+            return None
+
         # if orjson couldn't be imported, try simplejson
         # This is included for compatibility reasons because orjson
         # is rust-based and compiling on rarer architectures may not work
@@ -105,19 +115,38 @@ def _dumps_factory(**kwargs) -> Callable[[Any], str]:
 
         return _simplejson_dumps
 
-    except ModuleNotFoundError:
+    def stdlib_factory() -> Optional[Dumps]:
-        pass
-
         import json
         from .warnings import high
 
-    high("You might want to install 'orjson' to support serialization for lots more types! If that does not work for you, you can install 'simplejson' instead")
+        high(
+            "You might want to install 'orjson' to support serialization for lots more types! If that does not work for you, you can install 'simplejson' instead"
+        )
 
         def _stdlib_dumps(obj: Any) -> str:
             return json.dumps(obj, **kwargs)
 
         return _stdlib_dumps
 
+    factories = {
+        'orjson': orjson_factory,
+        'simplejson': simplejson_factory,
+        'stdlib': stdlib_factory,
+    }
+
+    if prefer_factory is not None:
+        factory = factories[prefer_factory]
+        res = factory()
+        assert res is not None, prefer_factory
+        return res
+
+    for factory in factories.values():
+        res = factory()
+        if res is not None:
+            return res
+    else:
+        raise RuntimeError("Should not happen!")
+
 
 def dumps(
     obj: Any,
@@ -154,8 +183,17 @@ def dumps(
     return _dumps_factory(default=default, **kwargs)(obj)
 
 
-def test_serialize_fallback() -> None:
+@parametrize('factory', ['orjson', 'simplejson', 'stdlib'])
-    import json as jsn  # dont cause possible conflicts with module code
+def test_dumps(factory: str) -> None:
+    import pytest
+
+    orig_dumps = globals()['dumps']  # hack to prevent error from using local variable before declaring
+
+    def dumps(*args, **kwargs) -> str:
+        kwargs['_prefer_factory'] = factory
+        return orig_dumps(*args, **kwargs)
+
+    import json as json_builtin  # dont cause possible conflicts with module code
 
     # can't use a namedtuple here, since the default json.dump serializer
     # serializes namedtuples as tuples, which become arrays
@@ -166,36 +204,12 @@ def test_serialize_fallback() -> None:
     # the lru_cache'd warning may have already been sent,
     # so checking may be nondeterministic?
     import warnings
+
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
-        res = jsn.loads(dumps(X))
+        res = json_builtin.loads(dumps(X))
         assert res == [5, 5.0]
 
-
-# this needs to be defined here to prevent a mypy bug
-# see https://github.com/python/mypy/issues/7281
-class _A(NamedTuple):
-    x: int
-    y: float
-
-
-def test_nt_serialize() -> None:
-    import json as jsn  # dont cause possible conflicts with module code
-    import orjson  # import to make sure this is installed
-
-    res: str = dumps(_A(x=1, y=2.0))
-    assert res == '{"x":1,"y":2.0}'
-
-    # test orjson option kwarg
-    data = {datetime.date(year=1970, month=1, day=1): 5}
-    res2 = jsn.loads(dumps(data, option=orjson.OPT_NON_STR_KEYS))
-    assert res2 == {'1970-01-01': 5}
-
-
-def test_default_serializer() -> None:
-    import pytest
-    import json as jsn  # dont cause possible conflicts with module code
-
     class Unserializable:
         def __init__(self, x: int):
             self.x = x
@@ -209,7 +223,7 @@ def test_default_serializer() -> None:
         def _serialize(self) -> Any:
             return {"x": self.x, "y": self.y}
 
-    res = jsn.loads(dumps(WithUnderscoreSerialize(6)))
+    res = json_builtin.loads(dumps(WithUnderscoreSerialize(6)))
     assert res == {"x": 6, "y": 6.0}
 
     # test passing additional 'default' func
@@ -221,5 +235,26 @@ def test_default_serializer() -> None:
     # this serializes both Unserializable, which is a custom type otherwise
     # not handled, and timedelta, which is handled by the '_default_encode'
     # in the 'wrapped_default' function
-    res2 = jsn.loads(dumps(Unserializable(10), default=_serialize_with_default))
+    res2 = json_builtin.loads(dumps(Unserializable(10), default=_serialize_with_default))
     assert res2 == {"x": 10, "y": 10.0}
+
+    if factory == 'orjson':
+        import orjson
+
+        # test orjson option kwarg
+        data = {datetime.date(year=1970, month=1, day=1): 5}
+        res2 = json_builtin.loads(dumps(data, option=orjson.OPT_NON_STR_KEYS))
+        assert res2 == {'1970-01-01': 5}
+
+
+@parametrize('factory', ['orjson', 'simplejson'])
+def test_dumps_namedtuple(factory: str) -> None:
+    import json as json_builtin  # dont cause possible conflicts with module code
+    import orjson  # import to make sure this is installed
+
+    class _A(NamedTuple):
+        x: int
+        y: float
+
+    res: str = dumps(_A(x=1, y=2.0), _prefer_factory=factory)
+    assert json_builtin.loads(res) == {'x': 1, 'y': 2.0}

setup.py

@@ -47,13 +47,16 @@ def main() -> None:
         install_requires=INSTALL_REQUIRES,
         extras_require={
             'testing': [
-                'pytest<8',  # FIXME <8 is temporary workaround till we fix collection with pytest 8; see https://docs.pytest.org/en/stable/changelog.html#collection-changes
+                'pytest',
                 'ruff',
                 'mypy',
                 'lxml', # for mypy coverage
 
                 # used in some tests.. although shouldn't rely on it
                 'pandas',
+
+                'orjson',  # for my.core.serialize and denylist
+                'simplejson',  # for my.core.serialize
             ],
             'optional': [
                 # todo document these?

tests/serialize.py

@@ -1 +0,0 @@
-from my.core.serialize      import *

tests/serialize_simplejson.py

@@ -1,23 +0,0 @@
-'''
-This file should only run when simplejson is installed,
-but orjson is not installed to check compatibility
-'''
-
-# none of these should fail
-
-import json
-import simplejson
-import pytest
-
-from my.core.serialize import dumps, _A
-
-def test_simplejson_fallback() -> None:
-
-    # should fail to import
-    with pytest.raises(ModuleNotFoundError):
-        import orjson
-
-    # simplejson should serialize namedtuple properly
-    res: str = dumps(_A(x=1, y=2.0))
-    assert json.loads(res) == {"x": 1, "y": 2.0}
-

tox.ini

@@ -34,14 +34,11 @@ commands =
 commands =
     {envpython} -m pip install --use-pep517 -e .[testing]
     
-    # seems that denylist tests rely on it? ideally we should get rid of this in tests-core
-    {envpython} -m pip install orjson
-
     {envpython} -m pytest \
         # importlib is the new suggested import-mode
         # without it test package names end up as core.tests.* instead of my.core.tests.*
         --import-mode=importlib     \
-        --pyargs my.core            \
+        --pyargs {[testenv]package_name}.core  \
         # ignore orgmode because it imports orgparse
         # tbh not sure if it even belongs to core, maybe move somewhere else..
         # same with pandas?
@@ -49,9 +46,6 @@ commands =
         # causes error during test collection on 3.8
         # dataset is deprecated anyway so whatever
         --ignore my/core/dataset.py \
-        # this test uses orjson which is an optional dependency
-        # it would be covered by tests-all
-        -k 'not test_nt_serialize' \
         {posargs}
 
 
@@ -63,14 +57,7 @@ setenv = MY_CONFIG = nonexistent
 commands =
     {envpython} -m pip install --use-pep517 -e .[testing]
 
-    # installed to test my.core.serialize while using simplejson and not orjson
-    {envpython} -m pip install simplejson
-    {envpython} -m pytest \
-        tests/serialize_simplejson.py \
-        {posargs}
-
     {envpython} -m pip install cachew
-    {envpython} -m pip install orjson
 
     {envpython} -m my.core module install my.location.google
     {envpython} -m pip install ijson # optional dependency
@@ -103,9 +90,7 @@ commands =
     {envpython} -m pytest tests \
         # ignore some tests which might take a while to run on ci..
         --ignore tests/takeout.py        \
-        --ignore tests/extra/polar.py \
+        --ignore tests/extra/polar.py
-        # dont run simplejson compatibility test since orjson is now installed
-        --ignore tests/serialize_simplejson.py \
         {posargs}