diff --git a/my/core/pytest.py b/my/core/pytest.py new file mode 100644 index 0000000..a2596fb --- /dev/null +++ b/my/core/pytest.py @@ -0,0 +1,22 @@ +""" +Helpers to prevent depending on pytest in runtime +""" + +from .common import assert_subpackage; assert_subpackage(__name__) + +import sys +import typing + +under_pytest = 'pytest' in sys.modules + +if typing.TYPE_CHECKING or under_pytest: + import pytest + + parametrize = pytest.mark.parametrize +else: + + def parametrize(*args, **kwargs): + def wrapper(f): + return f + + return wrapper diff --git a/my/core/serialize.py b/my/core/serialize.py index 1f55f40..563e114 100644 --- a/my/core/serialize.py +++ b/my/core/serialize.py @@ -2,11 +2,12 @@ import datetime from dataclasses import is_dataclass, asdict from pathlib import Path from decimal import Decimal -from typing import Any, Optional, Callable, NamedTuple +from typing import Any, Optional, Callable, NamedTuple, Protocol from functools import lru_cache from .common import is_namedtuple from .error import error_to_json +from .pytest import parametrize # note: it would be nice to combine the 'asdict' and _default_encode to some function # that takes a complex python object and returns JSON-compatible fields, while still @@ -16,6 +17,8 @@ from .error import error_to_json DefaultEncoder = Callable[[Any], Any] +Dumps = Callable[[Any], str] + def _default_encode(obj: Any) -> Any: """ @@ -75,22 +78,29 @@ def _dumps_factory(**kwargs) -> Callable[[Any], str]: kwargs["default"] = use_default - try: - import orjson + prefer_factory: Optional[str] = kwargs.pop('_prefer_factory', None) + + def orjson_factory() -> Optional[Dumps]: + try: + import orjson + except ModuleNotFoundError: + return None # todo: add orjson.OPT_NON_STR_KEYS? would require some bitwise ops # most keys are typically attributes from a NT/Dataclass, # so most seem to work: https://github.com/ijl/orjson#opt_non_str_keys - def _orjson_dumps(obj: Any) -> str: + def _orjson_dumps(obj: Any) -> str: # TODO rename? # orjson returns json as bytes, encode to string return orjson.dumps(obj, **kwargs).decode('utf-8') return _orjson_dumps - except ModuleNotFoundError: - pass - try: - from simplejson import dumps as simplejson_dumps + def simplejson_factory() -> Optional[Dumps]: + try: + from simplejson import dumps as simplejson_dumps + except ModuleNotFoundError: + return None + # if orjson couldn't be imported, try simplejson # This is included for compatibility reasons because orjson # is rust-based and compiling on rarer architectures may not work @@ -105,18 +115,37 @@ def _dumps_factory(**kwargs) -> Callable[[Any], str]: return _simplejson_dumps - except ModuleNotFoundError: - pass + def stdlib_factory() -> Optional[Dumps]: + import json + from .warnings import high - import json - from .warnings import high + high( + "You might want to install 'orjson' to support serialization for lots more types! If that does not work for you, you can install 'simplejson' instead" + ) - high("You might want to install 'orjson' to support serialization for lots more types! If that does not work for you, you can install 'simplejson' instead") + def _stdlib_dumps(obj: Any) -> str: + return json.dumps(obj, **kwargs) - def _stdlib_dumps(obj: Any) -> str: - return json.dumps(obj, **kwargs) + return _stdlib_dumps - return _stdlib_dumps + factories = { + 'orjson': orjson_factory, + 'simplejson': simplejson_factory, + 'stdlib': stdlib_factory, + } + + if prefer_factory is not None: + factory = factories[prefer_factory] + res = factory() + assert res is not None, prefer_factory + return res + + for factory in factories.values(): + res = factory() + if res is not None: + return res + else: + raise RuntimeError("Should not happen!") def dumps( @@ -154,8 +183,17 @@ def dumps( return _dumps_factory(default=default, **kwargs)(obj) -def test_serialize_fallback() -> None: - import json as jsn # dont cause possible conflicts with module code +@parametrize('factory', ['orjson', 'simplejson', 'stdlib']) +def test_dumps(factory: str) -> None: + import pytest + + orig_dumps = globals()['dumps'] # hack to prevent error from using local variable before declaring + + def dumps(*args, **kwargs) -> str: + kwargs['_prefer_factory'] = factory + return orig_dumps(*args, **kwargs) + + import json as json_builtin # dont cause possible conflicts with module code # can't use a namedtuple here, since the default json.dump serializer # serializes namedtuples as tuples, which become arrays @@ -166,36 +204,12 @@ def test_serialize_fallback() -> None: # the lru_cache'd warning may have already been sent, # so checking may be nondeterministic? import warnings + with warnings.catch_warnings(): warnings.simplefilter("ignore") - res = jsn.loads(dumps(X)) + res = json_builtin.loads(dumps(X)) assert res == [5, 5.0] - -# this needs to be defined here to prevent a mypy bug -# see https://github.com/python/mypy/issues/7281 -class _A(NamedTuple): - x: int - y: float - - -def test_nt_serialize() -> None: - import json as jsn # dont cause possible conflicts with module code - import orjson # import to make sure this is installed - - res: str = dumps(_A(x=1, y=2.0)) - assert res == '{"x":1,"y":2.0}' - - # test orjson option kwarg - data = {datetime.date(year=1970, month=1, day=1): 5} - res2 = jsn.loads(dumps(data, option=orjson.OPT_NON_STR_KEYS)) - assert res2 == {'1970-01-01': 5} - - -def test_default_serializer() -> None: - import pytest - import json as jsn # dont cause possible conflicts with module code - class Unserializable: def __init__(self, x: int): self.x = x @@ -209,7 +223,7 @@ def test_default_serializer() -> None: def _serialize(self) -> Any: return {"x": self.x, "y": self.y} - res = jsn.loads(dumps(WithUnderscoreSerialize(6))) + res = json_builtin.loads(dumps(WithUnderscoreSerialize(6))) assert res == {"x": 6, "y": 6.0} # test passing additional 'default' func @@ -221,5 +235,26 @@ def test_default_serializer() -> None: # this serializes both Unserializable, which is a custom type otherwise # not handled, and timedelta, which is handled by the '_default_encode' # in the 'wrapped_default' function - res2 = jsn.loads(dumps(Unserializable(10), default=_serialize_with_default)) + res2 = json_builtin.loads(dumps(Unserializable(10), default=_serialize_with_default)) assert res2 == {"x": 10, "y": 10.0} + + if factory == 'orjson': + import orjson + + # test orjson option kwarg + data = {datetime.date(year=1970, month=1, day=1): 5} + res2 = json_builtin.loads(dumps(data, option=orjson.OPT_NON_STR_KEYS)) + assert res2 == {'1970-01-01': 5} + + +@parametrize('factory', ['orjson', 'simplejson']) +def test_dumps_namedtuple(factory: str) -> None: + import json as json_builtin # dont cause possible conflicts with module code + import orjson # import to make sure this is installed + + class _A(NamedTuple): + x: int + y: float + + res: str = dumps(_A(x=1, y=2.0), _prefer_factory=factory) + assert json_builtin.loads(res) == {'x': 1, 'y': 2.0} diff --git a/setup.py b/setup.py index e6bc9fa..ab96616 100644 --- a/setup.py +++ b/setup.py @@ -47,13 +47,16 @@ def main() -> None: install_requires=INSTALL_REQUIRES, extras_require={ 'testing': [ - 'pytest<8', # FIXME <8 is temporary workaround till we fix collection with pytest 8; see https://docs.pytest.org/en/stable/changelog.html#collection-changes + 'pytest', 'ruff', 'mypy', 'lxml', # for mypy coverage # used in some tests.. although shouldn't rely on it 'pandas', + + 'orjson', # for my.core.serialize and denylist + 'simplejson', # for my.core.serialize ], 'optional': [ # todo document these? diff --git a/tests/serialize.py b/tests/serialize.py deleted file mode 100644 index d9ee9a3..0000000 --- a/tests/serialize.py +++ /dev/null @@ -1 +0,0 @@ -from my.core.serialize import * diff --git a/tests/serialize_simplejson.py b/tests/serialize_simplejson.py deleted file mode 100644 index d421a15..0000000 --- a/tests/serialize_simplejson.py +++ /dev/null @@ -1,23 +0,0 @@ -''' -This file should only run when simplejson is installed, -but orjson is not installed to check compatibility -''' - -# none of these should fail - -import json -import simplejson -import pytest - -from my.core.serialize import dumps, _A - -def test_simplejson_fallback() -> None: - - # should fail to import - with pytest.raises(ModuleNotFoundError): - import orjson - - # simplejson should serialize namedtuple properly - res: str = dumps(_A(x=1, y=2.0)) - assert json.loads(res) == {"x": 1, "y": 2.0} - diff --git a/tox.ini b/tox.ini index 0676eef..0b75b44 100644 --- a/tox.ini +++ b/tox.ini @@ -34,14 +34,11 @@ commands = commands = {envpython} -m pip install --use-pep517 -e .[testing] - # seems that denylist tests rely on it? ideally we should get rid of this in tests-core - {envpython} -m pip install orjson - {envpython} -m pytest \ # importlib is the new suggested import-mode # without it test package names end up as core.tests.* instead of my.core.tests.* --import-mode=importlib \ - --pyargs my.core \ + --pyargs {[testenv]package_name}.core \ # ignore orgmode because it imports orgparse # tbh not sure if it even belongs to core, maybe move somewhere else.. # same with pandas? @@ -49,9 +46,6 @@ commands = # causes error during test collection on 3.8 # dataset is deprecated anyway so whatever --ignore my/core/dataset.py \ - # this test uses orjson which is an optional dependency - # it would be covered by tests-all - -k 'not test_nt_serialize' \ {posargs} @@ -63,14 +57,7 @@ setenv = MY_CONFIG = nonexistent commands = {envpython} -m pip install --use-pep517 -e .[testing] - # installed to test my.core.serialize while using simplejson and not orjson - {envpython} -m pip install simplejson - {envpython} -m pytest \ - tests/serialize_simplejson.py \ - {posargs} - {envpython} -m pip install cachew - {envpython} -m pip install orjson {envpython} -m my.core module install my.location.google {envpython} -m pip install ijson # optional dependency @@ -103,9 +90,7 @@ commands = {envpython} -m pytest tests \ # ignore some tests which might take a while to run on ci.. --ignore tests/takeout.py \ - --ignore tests/extra/polar.py \ - # dont run simplejson compatibility test since orjson is now installed - --ignore tests/serialize_simplejson.py \ + --ignore tests/extra/polar.py {posargs}