fz0x1/HPI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tests: simplify tests for my.core.serialize a bit and simplify tox file

Browse source
This commit is contained in:
Dima Gerasimov 2024-08-06 23:09:56 +01:00 • committed by karlicoss
parent 3aebc573e8
commit fb8e9909a4
6 changed files with 109 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
my/core/pytest.py Normal file
View file

@ -0,0 +1,22 @@
"""
Helpers to prevent depending on pytest in runtime
"""
from .common import assert_subpackage; assert_subpackage(__name__)
import sys
import typing
under_pytest = 'pytest' in sys.modules
if typing.TYPE_CHECKING or under_pytest:
import pytest
parametrize = pytest.mark.parametrize
else:
def parametrize(*args, **kwargs):
def wrapper(f):
return f
return wrapper

127
my/core/serialize.py
View file

@ -2,11 +2,12 @@ import datetime
from dataclasses import is_dataclass, asdict
from pathlib import Path
from decimal import Decimal
from typing import Any, Optional, Callable, NamedTuple
from typing import Any, Optional, Callable, NamedTuple, Protocol
from functools import lru_cache
from .common import is_namedtuple
from .error import error_to_json
from .pytest import parametrize
# note: it would be nice to combine the 'asdict' and _default_encode to some function
# that takes a complex python object and returns JSON-compatible fields, while still
@ -16,6 +17,8 @@ from .error import error_to_json
DefaultEncoder = Callable[[Any], Any]
Dumps = Callable[[Any], str]
def _default_encode(obj: Any) -> Any:
"""
@ -75,22 +78,29 @@ def _dumps_factory(**kwargs) -> Callable[[Any], str]:
kwargs["default"] = use_default
try:
import orjson
prefer_factory: Optional[str] = kwargs.pop('_prefer_factory', None)
def orjson_factory() -> Optional[Dumps]:
try:
import orjson
except ModuleNotFoundError:
return None
# todo: add orjson.OPT_NON_STR_KEYS? would require some bitwise ops
# most keys are typically attributes from a NT/Dataclass,
# so most seem to work: https://github.com/ijl/orjson#opt_non_str_keys
def _orjson_dumps(obj: Any) -> str:
def _orjson_dumps(obj: Any) -> str: # TODO rename?
# orjson returns json as bytes, encode to string
return orjson.dumps(obj, **kwargs).decode('utf-8')
return _orjson_dumps
except ModuleNotFoundError:
pass
try:
from simplejson import dumps as simplejson_dumps
def simplejson_factory() -> Optional[Dumps]:
try:
from simplejson import dumps as simplejson_dumps
except ModuleNotFoundError:
return None
# if orjson couldn't be imported, try simplejson
# This is included for compatibility reasons because orjson
# is rust-based and compiling on rarer architectures may not work
@ -105,18 +115,37 @@ def _dumps_factory(**kwargs) -> Callable[[Any], str]:
return _simplejson_dumps
except ModuleNotFoundError:
pass
def stdlib_factory() -> Optional[Dumps]:
import json
from .warnings import high
import json
from .warnings import high
high(
"You might want to install 'orjson' to support serialization for lots more types! If that does not work for you, you can install 'simplejson' instead"
)
high("You might want to install 'orjson' to support serialization for lots more types! If that does not work for you, you can install 'simplejson' instead")
def _stdlib_dumps(obj: Any) -> str:
return json.dumps(obj, **kwargs)
def _stdlib_dumps(obj: Any) -> str:
return json.dumps(obj, **kwargs)
return _stdlib_dumps
return _stdlib_dumps
factories = {
'orjson': orjson_factory,
'simplejson': simplejson_factory,
'stdlib': stdlib_factory,
}
if prefer_factory is not None:
factory = factories[prefer_factory]
res = factory()
assert res is not None, prefer_factory
return res
for factory in factories.values():
res = factory()
if res is not None:
return res
else:
raise RuntimeError("Should not happen!")
def dumps(
@ -154,8 +183,17 @@ def dumps(
return _dumps_factory(default=default, **kwargs)(obj)
def test_serialize_fallback() -> None:
import json as jsn # dont cause possible conflicts with module code
@parametrize('factory', ['orjson', 'simplejson', 'stdlib'])
def test_dumps(factory: str) -> None:
import pytest
orig_dumps = globals()['dumps'] # hack to prevent error from using local variable before declaring
def dumps(*args, **kwargs) -> str:
kwargs['_prefer_factory'] = factory
return orig_dumps(*args, **kwargs)
import json as json_builtin # dont cause possible conflicts with module code
# can't use a namedtuple here, since the default json.dump serializer
# serializes namedtuples as tuples, which become arrays
@ -166,36 +204,12 @@ def test_serialize_fallback() -> None:
# the lru_cache'd warning may have already been sent,
# so checking may be nondeterministic?
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
res = jsn.loads(dumps(X))
res = json_builtin.loads(dumps(X))
assert res == [5, 5.0]
# this needs to be defined here to prevent a mypy bug
# see https://github.com/python/mypy/issues/7281
class _A(NamedTuple):
x: int
y: float
def test_nt_serialize() -> None:
import json as jsn # dont cause possible conflicts with module code
import orjson # import to make sure this is installed
res: str = dumps(_A(x=1, y=2.0))
assert res == '{"x":1,"y":2.0}'
# test orjson option kwarg
data = {datetime.date(year=1970, month=1, day=1): 5}
res2 = jsn.loads(dumps(data, option=orjson.OPT_NON_STR_KEYS))
assert res2 == {'1970-01-01': 5}
def test_default_serializer() -> None:
import pytest
import json as jsn # dont cause possible conflicts with module code
class Unserializable:
def __init__(self, x: int):
self.x = x
@ -209,7 +223,7 @@ def test_default_serializer() -> None:
def _serialize(self) -> Any:
return {"x": self.x, "y": self.y}
res = jsn.loads(dumps(WithUnderscoreSerialize(6)))
res = json_builtin.loads(dumps(WithUnderscoreSerialize(6)))
assert res == {"x": 6, "y": 6.0}
# test passing additional 'default' func
@ -221,5 +235,26 @@ def test_default_serializer() -> None:
# this serializes both Unserializable, which is a custom type otherwise
# not handled, and timedelta, which is handled by the '_default_encode'
# in the 'wrapped_default' function
res2 = jsn.loads(dumps(Unserializable(10), default=_serialize_with_default))
res2 = json_builtin.loads(dumps(Unserializable(10), default=_serialize_with_default))
assert res2 == {"x": 10, "y": 10.0}
if factory == 'orjson':
import orjson
# test orjson option kwarg
data = {datetime.date(year=1970, month=1, day=1): 5}
res2 = json_builtin.loads(dumps(data, option=orjson.OPT_NON_STR_KEYS))
assert res2 == {'1970-01-01': 5}
@parametrize('factory', ['orjson', 'simplejson'])
def test_dumps_namedtuple(factory: str) -> None:
import json as json_builtin # dont cause possible conflicts with module code
import orjson # import to make sure this is installed
class _A(NamedTuple):
x: int
y: float
res: str = dumps(_A(x=1, y=2.0), _prefer_factory=factory)
assert json_builtin.loads(res) == {'x': 1, 'y': 2.0}

5
setup.py
View file

@ -47,13 +47,16 @@ def main() -> None:
install_requires=INSTALL_REQUIRES,
extras_require={
'testing': [
'pytest<8', # FIXME <8 is temporary workaround till we fix collection with pytest 8; see https://docs.pytest.org/en/stable/changelog.html#collection-changes
'pytest',
'ruff',
'mypy',
'lxml', # for mypy coverage
# used in some tests.. although shouldn't rely on it
'pandas',
'orjson', # for my.core.serialize and denylist
'simplejson', # for my.core.serialize
],
'optional': [
# todo document these?

1
tests/serialize.py
View file

@ -1 +0,0 @@
from my.core.serialize import *

23
tests/serialize_simplejson.py
View file

@ -1,23 +0,0 @@
'''
This file should only run when simplejson is installed,
but orjson is not installed to check compatibility
'''
# none of these should fail
import json
import simplejson
import pytest
from my.core.serialize import dumps, _A
def test_simplejson_fallback() -> None:
# should fail to import
with pytest.raises(ModuleNotFoundError):
import orjson
# simplejson should serialize namedtuple properly
res: str = dumps(_A(x=1, y=2.0))
assert json.loads(res) == {"x": 1, "y": 2.0}

19
tox.ini
View file

@ -34,14 +34,11 @@ commands =
commands =
{envpython} -m pip install --use-pep517 -e .[testing]
# seems that denylist tests rely on it? ideally we should get rid of this in tests-core
{envpython} -m pip install orjson
{envpython} -m pytest \
# importlib is the new suggested import-mode
# without it test package names end up as core.tests.* instead of my.core.tests.*
--import-mode=importlib \
--pyargs my.core \
--pyargs {[testenv]package_name}.core \
# ignore orgmode because it imports orgparse
# tbh not sure if it even belongs to core, maybe move somewhere else..
# same with pandas?
@ -49,9 +46,6 @@ commands =
# causes error during test collection on 3.8
# dataset is deprecated anyway so whatever
--ignore my/core/dataset.py \
# this test uses orjson which is an optional dependency
# it would be covered by tests-all
-k 'not test_nt_serialize' \
{posargs}
@ -63,14 +57,7 @@ setenv = MY_CONFIG = nonexistent
commands =
{envpython} -m pip install --use-pep517 -e .[testing]
# installed to test my.core.serialize while using simplejson and not orjson
{envpython} -m pip install simplejson
{envpython} -m pytest \
tests/serialize_simplejson.py \
{posargs}
{envpython} -m pip install cachew
{envpython} -m pip install orjson
{envpython} -m my.core module install my.location.google
{envpython} -m pip install ijson # optional dependency
@ -103,9 +90,7 @@ commands =
{envpython} -m pytest tests \
# ignore some tests which might take a while to run on ci..
--ignore tests/takeout.py \
--ignore tests/extra/polar.py \
# dont run simplejson compatibility test since orjson is now installed
--ignore tests/serialize_simplejson.py \
--ignore tests/extra/polar.py
{posargs}