From 0593c690566ccbbf54aea24f3e0ce3613e6603da Mon Sep 17 00:00:00 2001 From: Sean Breckenridge Date: Tue, 16 Mar 2021 22:32:38 -0700 Subject: [PATCH] basic orjson serialize, json.dumps fallback --- my/core/common.py | 14 ++++++++++---- my/core/serialize.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 my/core/serialize.py diff --git a/my/core/common.py b/my/core/common.py index 6e2251b..d2a18a7 100644 --- a/my/core/common.py +++ b/my/core/common.py @@ -531,7 +531,13 @@ def test_guess_datetime() -> None: # TODO test @property? -def asdict(thing) -> Json: +def is_namedtuple(thing: Any) -> bool: + # basic check to see if this is a namedtuple-like + _asdict = getattr(thing, '_asdict', None) + return _asdict and callable(_asdict) + + +def asdict(thing: Any) -> Json: # todo primitive? # todo exception? if isinstance(thing, dict): @@ -539,9 +545,9 @@ def asdict(thing) -> Json: import dataclasses as D if D.is_dataclass(thing): return D.asdict(thing) - # must be a NT otherwise? - # todo add a proper check.. () - return thing._asdict() + if is_namedtuple(thing): + return thing._asdict() + raise TypeError(f'Could not convert object {thing} to dict') # todo not sure about naming diff --git a/my/core/serialize.py b/my/core/serialize.py new file mode 100644 index 0000000..7757de2 --- /dev/null +++ b/my/core/serialize.py @@ -0,0 +1,40 @@ +from typing import Any + +from .common import is_namedtuple +from .error import error_to_json + +# note: it would be nice to combine the 'asdict' and _orjson_default to some function +# that takes a complex python object and returns JSON-compatible fields, while still +# being a dictionary. +# a workaround is to encode with dumps below and then json.loads it immediately + + +def _orjson_default(obj: Any) -> Any: + """ + Encodes complex python datatypes to simpler representations, + before they're serialized to JSON string + """ + # orjson doesn't serialize namedtuples to avoid serializing + # them as tuples (arrays), since they're technically a subclass + if is_namedtuple(obj): + return obj._asdict() + if isinstance(obj, Exception): + err = error_to_json(obj) + # remove unrelated dt key? maybe error_to_json should be refactored? + err.pop('dt', None) + return err + raise TypeError(f"Could not serialize object of type {obj.__type__.__name__}") + + +def dumps(obj: Any) -> str: + try: + import orjson + # serialize 'b'"1970-01-01T00:00:00"' instead of b'"1970-01-01T00:00:00.000000" + opts = orjson.OPT_OMIT_MICROSECONDS + json_bytes = orjson.dumps(obj, default=_orjson_default, option=opts) + return json_bytes.decode('utf-8') + except ModuleNotFoundError: + import warnings + warnings.warn("You might want to install 'orjson' to support serialization for lots more types!") + import json + return json.dumps(obj, default=_orjson_default)