core.common: move Json, datetime_aware, datetime_naive, is_namedtuple, asdict to my.core.types

my/core/__init__.py

@@ -2,9 +2,12 @@
 from typing import TYPE_CHECKING
 
 from .common import get_files, PathIsh, Paths
-from .common import Json
 from .stats import stat, Stats
-from .common import datetime_naive, datetime_aware
+from .types import (
+    Json,
+    datetime_aware,
+    datetime_naive,
+)
 from .compat import assert_never
 from .utils.itertools import warn_if_empty
 

my/core/common.py

@@ -1,12 +1,8 @@
 from glob import glob as do_glob
 from pathlib import Path
-from datetime import datetime
-from dataclasses import is_dataclass, asdict as dataclasses_asdict
 import os
 from typing import (
-    Any,
     Callable,
-    Dict,
     Iterable,
     List,
     Sequence,
@@ -116,9 +112,6 @@ def get_files(
     return tuple(paths)
 
 
-Json = Dict[str, Any]
-
-
 from typing import TypeVar, Callable, Generic
 
 _R = TypeVar('_R')
@@ -141,11 +134,6 @@ class classproperty(Generic[_R]):
 #     def __get__(self) -> _R:
 #         return self.f()
 
-# for now just serves documentation purposes... but one day might make it statically verifiable where possible?
-# TODO e.g. maybe use opaque mypy alias?
-datetime_naive = datetime
-datetime_aware = datetime
-
 
 import re
 # https://stackoverflow.com/a/295466/706389
@@ -154,25 +142,6 @@ def get_valid_filename(s: str) -> str:
     return re.sub(r'(?u)[^-\w.]', '', s)
 
 
-def is_namedtuple(thing: Any) -> bool:
-    # basic check to see if this is namedtuple-like
-    _asdict = getattr(thing, '_asdict', None)
-    return (_asdict is not None) and callable(_asdict)
-
-
-def asdict(thing: Any) -> Json:
-    # todo primitive?
-    # todo exception?
-    if isinstance(thing, dict):
-        return thing
-    if is_dataclass(thing):
-        assert not isinstance(thing, type)  # to help mypy
-        return dataclasses_asdict(thing)
-    if is_namedtuple(thing):
-        return thing._asdict()
-    raise TypeError(f'Could not convert object {thing} to dict')
-
-
 # TODO deprecate and suggest to use one from my.core directly? not sure
 from .utils.itertools import unique_everseen
 
@@ -182,8 +151,6 @@ from .utils.itertools import unique_everseen
 ## in principle, warnings.deprecated decorator should cooperate with mypy, but doesn't look like it works atm?
 ## perhaps it doesn't work when it's used from typing_extensions
 
-from .compat import Never
-
 if not TYPE_CHECKING:
 
     @deprecated('use my.core.compat.assert_never instead')
@@ -243,12 +210,26 @@ if not TYPE_CHECKING:
     # todo wrap these in deprecated decorator as well?
     from .cachew import mcachew  # noqa: F401
 
+    # TODO hmm how to deprecate these in runtime?
+    # tricky cause they are actually classes/types
+
     from typing import Literal  # noqa: F401
 
-    # TODO hmm how to deprecate it in runtime? tricky cause it's actually a class?
+    from .stats import Stats
+    from .types import (
+        Json,
+        datetime_naive,
+        datetime_aware,
+    )
+
     tzdatetime = datetime_aware
 else:
-    tzdatetime = Never  # makes it invalid as a type while working in runtime
+    from .compat import Never
 
+    # make these invalid during type check while working in runtime
     Stats = Never
+    tzdatetime = Never
+    Json = Never
+    datetime_naive = Never
+    datetime_aware = Never
 ###

my/core/error.py

@@ -6,6 +6,8 @@ See https://beepb00p.xyz/mypy-error-handling.html#kiss for more detail
 from itertools import tee
 from typing import Union, TypeVar, Iterable, List, Tuple, Type, Optional, Callable, Any, cast, Iterator, Literal
 
+from .types import Json
+
 
 T = TypeVar('T')
 E = TypeVar('E', bound=Exception) # TODO make covariant?
@@ -176,7 +178,6 @@ def extract_error_datetime(e: Exception) -> Optional[datetime]:
 
 
 import traceback
-from .common import Json
 def error_to_json(e: Exception) -> Json:
     estr = ''.join(traceback.format_exception(Exception, e, e.__traceback__))
     return {'error': estr}

my/core/influxdb.py

@@ -6,7 +6,8 @@ from .internal import assert_subpackage; assert_subpackage(__name__)
 
 from typing import Iterable, Any, Optional, Dict
 
-from .common import LazyLogger, asdict, Json
+from .common import LazyLogger
+from .types import asdict, Json
 
 
 logger = LazyLogger(__name__)

my/core/pandas.py

@@ -13,7 +13,8 @@ from typing import TYPE_CHECKING, Any, Iterable, Type, Dict, Literal, Callable,
 from decorator import decorator
 
 from . import warnings, Res
-from .common import LazyLogger, Json, asdict
+from .common import LazyLogger
+from .types import Json, asdict
 from .error import error_to_json, extract_error_datetime
 
 

my/core/query.py

@@ -14,8 +14,8 @@ from typing import TypeVar, Tuple, Optional, Union, Callable, Iterable, Iterator
 
 import more_itertools
 
-import my.core.error as err
+from . import error as err
-from .common import is_namedtuple
+from .types import is_namedtuple
 from .error import Res, unwrap
 from .warnings import low
 

my/core/serialize.py

@@ -5,8 +5,8 @@ from decimal import Decimal
 from typing import Any, Optional, Callable, NamedTuple
 from functools import lru_cache
 
-from .common import is_namedtuple
 from .error import error_to_json
+from .types import is_namedtuple
 from .pytest import parametrize
 
 # note: it would be nice to combine the 'asdict' and _default_encode to some function

my/core/stats.py

@@ -24,6 +24,8 @@ from typing import (
     cast,
 )
 
+from .types import asdict
+
 
 Stats = Dict[str, Any]
 
@@ -432,8 +434,6 @@ def test_stat_iterable() -> None:
 
 # experimental, not sure about it..
 def _guess_datetime(x: Any) -> Optional[datetime]:
-    from .common import asdict  # avoid circular imports
-
     # todo hmm implement without exception..
     try:
         d = asdict(x)

my/core/time.py

@@ -3,7 +3,7 @@ from typing import Sequence, Dict
 
 import pytz
 
-from .common import datetime_aware, datetime_naive
+from .types import datetime_aware, datetime_naive
 
 
 def user_forced() -> Sequence[str]:

my/core/types.py

@@ -0,0 +1,36 @@
+from .internal import assert_subpackage; assert_subpackage(__name__)
+
+from dataclasses import is_dataclass, asdict as dataclasses_asdict
+from datetime import datetime
+from typing import (
+    Any,
+    Dict,
+)
+
+
+Json = Dict[str, Any]
+
+
+# for now just serves documentation purposes... but one day might make it statically verifiable where possible?
+# TODO e.g. maybe use opaque mypy alias?
+datetime_naive = datetime
+datetime_aware = datetime
+
+
+def is_namedtuple(thing: Any) -> bool:
+    # basic check to see if this is namedtuple-like
+    _asdict = getattr(thing, '_asdict', None)
+    return (_asdict is not None) and callable(_asdict)
+
+
+def asdict(thing: Any) -> Json:
+    # todo primitive?
+    # todo exception?
+    if isinstance(thing, dict):
+        return thing
+    if is_dataclass(thing):
+        assert not isinstance(thing, type)  # to help mypy
+        return dataclasses_asdict(thing)
+    if is_namedtuple(thing):
+        return thing._asdict()
+    raise TypeError(f'Could not convert object {thing} to dict')

my/goodreads.py

@@ -7,7 +7,7 @@ REQUIRES = [
 
 
 from dataclasses import dataclass
-from my.core import Paths
+from my.core import datetime_aware, Paths
 from my.config import goodreads as user_config
 
 @dataclass
@@ -61,7 +61,6 @@ def books() -> Iterator[dal.Book]:
 #######
 # todo ok, not sure these really belong here...
 
-from my.core.common import datetime_aware
 @dataclass
 class Event:
     dt: datetime_aware

my/lastfm.py

@@ -3,7 +3,7 @@ Last.fm scrobbles
 '''
 
 from dataclasses import dataclass
-from my.core import Paths, make_logger
+from my.core import Paths, Json, make_logger, get_files
 from my.config import lastfm as user_config
 
 
@@ -28,7 +28,6 @@ from pathlib import Path
 from typing import NamedTuple, Sequence, Iterable
 
 from my.core.cachew import mcachew
-from my.core.common import Json, get_files
 
 
 def inputs() -> Sequence[Path]:

my/runnerup.py

@@ -10,9 +10,8 @@ from datetime import timedelta
 from pathlib import Path
 from typing import Iterable
 
-from .core import Res, get_files
+from my.core import Res, get_files, Json
-from .core.common import Json
+from my.core.compat import fromisoformat
-from .core.compat import fromisoformat
 
 import tcxparser  # type: ignore[import-untyped]
 

my/time/tz/common.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Callable, Literal, cast
 
-from my.core.common import datetime_aware
+from my.core import datetime_aware
 
 
 '''

my/time/tz/main.py

@@ -1,8 +1,10 @@
 '''
 Timezone data provider, used to localize timezone-unaware timestamps for other modules
 '''
+
 from datetime import datetime
-from my.core.common import datetime_aware
+
+from my.core import datetime_aware
 
 # todo hmm, kwargs isn't mypy friendly.. but specifying types would require duplicating default args. uhoh
 def localize(dt: datetime, **kwargs) -> datetime_aware: