core: cleanup deprecations, exclude from type checking and show runtime warnings

among affected things:

- core.common.assert_never
- core.common.cproperty
- core.common.isoparse
- core.common.mcachew
- core.common.the
- core.common.tzdatetime
- core.compat.sqlite_backup

my/bluemaestro.py

@@ -21,7 +21,7 @@ from my.core import (
     Stats,
     influxdb,
 )
-from my.core.common import mcachew
+from my.core.cachew import mcachew
 from my.core.error import unwrap
 from my.core.pandas import DataFrameT, as_dataframe
 from my.core.sqlite import sqlite_connect_immutable

my/browser/export.py

@@ -16,7 +16,7 @@ from my.core import (
     make_logger,
     stat,
 )
-from my.core.common import mcachew
+from my.core.cachew import mcachew
 
 from browserexport.merge import read_and_merge, Visit
 

my/coding/commits.py

@@ -14,8 +14,7 @@ from typing import List, Optional, Iterator, Set, Sequence, cast
 
 
 from my.core import PathIsh, LazyLogger, make_config
-from my.core.cachew import cache_dir
+from my.core.cachew import cache_dir, mcachew
-from my.core.common import mcachew
 from my.core.warnings import high
 
 

my/core/common.py

@@ -14,7 +14,6 @@ from typing import (
     Iterable,
     Iterator,
     List,
-    NoReturn,
     Optional,
     Sequence,
     TYPE_CHECKING,
@@ -70,17 +69,6 @@ T = TypeVar('T')
 K = TypeVar('K')
 V = TypeVar('V')
 
-# TODO deprecate? more_itertools.one should be used
-def the(l: Iterable[T]) -> T:
-    it = iter(l)
-    try:
-        first = next(it)
-    except StopIteration:
-        raise RuntimeError('Empty iterator?')
-    assert all(e == first for e in it)
-    return first
-
-
 # TODO more_itertools.bucket?
 def group_by_key(l: Iterable[T], key: Callable[[T], K]) -> Dict[K, List[T]]:
     res: Dict[K, List[T]] = {}
@@ -322,14 +310,6 @@ datetime_naive = datetime
 datetime_aware = datetime
 
 
-# TODO deprecate
-tzdatetime = datetime_aware
-
-
-# TODO deprecate (although could be used in modules)
-from .compat import fromisoformat as isoparse
-
-
 import re
 # https://stackoverflow.com/a/295466/706389
 def get_valid_filename(s: str) -> str:
@@ -554,7 +534,7 @@ def test_guess_datetime() -> None:
     from dataclasses import dataclass
     from typing import NamedTuple
 
-    dd = isoparse('2021-02-01T12:34:56Z')
+    dd = compat.fromisoformat('2021-02-01T12:34:56Z')
 
     # ugh.. https://github.com/python/mypy/issues/7281
     A = NamedTuple('A', [('x', int)])
@@ -690,15 +670,41 @@ def unique_everseen(
     return more_itertools.unique_everseen(iterable=iterable, key=key)
 
 
-## legacy imports, keeping them here for backwards compatibility
+### legacy imports, keeping them here for backwards compatibility
 ## hiding behind TYPE_CHECKING so it works in runtime
 ## in principle, warnings.deprecated decorator should cooperate with mypy, but doesn't look like it works atm?
 ## perhaps it doesn't work when it's used from typing_extensions
 if not TYPE_CHECKING:
-    assert_never = deprecated('use my.core.compat.assert_never instead')(compat.assert_never)
 
-# TODO wrap in deprecated decorator as well?
+    @deprecated('use my.core.compat.assert_never instead')
-from functools import cached_property as cproperty
+    def assert_never(*args, **kwargs):
-from typing import Literal
+        return compat.assert_never(*args, **kwargs)
-from .cachew import mcachew
+
-##
+    @deprecated('use my.core.compat.fromisoformat instead')
+    def isoparse(*args, **kwargs):
+        return compat.fromisoformat(*args, **kwargs)
+
+    @deprecated('use more_itertools.one instead')
+    def the(*args, **kwargs):
+        import more_itertools
+
+        return more_itertools.one(*args, **kwargs)
+
+    @deprecated('use functools.cached_property instead')
+    def cproperty(*args, **kwargs):
+        import functools
+
+        return functools.cached_property(*args, **kwargs)
+
+    # todo wrap these in deprecated decorator as well?
+    from .cachew import mcachew  # noqa: F401
+
+    from typing import Literal  # noqa: F401
+
+    # TODO hmm how to deprecate it in runtime? tricky cause it's actually a class?
+    tzdatetime = datetime_aware
+else:
+    from .compat import Never
+
+    tzdatetime = Never  # makes it invalid as a type while working in runtime
+###

my/core/compat.py

@@ -2,56 +2,58 @@
 Contains backwards compatibility helpers for different python versions.
 If something is relevant to HPI itself, please put it in .hpi_compat instead
 '''
-import os
+
 import sys
 from typing import TYPE_CHECKING
 
 
-windows = os.name == 'nt'
+if sys.version_info[:2] >= (3, 13):
+    from warnings import deprecated
+else:
+    from typing_extensions import deprecated
 
 
 # keeping just for backwards compatibility, used to have compat implementation for 3.6
-import sqlite3
+if not TYPE_CHECKING:
-def sqlite_backup(*, source: sqlite3.Connection, dest: sqlite3.Connection, **kwargs) -> None:
+    import sqlite3
+
+    @deprecated('use .backup method on sqlite3.Connection directly instead')
+    def sqlite_backup(*, source: sqlite3.Connection, dest: sqlite3.Connection, **kwargs) -> None:
+        # TODO warn here?
         source.backup(dest, **kwargs)
 
 
 # can remove after python3.9 (although need to keep the method itself for bwd compat)
 def removeprefix(text: str, prefix: str) -> str:
     if text.startswith(prefix):
-        return text[len(prefix):]
+        return text[len(prefix) :]
     return text
 
 
-## used to have compat function before 3.8 for these
+## used to have compat function before 3.8 for these, keeping for runtime back compatibility
-from functools import cached_property
+if not TYPE_CHECKING:
-from typing import Literal, Protocol, TypedDict
+    from functools import cached_property
+    from typing import Literal, Protocol, TypedDict
+else:
+    from typing_extensions import Literal, Protocol, TypedDict
 ##
 
 
 if sys.version_info[:2] >= (3, 10):
     from typing import ParamSpec
 else:
-    if TYPE_CHECKING:
     from typing_extensions import ParamSpec
-    else:
-        from typing import NamedTuple, Any
-        # erm.. I guess as long as it's not crashing, whatever...
-        class _ParamSpec:
-            def __call__(self, args):
-                class _res:
-                    args = None
-                    kwargs = None
-                return _res
-        ParamSpec = _ParamSpec()
 
 
 # bisect_left doesn't have a 'key' parameter (which we use)
 # till python3.10
 if sys.version_info[:2] <= (3, 9):
     from typing import List, TypeVar, Any, Optional, Callable
+
     X = TypeVar('X')
+
     # copied from python src
+    # fmt: off
     def bisect_left(a: List[Any], x: Any, lo: int=0, hi: Optional[int]=None, *, key: Optional[Callable[..., Any]]=None) -> int:
         if lo < 0:
             raise ValueError('lo must be non-negative')
@@ -74,19 +76,22 @@ if sys.version_info[:2] <= (3, 9):
                 else:
                     hi = mid
         return lo
+    # fmt: on
+
 else:
     from bisect import bisect_left
 
 
 from datetime import datetime
+
 if sys.version_info[:2] >= (3, 11):
     fromisoformat = datetime.fromisoformat
 else:
-    def fromisoformat(date_string: str) -> datetime:
+    # fromisoformat didn't support Z as "utc" before 3.11
-        # didn't support Z as "utc" before 3.11
-        if date_string.endswith('Z'):
-            # NOTE: can be removed from 3.11?
     # https://docs.python.org/3/library/datetime.html#datetime.datetime.fromisoformat
+
+    def fromisoformat(date_string: str) -> datetime:
+        if date_string.endswith('Z'):
             date_string = date_string[:-1] + '+00:00'
         return datetime.fromisoformat(date_string)
 
@@ -94,6 +99,7 @@ else:
 def test_fromisoformat() -> None:
     from datetime import timezone
 
+    # fmt: off
     # feedbin has this format
     assert fromisoformat('2020-05-01T10:32:02.925961Z') == datetime(
         2020, 5, 1, 10, 32, 2, 925961, timezone.utc,
@@ -108,6 +114,7 @@ def test_fromisoformat() -> None:
     assert fromisoformat('2020-11-30T00:53:12Z') == datetime(
         2020, 11, 30, 0, 53, 12, 0, timezone.utc,
     )
+    # fmt: on
 
     # arbtt has this format (sometimes less/more than 6 digits in milliseconds)
     # TODO doesn't work atm, not sure if really should be supported...
@@ -123,13 +130,13 @@ else:
     NoneType = type(None)
 
 
-if sys.version_info[:2] >= (3, 13):
-    from warnings import deprecated
-else:
-    from typing_extensions import deprecated
-
-
 if sys.version_info[:2] >= (3, 11):
     from typing import assert_never
 else:
     from typing_extensions import assert_never
+
+
+if sys.version_info[:2] >= (3, 11):
+    from typing import Never
+else:
+    from typing_extensions import Never

my/core/tests/test_cachew.py

@@ -10,7 +10,7 @@ def test_cachew() -> None:
 
     settings.ENABLE = True  # by default it's off in tests (see conftest.py)
 
-    from my.core.common import mcachew
+    from my.core.cachew import mcachew
 
     called = 0
 
@@ -36,7 +36,7 @@ def test_cachew_dir_none() -> None:
     settings.ENABLE = True  # by default it's off in tests (see conftest.py)
 
     from my.core.cachew import cache_dir
-    from my.core.common import mcachew
+    from my.core.cachew import mcachew
     from my.core.core_config import _reset_config as reset
 
     with reset() as cc:

my/core/tests/test_get_files.py

@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
 import zipfile
 
 from ..common import get_files
-from ..compat import windows
 from ..kompress import CPath, ZipPath
 
 import pytest
@@ -56,8 +55,9 @@ def test_single_file() -> None:
     '''
     assert get_files('/tmp/hpi_test/file.ext') == (Path('/tmp/hpi_test/file.ext'),)
 
+    is_windows = os.name == 'nt'
     "if the path starts with ~, we expand it"
-    if not windows:  # windows doesn't have bashrc.. ugh
+    if not is_windows:  # windows doesn't have bashrc.. ugh
         assert get_files('~/.bashrc') == (Path('~').expanduser() / '.bashrc',)
 
 

my/emfit/__init__.py

@@ -21,8 +21,7 @@ from my.core import (
     Res,
     Stats,
 )
-from my.core.common import mcachew
+from my.core.cachew import cache_dir, mcachew
-from my.core.cachew import cache_dir
 from my.core.error import set_error_datetime, extract_error_datetime
 from my.core.pandas import DataFrameT
 

my/github/ghexport.py

@@ -42,10 +42,11 @@ except ModuleNotFoundError as e:
 ############################
 
 from functools import lru_cache
+from pathlib import Path
 from typing import Tuple, Dict, Sequence, Optional
 
-from my.core import get_files, Path, LazyLogger
+from my.core import get_files, LazyLogger
-from my.core.common import mcachew
+from my.core.cachew import mcachew
 
 from .common import Event, parse_dt, Results, EventIds
 

my/google/takeout/parser.py

@@ -19,7 +19,8 @@ import os
 from typing import List, Sequence, cast
 from pathlib import Path
 from my.core import make_config, dataclass
-from my.core.common import Stats, LazyLogger, mcachew, get_files, Paths
+from my.core.cachew import mcachew
+from my.core.common import Stats, LazyLogger, get_files, Paths
 from my.core.error import ErrorPolicy
 from my.core.structure import match_structure
 

my/lastfm.py

@@ -26,7 +26,8 @@ import json
 from pathlib import Path
 from typing import NamedTuple, Sequence, Iterable
 
-from my.core.common import mcachew, Json, get_files
+from my.core.cachew import mcachew
+from my.core.common import Json, get_files
 
 
 def inputs() -> Sequence[Path]:

my/location/google.py

@@ -19,8 +19,8 @@ import re
 # pip3 install geopy
 import geopy # type: ignore
 
-from ..core.common import LazyLogger, mcachew
+from my.core.common import LazyLogger
-from ..core.cachew import cache_dir
+from my.core.cachew import cache_dir, mcachew
 
 from my.core.warnings import high
 

my/location/google_takeout.py

@@ -9,7 +9,8 @@ from typing import Iterator
 from my.google.takeout.parser import events, _cachew_depends_on
 from google_takeout_parser.models import Location as GoogleLocation
 
-from my.core.common import mcachew, LazyLogger, Stats
+from my.core.cachew import mcachew
+from my.core.common import LazyLogger, stat, Stats
 from .common import Location
 
 logger = LazyLogger(__name__)
@@ -33,6 +34,4 @@ def locations() -> Iterator[Location]:
 
 
 def stats() -> Stats:
-    from my.core import stat
+    return stat(locations)
-
-    return {**stat(locations)}

my/location/google_takeout_semantic.py

@@ -13,7 +13,8 @@ from my.google.takeout.parser import events, _cachew_depends_on as _parser_cache
 from google_takeout_parser.models import PlaceVisit as SemanticLocation
 
 from my.core import dataclass, make_config
-from my.core.common import mcachew, LazyLogger, Stats
+from my.core.cachew import mcachew
+from my.core.common import LazyLogger, Stats, stat
 from my.core.error import Res
 from .common import Location
 
@@ -72,6 +73,4 @@ def locations() -> Iterator[Res[Location]]:
 
 
 def stats() -> Stats:
-    from my.core import stat
+    return stat(locations)
-
-    return {**stat(locations)}

my/location/gpslogger.py

@@ -27,7 +27,8 @@ from gpxpy.gpx import GPXXMLSyntaxException
 from more_itertools import unique_everseen
 
 from my.core import Stats, LazyLogger
-from my.core.common import get_files, mcachew
+from my.core.cachew import mcachew
+from my.core.common import get_files
 from .common import Location
 
 

my/orgmode.py

@@ -12,8 +12,7 @@ import re
 from typing import List, Sequence, Iterable, NamedTuple, Optional, Tuple
 
 from my.core import get_files
-from my.core.common import mcachew
+from my.core.cachew import cache_dir, mcachew
-from my.core.cachew import cache_dir
 from my.core.orgmode import collect
 
 from my.config import orgmode as user_config

my/pdfs.py

@@ -15,8 +15,9 @@ from typing import NamedTuple, List, Optional, Iterator, Sequence
 
 
 from my.core import LazyLogger, get_files, Paths, PathIsh
+from my.core.cachew import mcachew
 from my.core.cfg import Attrs, make_config
-from my.core.common import mcachew, group_by_key
+from my.core.common import group_by_key
 from my.core.error import Res, split_errors
 
 

my/photos/main.py

@@ -15,9 +15,9 @@ from typing import Optional, NamedTuple, Iterator, Iterable, List
 
 from geopy.geocoders import Nominatim  # type: ignore
 
-from ..core.common import LazyLogger, mcachew, fastermime
+from my.core.common import LazyLogger, fastermime
-from ..core.error import Res, sort_res_by
+from my.core.error import Res, sort_res_by
-from ..core.cachew import cache_dir
+from my.core.cachew import cache_dir, mcachew
 
 from my.config import photos as config  # type: ignore[attr-defined]
 

my/reddit/rexport.py

@@ -20,8 +20,8 @@ from my.core import (
     Paths,
     Stats,
 )
+from my.core.cachew import mcachew
 from my.core.cfg import make_config, Attrs
-from my.core.common import mcachew
 
 from my.config import reddit as uconfig
 

my/rescuetime.py

@@ -10,7 +10,7 @@ from datetime import timedelta
 from typing import Sequence, Iterable
 
 from my.core import get_files, make_logger
-from my.core.common import mcachew
+from my.core.cachew import mcachew
 from my.core.error import Res, split_errors
 
 from my.config import rescuetime as config

my/rss/feedbin.py

@@ -7,8 +7,8 @@ from my.config import feedbin as config
 from pathlib import Path
 from typing import Sequence
 
-from ..core.common import listify, get_files
+from my.core.common import listify, get_files
-from ..core.compat import fromisoformat
+from my.core.compat import fromisoformat
 from .common import Subscription
 
 
@@ -33,12 +33,10 @@ def parse_file(f: Path):
 from typing import Iterable
 from .common import SubscriptionState
 def states() -> Iterable[SubscriptionState]:
-    # meh
-    from dateutil.parser import isoparse
     for f in inputs():
         # TODO ugh. depends on my naming. not sure if useful?
         dts = f.stem.split('_')[-1]
-        dt = isoparse(dts)
+        dt = fromisoformat(dts)
         subs = parse_file(f)
         yield dt, subs
 

my/time/tz/common.py

@@ -1,7 +1,7 @@
 from datetime import datetime
-from typing import Callable, cast
+from typing import Callable, Literal, cast
 
-from ...core.common import tzdatetime, Literal
+from my.core.common import datetime_aware
 
 
 '''
@@ -30,7 +30,11 @@ def default_policy() -> TzPolicy:
         return 'keep'
 
 
-def localize_with_policy(lfun: Callable[[datetime], tzdatetime], dt: datetime, policy: TzPolicy=default_policy()) -> tzdatetime:
+def localize_with_policy(
+    lfun: Callable[[datetime], datetime_aware],
+    dt: datetime,
+    policy: TzPolicy=default_policy()
+) -> datetime_aware:
     tz = dt.tzinfo
     if tz is None:
         return lfun(dt)

my/time/tz/main.py

@@ -2,10 +2,10 @@
 Timezone data provider, used to localize timezone-unaware timestamps for other modules
 '''
 from datetime import datetime
-from ...core.common import tzdatetime
+from my.core.common import datetime_aware
 
 # todo hmm, kwargs isn't mypy friendly.. but specifying types would require duplicating default args. uhoh
-def localize(dt: datetime, **kwargs) -> tzdatetime:
+def localize(dt: datetime, **kwargs) -> datetime_aware:
     # todo document patterns for combining multiple data sources
     # e.g. see https://github.com/karlicoss/HPI/issues/89#issuecomment-716495136
     from . import via_location as L

my/time/tz/via_location.py

@@ -17,8 +17,8 @@ from typing import Iterator, Optional, Tuple, Any, List, Iterable, Set, Dict
 
 import pytz
 
+from my.core.cachew import mcachew
 from my.core import make_logger, stat, Stats, datetime_aware
-from my.core.common import mcachew
 from my.core.source import import_source
 from my.core.warnings import high