diff --git a/my/bluemaestro.py b/my/bluemaestro.py index 8f05aac..3e25cae 100644 --- a/my/bluemaestro.py +++ b/my/bluemaestro.py @@ -21,7 +21,7 @@ from my.core import ( Stats, influxdb, ) -from my.core.common import mcachew +from my.core.cachew import mcachew from my.core.error import unwrap from my.core.pandas import DataFrameT, as_dataframe from my.core.sqlite import sqlite_connect_immutable diff --git a/my/browser/export.py b/my/browser/export.py index ce5a6de..1b428b5 100644 --- a/my/browser/export.py +++ b/my/browser/export.py @@ -16,7 +16,7 @@ from my.core import ( make_logger, stat, ) -from my.core.common import mcachew +from my.core.cachew import mcachew from browserexport.merge import read_and_merge, Visit diff --git a/my/coding/commits.py b/my/coding/commits.py index 51f9222..dac3b1f 100644 --- a/my/coding/commits.py +++ b/my/coding/commits.py @@ -14,8 +14,7 @@ from typing import List, Optional, Iterator, Set, Sequence, cast from my.core import PathIsh, LazyLogger, make_config -from my.core.cachew import cache_dir -from my.core.common import mcachew +from my.core.cachew import cache_dir, mcachew from my.core.warnings import high diff --git a/my/core/common.py b/my/core/common.py index 9874bed..460a658 100644 --- a/my/core/common.py +++ b/my/core/common.py @@ -14,7 +14,6 @@ from typing import ( Iterable, Iterator, List, - NoReturn, Optional, Sequence, TYPE_CHECKING, @@ -70,17 +69,6 @@ T = TypeVar('T') K = TypeVar('K') V = TypeVar('V') -# TODO deprecate? more_itertools.one should be used -def the(l: Iterable[T]) -> T: - it = iter(l) - try: - first = next(it) - except StopIteration: - raise RuntimeError('Empty iterator?') - assert all(e == first for e in it) - return first - - # TODO more_itertools.bucket? def group_by_key(l: Iterable[T], key: Callable[[T], K]) -> Dict[K, List[T]]: res: Dict[K, List[T]] = {} @@ -322,14 +310,6 @@ datetime_naive = datetime datetime_aware = datetime -# TODO deprecate -tzdatetime = datetime_aware - - -# TODO deprecate (although could be used in modules) -from .compat import fromisoformat as isoparse - - import re # https://stackoverflow.com/a/295466/706389 def get_valid_filename(s: str) -> str: @@ -554,7 +534,7 @@ def test_guess_datetime() -> None: from dataclasses import dataclass from typing import NamedTuple - dd = isoparse('2021-02-01T12:34:56Z') + dd = compat.fromisoformat('2021-02-01T12:34:56Z') # ugh.. https://github.com/python/mypy/issues/7281 A = NamedTuple('A', [('x', int)]) @@ -690,15 +670,41 @@ def unique_everseen( return more_itertools.unique_everseen(iterable=iterable, key=key) -## legacy imports, keeping them here for backwards compatibility +### legacy imports, keeping them here for backwards compatibility ## hiding behind TYPE_CHECKING so it works in runtime ## in principle, warnings.deprecated decorator should cooperate with mypy, but doesn't look like it works atm? ## perhaps it doesn't work when it's used from typing_extensions if not TYPE_CHECKING: - assert_never = deprecated('use my.core.compat.assert_never instead')(compat.assert_never) -# TODO wrap in deprecated decorator as well? -from functools import cached_property as cproperty -from typing import Literal -from .cachew import mcachew -## + @deprecated('use my.core.compat.assert_never instead') + def assert_never(*args, **kwargs): + return compat.assert_never(*args, **kwargs) + + @deprecated('use my.core.compat.fromisoformat instead') + def isoparse(*args, **kwargs): + return compat.fromisoformat(*args, **kwargs) + + @deprecated('use more_itertools.one instead') + def the(*args, **kwargs): + import more_itertools + + return more_itertools.one(*args, **kwargs) + + @deprecated('use functools.cached_property instead') + def cproperty(*args, **kwargs): + import functools + + return functools.cached_property(*args, **kwargs) + + # todo wrap these in deprecated decorator as well? + from .cachew import mcachew # noqa: F401 + + from typing import Literal # noqa: F401 + + # TODO hmm how to deprecate it in runtime? tricky cause it's actually a class? + tzdatetime = datetime_aware +else: + from .compat import Never + + tzdatetime = Never # makes it invalid as a type while working in runtime +### diff --git a/my/core/compat.py b/my/core/compat.py index 2c1687d..d73c60c 100644 --- a/my/core/compat.py +++ b/my/core/compat.py @@ -2,56 +2,58 @@ Contains backwards compatibility helpers for different python versions. If something is relevant to HPI itself, please put it in .hpi_compat instead ''' -import os + import sys from typing import TYPE_CHECKING -windows = os.name == 'nt' +if sys.version_info[:2] >= (3, 13): + from warnings import deprecated +else: + from typing_extensions import deprecated # keeping just for backwards compatibility, used to have compat implementation for 3.6 -import sqlite3 -def sqlite_backup(*, source: sqlite3.Connection, dest: sqlite3.Connection, **kwargs) -> None: - source.backup(dest, **kwargs) +if not TYPE_CHECKING: + import sqlite3 + + @deprecated('use .backup method on sqlite3.Connection directly instead') + def sqlite_backup(*, source: sqlite3.Connection, dest: sqlite3.Connection, **kwargs) -> None: + # TODO warn here? + source.backup(dest, **kwargs) # can remove after python3.9 (although need to keep the method itself for bwd compat) def removeprefix(text: str, prefix: str) -> str: if text.startswith(prefix): - return text[len(prefix):] + return text[len(prefix) :] return text -## used to have compat function before 3.8 for these -from functools import cached_property -from typing import Literal, Protocol, TypedDict +## used to have compat function before 3.8 for these, keeping for runtime back compatibility +if not TYPE_CHECKING: + from functools import cached_property + from typing import Literal, Protocol, TypedDict +else: + from typing_extensions import Literal, Protocol, TypedDict ## if sys.version_info[:2] >= (3, 10): from typing import ParamSpec else: - if TYPE_CHECKING: - from typing_extensions import ParamSpec - else: - from typing import NamedTuple, Any - # erm.. I guess as long as it's not crashing, whatever... - class _ParamSpec: - def __call__(self, args): - class _res: - args = None - kwargs = None - return _res - ParamSpec = _ParamSpec() + from typing_extensions import ParamSpec # bisect_left doesn't have a 'key' parameter (which we use) # till python3.10 if sys.version_info[:2] <= (3, 9): from typing import List, TypeVar, Any, Optional, Callable + X = TypeVar('X') + # copied from python src + # fmt: off def bisect_left(a: List[Any], x: Any, lo: int=0, hi: Optional[int]=None, *, key: Optional[Callable[..., Any]]=None) -> int: if lo < 0: raise ValueError('lo must be non-negative') @@ -74,19 +76,22 @@ if sys.version_info[:2] <= (3, 9): else: hi = mid return lo + # fmt: on + else: from bisect import bisect_left from datetime import datetime + if sys.version_info[:2] >= (3, 11): fromisoformat = datetime.fromisoformat else: + # fromisoformat didn't support Z as "utc" before 3.11 + # https://docs.python.org/3/library/datetime.html#datetime.datetime.fromisoformat + def fromisoformat(date_string: str) -> datetime: - # didn't support Z as "utc" before 3.11 if date_string.endswith('Z'): - # NOTE: can be removed from 3.11? - # https://docs.python.org/3/library/datetime.html#datetime.datetime.fromisoformat date_string = date_string[:-1] + '+00:00' return datetime.fromisoformat(date_string) @@ -94,6 +99,7 @@ else: def test_fromisoformat() -> None: from datetime import timezone + # fmt: off # feedbin has this format assert fromisoformat('2020-05-01T10:32:02.925961Z') == datetime( 2020, 5, 1, 10, 32, 2, 925961, timezone.utc, @@ -108,6 +114,7 @@ def test_fromisoformat() -> None: assert fromisoformat('2020-11-30T00:53:12Z') == datetime( 2020, 11, 30, 0, 53, 12, 0, timezone.utc, ) + # fmt: on # arbtt has this format (sometimes less/more than 6 digits in milliseconds) # TODO doesn't work atm, not sure if really should be supported... @@ -123,13 +130,13 @@ else: NoneType = type(None) -if sys.version_info[:2] >= (3, 13): - from warnings import deprecated -else: - from typing_extensions import deprecated - - if sys.version_info[:2] >= (3, 11): from typing import assert_never else: from typing_extensions import assert_never + + +if sys.version_info[:2] >= (3, 11): + from typing import Never +else: + from typing_extensions import Never diff --git a/my/core/tests/test_cachew.py b/my/core/tests/test_cachew.py index 86344fd..5f7dd65 100644 --- a/my/core/tests/test_cachew.py +++ b/my/core/tests/test_cachew.py @@ -10,7 +10,7 @@ def test_cachew() -> None: settings.ENABLE = True # by default it's off in tests (see conftest.py) - from my.core.common import mcachew + from my.core.cachew import mcachew called = 0 @@ -36,7 +36,7 @@ def test_cachew_dir_none() -> None: settings.ENABLE = True # by default it's off in tests (see conftest.py) from my.core.cachew import cache_dir - from my.core.common import mcachew + from my.core.cachew import mcachew from my.core.core_config import _reset_config as reset with reset() as cc: diff --git a/my/core/tests/test_get_files.py b/my/core/tests/test_get_files.py index e9f216a..52e43f8 100644 --- a/my/core/tests/test_get_files.py +++ b/my/core/tests/test_get_files.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING import zipfile from ..common import get_files -from ..compat import windows from ..kompress import CPath, ZipPath import pytest @@ -56,8 +55,9 @@ def test_single_file() -> None: ''' assert get_files('/tmp/hpi_test/file.ext') == (Path('/tmp/hpi_test/file.ext'),) + is_windows = os.name == 'nt' "if the path starts with ~, we expand it" - if not windows: # windows doesn't have bashrc.. ugh + if not is_windows: # windows doesn't have bashrc.. ugh assert get_files('~/.bashrc') == (Path('~').expanduser() / '.bashrc',) diff --git a/my/emfit/__init__.py b/my/emfit/__init__.py index 30b693c..7fae8ea 100644 --- a/my/emfit/__init__.py +++ b/my/emfit/__init__.py @@ -21,8 +21,7 @@ from my.core import ( Res, Stats, ) -from my.core.common import mcachew -from my.core.cachew import cache_dir +from my.core.cachew import cache_dir, mcachew from my.core.error import set_error_datetime, extract_error_datetime from my.core.pandas import DataFrameT diff --git a/my/github/ghexport.py b/my/github/ghexport.py index d446c35..9dc8fd5 100644 --- a/my/github/ghexport.py +++ b/my/github/ghexport.py @@ -42,10 +42,11 @@ except ModuleNotFoundError as e: ############################ from functools import lru_cache +from pathlib import Path from typing import Tuple, Dict, Sequence, Optional -from my.core import get_files, Path, LazyLogger -from my.core.common import mcachew +from my.core import get_files, LazyLogger +from my.core.cachew import mcachew from .common import Event, parse_dt, Results, EventIds diff --git a/my/google/takeout/parser.py b/my/google/takeout/parser.py index 2322ef0..952c9b6 100644 --- a/my/google/takeout/parser.py +++ b/my/google/takeout/parser.py @@ -19,7 +19,8 @@ import os from typing import List, Sequence, cast from pathlib import Path from my.core import make_config, dataclass -from my.core.common import Stats, LazyLogger, mcachew, get_files, Paths +from my.core.cachew import mcachew +from my.core.common import Stats, LazyLogger, get_files, Paths from my.core.error import ErrorPolicy from my.core.structure import match_structure diff --git a/my/lastfm.py b/my/lastfm.py index 90484b4..64ef1b3 100644 --- a/my/lastfm.py +++ b/my/lastfm.py @@ -26,7 +26,8 @@ import json from pathlib import Path from typing import NamedTuple, Sequence, Iterable -from my.core.common import mcachew, Json, get_files +from my.core.cachew import mcachew +from my.core.common import Json, get_files def inputs() -> Sequence[Path]: diff --git a/my/location/google.py b/my/location/google.py index ed37231..c1539e7 100644 --- a/my/location/google.py +++ b/my/location/google.py @@ -19,8 +19,8 @@ import re # pip3 install geopy import geopy # type: ignore -from ..core.common import LazyLogger, mcachew -from ..core.cachew import cache_dir +from my.core.common import LazyLogger +from my.core.cachew import cache_dir, mcachew from my.core.warnings import high diff --git a/my/location/google_takeout.py b/my/location/google_takeout.py index a1c1403..2fac270 100644 --- a/my/location/google_takeout.py +++ b/my/location/google_takeout.py @@ -9,7 +9,8 @@ from typing import Iterator from my.google.takeout.parser import events, _cachew_depends_on from google_takeout_parser.models import Location as GoogleLocation -from my.core.common import mcachew, LazyLogger, Stats +from my.core.cachew import mcachew +from my.core.common import LazyLogger, stat, Stats from .common import Location logger = LazyLogger(__name__) @@ -33,6 +34,4 @@ def locations() -> Iterator[Location]: def stats() -> Stats: - from my.core import stat - - return {**stat(locations)} + return stat(locations) diff --git a/my/location/google_takeout_semantic.py b/my/location/google_takeout_semantic.py index b4f16db..014959c 100644 --- a/my/location/google_takeout_semantic.py +++ b/my/location/google_takeout_semantic.py @@ -13,7 +13,8 @@ from my.google.takeout.parser import events, _cachew_depends_on as _parser_cache from google_takeout_parser.models import PlaceVisit as SemanticLocation from my.core import dataclass, make_config -from my.core.common import mcachew, LazyLogger, Stats +from my.core.cachew import mcachew +from my.core.common import LazyLogger, Stats, stat from my.core.error import Res from .common import Location @@ -72,6 +73,4 @@ def locations() -> Iterator[Res[Location]]: def stats() -> Stats: - from my.core import stat - - return {**stat(locations)} + return stat(locations) diff --git a/my/location/gpslogger.py b/my/location/gpslogger.py index 29e2547..8fb59d0 100644 --- a/my/location/gpslogger.py +++ b/my/location/gpslogger.py @@ -27,7 +27,8 @@ from gpxpy.gpx import GPXXMLSyntaxException from more_itertools import unique_everseen from my.core import Stats, LazyLogger -from my.core.common import get_files, mcachew +from my.core.cachew import mcachew +from my.core.common import get_files from .common import Location diff --git a/my/orgmode.py b/my/orgmode.py index 8293b74..c27f5a7 100644 --- a/my/orgmode.py +++ b/my/orgmode.py @@ -12,8 +12,7 @@ import re from typing import List, Sequence, Iterable, NamedTuple, Optional, Tuple from my.core import get_files -from my.core.common import mcachew -from my.core.cachew import cache_dir +from my.core.cachew import cache_dir, mcachew from my.core.orgmode import collect from my.config import orgmode as user_config diff --git a/my/pdfs.py b/my/pdfs.py index 5355d8a..3305eca 100644 --- a/my/pdfs.py +++ b/my/pdfs.py @@ -15,8 +15,9 @@ from typing import NamedTuple, List, Optional, Iterator, Sequence from my.core import LazyLogger, get_files, Paths, PathIsh +from my.core.cachew import mcachew from my.core.cfg import Attrs, make_config -from my.core.common import mcachew, group_by_key +from my.core.common import group_by_key from my.core.error import Res, split_errors diff --git a/my/photos/main.py b/my/photos/main.py index c491ac1..622d475 100644 --- a/my/photos/main.py +++ b/my/photos/main.py @@ -13,11 +13,11 @@ import json from pathlib import Path from typing import Optional, NamedTuple, Iterator, Iterable, List -from geopy.geocoders import Nominatim # type: ignore +from geopy.geocoders import Nominatim # type: ignore -from ..core.common import LazyLogger, mcachew, fastermime -from ..core.error import Res, sort_res_by -from ..core.cachew import cache_dir +from my.core.common import LazyLogger, fastermime +from my.core.error import Res, sort_res_by +from my.core.cachew import cache_dir, mcachew from my.config import photos as config # type: ignore[attr-defined] diff --git a/my/reddit/rexport.py b/my/reddit/rexport.py index a7be39b..6a6be61 100644 --- a/my/reddit/rexport.py +++ b/my/reddit/rexport.py @@ -20,8 +20,8 @@ from my.core import ( Paths, Stats, ) +from my.core.cachew import mcachew from my.core.cfg import make_config, Attrs -from my.core.common import mcachew from my.config import reddit as uconfig diff --git a/my/rescuetime.py b/my/rescuetime.py index 75684d9..774b587 100644 --- a/my/rescuetime.py +++ b/my/rescuetime.py @@ -10,7 +10,7 @@ from datetime import timedelta from typing import Sequence, Iterable from my.core import get_files, make_logger -from my.core.common import mcachew +from my.core.cachew import mcachew from my.core.error import Res, split_errors from my.config import rescuetime as config diff --git a/my/rss/feedbin.py b/my/rss/feedbin.py index 6160abc..16d4417 100644 --- a/my/rss/feedbin.py +++ b/my/rss/feedbin.py @@ -7,8 +7,8 @@ from my.config import feedbin as config from pathlib import Path from typing import Sequence -from ..core.common import listify, get_files -from ..core.compat import fromisoformat +from my.core.common import listify, get_files +from my.core.compat import fromisoformat from .common import Subscription @@ -33,12 +33,10 @@ def parse_file(f: Path): from typing import Iterable from .common import SubscriptionState def states() -> Iterable[SubscriptionState]: - # meh - from dateutil.parser import isoparse for f in inputs(): # TODO ugh. depends on my naming. not sure if useful? dts = f.stem.split('_')[-1] - dt = isoparse(dts) + dt = fromisoformat(dts) subs = parse_file(f) yield dt, subs diff --git a/my/time/tz/common.py b/my/time/tz/common.py index e2c428d..107410a 100644 --- a/my/time/tz/common.py +++ b/my/time/tz/common.py @@ -1,7 +1,7 @@ from datetime import datetime -from typing import Callable, cast +from typing import Callable, Literal, cast -from ...core.common import tzdatetime, Literal +from my.core.common import datetime_aware ''' @@ -30,7 +30,11 @@ def default_policy() -> TzPolicy: return 'keep' -def localize_with_policy(lfun: Callable[[datetime], tzdatetime], dt: datetime, policy: TzPolicy=default_policy()) -> tzdatetime: +def localize_with_policy( + lfun: Callable[[datetime], datetime_aware], + dt: datetime, + policy: TzPolicy=default_policy() +) -> datetime_aware: tz = dt.tzinfo if tz is None: return lfun(dt) diff --git a/my/time/tz/main.py b/my/time/tz/main.py index 624d7aa..6180160 100644 --- a/my/time/tz/main.py +++ b/my/time/tz/main.py @@ -2,10 +2,10 @@ Timezone data provider, used to localize timezone-unaware timestamps for other modules ''' from datetime import datetime -from ...core.common import tzdatetime +from my.core.common import datetime_aware # todo hmm, kwargs isn't mypy friendly.. but specifying types would require duplicating default args. uhoh -def localize(dt: datetime, **kwargs) -> tzdatetime: +def localize(dt: datetime, **kwargs) -> datetime_aware: # todo document patterns for combining multiple data sources # e.g. see https://github.com/karlicoss/HPI/issues/89#issuecomment-716495136 from . import via_location as L diff --git a/my/time/tz/via_location.py b/my/time/tz/via_location.py index 612341a..b66ff8a 100644 --- a/my/time/tz/via_location.py +++ b/my/time/tz/via_location.py @@ -17,8 +17,8 @@ from typing import Iterator, Optional, Tuple, Any, List, Iterable, Set, Dict import pytz +from my.core.cachew import mcachew from my.core import make_logger, stat, Stats, datetime_aware -from my.core.common import mcachew from my.core.source import import_source from my.core.warnings import high