This commit is contained in:
Dima Gerasimov 2024-10-19 19:15:36 +01:00
parent bc7c3ac253
commit d1511929a8
33 changed files with 117 additions and 117 deletions

View file

@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
from .cfg import make_config from .cfg import make_config
from .common import PathIsh, Paths, get_files from .common import PathIsh, Paths, get_files
from .compat import assert_never from .compat import assert_never
from .error import Res, unwrap, notnone from .error import Res, notnone, unwrap
from .logging import ( from .logging import (
make_logger, make_logger,
) )

View file

@ -7,11 +7,12 @@ import shutil
import sys import sys
import tempfile import tempfile
import traceback import traceback
from collections.abc import Iterable, Sequence
from contextlib import ExitStack from contextlib import ExitStack
from itertools import chain from itertools import chain
from pathlib import Path from pathlib import Path
from subprocess import PIPE, CompletedProcess, Popen, check_call, run from subprocess import PIPE, CompletedProcess, Popen, check_call, run
from typing import Any, Callable, Iterable, List, Optional, Sequence, Type from typing import Any, Callable, List, Optional, Type
import click import click

View file

@ -7,10 +7,11 @@ from __future__ import annotations
import io import io
import pathlib import pathlib
import sys import sys
from collections.abc import Iterator, Sequence
from datetime import datetime from datetime import datetime
from functools import total_ordering from functools import total_ordering
from pathlib import Path from pathlib import Path
from typing import IO, Any, Iterator, Sequence, Union from typing import IO, Any, Union
PathIsh = Union[Path, str] PathIsh = Union[Path, str]

View file

@ -2,13 +2,13 @@ from .internal import assert_subpackage; assert_subpackage(__name__)
import logging import logging
import sys import sys
from collections.abc import Iterator
from contextlib import contextmanager from contextlib import contextmanager
from pathlib import Path from pathlib import Path
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Callable, Callable,
Iterator,
Optional, Optional,
Type, Type,
TypeVar, TypeVar,
@ -21,7 +21,6 @@ import appdirs # type: ignore[import-untyped]
from . import warnings from . import warnings
PathIsh = Union[str, Path] # avoid circular import from .common PathIsh = Union[str, Path] # avoid circular import from .common

View file

@ -3,8 +3,9 @@ from __future__ import annotations
import importlib import importlib
import re import re
import sys import sys
from collections.abc import Iterator
from contextlib import ExitStack, contextmanager from contextlib import ExitStack, contextmanager
from typing import Any, Callable, Dict, Iterator, Optional, Type, TypeVar from typing import Any, Callable, Dict, Optional, Type, TypeVar
Attrs = Dict[str, Any] Attrs = Dict[str, Any]

View file

@ -1,20 +1,18 @@
from __future__ import annotations
import os import os
from collections.abc import Iterable, Sequence
from glob import glob as do_glob from glob import glob as do_glob
from pathlib import Path from pathlib import Path
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Callable, Callable,
Generic, Generic,
Iterable,
List,
Sequence,
Tuple,
TypeVar, TypeVar,
Union, Union,
) )
from . import compat from . import compat, warnings
from . import warnings
# some helper functions # some helper functions
# TODO start deprecating this? soon we'd be able to use Path | str syntax which is shorter and more explicit # TODO start deprecating this? soon we'd be able to use Path | str syntax which is shorter and more explicit
@ -30,14 +28,14 @@ def get_files(
*, *,
sort: bool=True, sort: bool=True,
guess_compression: bool=True, guess_compression: bool=True,
) -> Tuple[Path, ...]: ) -> tuple[Path, ...]:
""" """
Helper function to avoid boilerplate. Helper function to avoid boilerplate.
Tuple as return type is a bit friendlier for hashing/caching, so hopefully makes sense Tuple as return type is a bit friendlier for hashing/caching, so hopefully makes sense
""" """
# TODO FIXME mm, some wrapper to assert iterator isn't empty? # TODO FIXME mm, some wrapper to assert iterator isn't empty?
sources: List[Path] sources: list[Path]
if isinstance(pp, Path): if isinstance(pp, Path):
sources = [pp] sources = [pp]
elif isinstance(pp, str): elif isinstance(pp, str):
@ -54,7 +52,7 @@ def get_files(
# TODO ugh. very flaky... -3 because [<this function>, get_files(), <actual caller>] # TODO ugh. very flaky... -3 because [<this function>, get_files(), <actual caller>]
return traceback.extract_stack()[-3].filename return traceback.extract_stack()[-3].filename
paths: List[Path] = [] paths: list[Path] = []
for src in sources: for src in sources:
if src.parts[0] == '~': if src.parts[0] == '~':
src = src.expanduser() src = src.expanduser()
@ -234,16 +232,14 @@ if not TYPE_CHECKING:
return types.asdict(*args, **kwargs) return types.asdict(*args, **kwargs)
# todo wrap these in deprecated decorator as well? # todo wrap these in deprecated decorator as well?
# TODO hmm how to deprecate these in runtime?
# tricky cause they are actually classes/types
from typing import Literal # noqa: F401
from .cachew import mcachew # noqa: F401 from .cachew import mcachew # noqa: F401
# this is kinda internal, should just use my.core.logging.setup_logger if necessary # this is kinda internal, should just use my.core.logging.setup_logger if necessary
from .logging import setup_logger from .logging import setup_logger
# TODO hmm how to deprecate these in runtime?
# tricky cause they are actually classes/types
from typing import Literal # noqa: F401
from .stats import Stats from .stats import Stats
from .types import ( from .types import (
Json, Json,

View file

@ -3,9 +3,10 @@ Bindings for the 'core' HPI configuration
''' '''
import re import re
from collections.abc import Sequence
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Optional, Sequence from typing import Optional
from . import PathIsh, warnings from . import PathIsh, warnings
@ -121,8 +122,8 @@ config = make_config(Config)
### tests start ### tests start
from collections.abc import Iterator
from contextlib import contextmanager as ctx from contextlib import contextmanager as ctx
from typing import Iterator
@ctx @ctx

View file

@ -9,8 +9,9 @@ import functools
import json import json
import sys import sys
from collections import defaultdict from collections import defaultdict
from collections.abc import Iterator, Mapping
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Iterator, List, Mapping, Set, TypeVar from typing import Any, Dict, List, Set, TypeVar
import click import click
from more_itertools import seekable from more_itertools import seekable

View file

@ -19,8 +19,9 @@ import ast
import logging import logging
import os import os
import re import re
from collections.abc import Iterable, Sequence
from pathlib import Path from pathlib import Path
from typing import Any, Iterable, List, NamedTuple, Optional, Sequence, cast from typing import Any, List, NamedTuple, Optional, cast
''' '''
None means that requirements weren't defined (different from empty requirements) None means that requirements weren't defined (different from empty requirements)

View file

@ -4,13 +4,12 @@ See https://beepb00p.xyz/mypy-error-handling.html#kiss for more detail
""" """
import traceback import traceback
from collections.abc import Iterable, Iterator
from datetime import datetime from datetime import datetime
from itertools import tee from itertools import tee
from typing import ( from typing import (
Any, Any,
Callable, Callable,
Iterable,
Iterator,
List, List,
Literal, Literal,
Optional, Optional,

View file

@ -6,8 +6,9 @@ Contains various backwards compatibility/deprecation helpers relevant to HPI its
import inspect import inspect
import os import os
import re import re
from collections.abc import Iterator, Sequence
from types import ModuleType from types import ModuleType
from typing import Iterator, List, Optional, Sequence, TypeVar from typing import List, Optional, TypeVar
from . import warnings from . import warnings

View file

@ -4,7 +4,8 @@ TODO doesn't really belong to 'core' morally, but can think of moving out later
from .internal import assert_subpackage; assert_subpackage(__name__) from .internal import assert_subpackage; assert_subpackage(__name__)
from typing import Any, Dict, Iterable, Optional from collections.abc import Iterable
from typing import Any, Dict, Optional
import click import click

View file

@ -122,8 +122,8 @@ def _wrap(j, parent=None) -> Tuple[Zoomable, List[Zoomable]]:
raise RuntimeError(f'Unexpected type: {type(j)} {j}') raise RuntimeError(f'Unexpected type: {type(j)} {j}')
from collections.abc import Iterator
from contextlib import contextmanager from contextlib import contextmanager
from typing import Iterator
class UnconsumedError(Exception): class UnconsumedError(Exception):

View file

@ -22,7 +22,8 @@ def parse_org_datetime(s: str) -> datetime:
# TODO I guess want to borrow inspiration from bs4? element type <-> tag; and similar logic for find_one, find_all # TODO I guess want to borrow inspiration from bs4? element type <-> tag; and similar logic for find_one, find_all
from typing import Callable, Iterable, TypeVar from collections.abc import Iterable
from typing import Callable, TypeVar
from orgparse import OrgNode from orgparse import OrgNode

View file

@ -7,6 +7,7 @@ from __future__ import annotations
# todo not sure if belongs to 'core'. It's certainly 'more' core than actual modules, but still not essential # todo not sure if belongs to 'core'. It's certainly 'more' core than actual modules, but still not essential
# NOTE: this file is meant to be importable without Pandas installed # NOTE: this file is meant to be importable without Pandas installed
import dataclasses import dataclasses
from collections.abc import Iterable, Iterator
from datetime import datetime, timezone from datetime import datetime, timezone
from pprint import pformat from pprint import pformat
from typing import ( from typing import (
@ -14,8 +15,6 @@ from typing import (
Any, Any,
Callable, Callable,
Dict, Dict,
Iterable,
Iterator,
Literal, Literal,
Type, Type,
TypeVar, TypeVar,

View file

@ -9,13 +9,12 @@ import dataclasses
import importlib import importlib
import inspect import inspect
import itertools import itertools
from collections.abc import Iterable, Iterator
from datetime import datetime from datetime import datetime
from typing import ( from typing import (
Any, Any,
Callable, Callable,
Dict, Dict,
Iterable,
Iterator,
List, List,
NamedTuple, NamedTuple,
Optional, Optional,

View file

@ -9,9 +9,10 @@ See the select_range function below
import re import re
import time import time
from collections.abc import Iterator
from datetime import date, datetime, timedelta from datetime import date, datetime, timedelta
from functools import lru_cache from functools import cache, lru_cache
from typing import Any, Callable, Iterator, NamedTuple, Optional, Type from typing import Any, Callable, NamedTuple, Optional, Type
import more_itertools import more_itertools
@ -98,7 +99,7 @@ def parse_datetime_float(date_str: str) -> float:
# probably DateLike input? but a user could specify an order_key # probably DateLike input? but a user could specify an order_key
# which is an epoch timestamp or a float value which they # which is an epoch timestamp or a float value which they
# expect to be converted to a datetime to compare # expect to be converted to a datetime to compare
@lru_cache(maxsize=None) @cache
def _datelike_to_float(dl: Any) -> float: def _datelike_to_float(dl: Any) -> float:
if isinstance(dl, datetime): if isinstance(dl, datetime):
return dl.timestamp() return dl.timestamp()

View file

@ -1,7 +1,7 @@
import datetime import datetime
from dataclasses import asdict, is_dataclass from dataclasses import asdict, is_dataclass
from decimal import Decimal from decimal import Decimal
from functools import lru_cache from functools import cache, lru_cache
from pathlib import Path from pathlib import Path
from typing import Any, Callable, NamedTuple, Optional from typing import Any, Callable, NamedTuple, Optional
@ -57,7 +57,7 @@ def _default_encode(obj: Any) -> Any:
# could possibly run multiple times/raise warning if you provide different 'default' # could possibly run multiple times/raise warning if you provide different 'default'
# functions or change the kwargs? The alternative is to maintain all of this at the module # functions or change the kwargs? The alternative is to maintain all of this at the module
# level, which is just as annoying # level, which is just as annoying
@lru_cache(maxsize=None) @cache
def _dumps_factory(**kwargs) -> Callable[[Any], str]: def _dumps_factory(**kwargs) -> Callable[[Any], str]:
use_default: DefaultEncoder = _default_encode use_default: DefaultEncoder = _default_encode
# if the user passed an additional 'default' parameter, # if the user passed an additional 'default' parameter,

View file

@ -4,8 +4,9 @@ and yielding nothing (or a default) when its not available
""" """
import warnings import warnings
from collections.abc import Iterable, Iterator
from functools import wraps from functools import wraps
from typing import Any, Callable, Iterable, Iterator, Optional, TypeVar from typing import Any, Callable, Optional, TypeVar
from .warnings import medium from .warnings import medium

View file

@ -3,10 +3,11 @@ from .internal import assert_subpackage; assert_subpackage(__name__)
import shutil import shutil
import sqlite3 import sqlite3
from collections.abc import Iterator
from contextlib import contextmanager from contextlib import contextmanager
from pathlib import Path from pathlib import Path
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
from typing import Any, Callable, Iterator, Literal, Optional, Tuple, Union, overload from typing import Any, Callable, Literal, Optional, Tuple, Union, overload
from .common import PathIsh from .common import PathIsh
from .compat import assert_never from .compat import assert_never

View file

@ -1,11 +1,13 @@
''' '''
Helpers for hpi doctor/stats functionality. Helpers for hpi doctor/stats functionality.
''' '''
from __future__ import annotations
import collections.abc import collections.abc
import importlib import importlib
import inspect import inspect
import typing import typing
from collections.abc import Iterable, Iterator, Sequence
from contextlib import contextmanager from contextlib import contextmanager
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
@ -13,20 +15,13 @@ from types import ModuleType
from typing import ( from typing import (
Any, Any,
Callable, Callable,
Dict,
Iterable,
Iterator,
List,
Optional,
Protocol, Protocol,
Sequence,
Union,
cast, cast,
) )
from .types import asdict from .types import asdict
Stats = Dict[str, Any] Stats = dict[str, Any]
class StatsFun(Protocol): class StatsFun(Protocol):
@ -55,10 +50,10 @@ def quick_stats():
def stat( def stat(
func: Union[Callable[[], Iterable[Any]], Iterable[Any]], func: Callable[[], Iterable[Any]] | Iterable[Any],
*, *,
quick: bool = False, quick: bool = False,
name: Optional[str] = None, name: str | None = None,
) -> Stats: ) -> Stats:
""" """
Extracts various statistics from a passed iterable/callable, e.g.: Extracts various statistics from a passed iterable/callable, e.g.:
@ -153,8 +148,8 @@ def test_stat() -> None:
# #
def get_stats(module_name: str, *, guess: bool = False) -> Optional[StatsFun]: def get_stats(module_name: str, *, guess: bool = False) -> StatsFun | None:
stats: Optional[StatsFun] = None stats: StatsFun | None = None
try: try:
module = importlib.import_module(module_name) module = importlib.import_module(module_name)
except Exception: except Exception:
@ -167,7 +162,7 @@ def get_stats(module_name: str, *, guess: bool = False) -> Optional[StatsFun]:
# TODO maybe could be enough to annotate OUTPUTS or something like that? # TODO maybe could be enough to annotate OUTPUTS or something like that?
# then stats could just use them as hints? # then stats could just use them as hints?
def guess_stats(module: ModuleType) -> Optional[StatsFun]: def guess_stats(module: ModuleType) -> StatsFun | None:
""" """
If the module doesn't have explicitly defined 'stat' function, If the module doesn't have explicitly defined 'stat' function,
this is used to try to guess what could be included in stats automatically this is used to try to guess what could be included in stats automatically
@ -206,7 +201,7 @@ def test_guess_stats() -> None:
} }
def _guess_data_providers(module: ModuleType) -> Dict[str, Callable]: def _guess_data_providers(module: ModuleType) -> dict[str, Callable]:
mfunctions = inspect.getmembers(module, inspect.isfunction) mfunctions = inspect.getmembers(module, inspect.isfunction)
return {k: v for k, v in mfunctions if is_data_provider(v)} return {k: v for k, v in mfunctions if is_data_provider(v)}
@ -263,7 +258,7 @@ def test_is_data_provider() -> None:
lam = lambda: [1, 2] lam = lambda: [1, 2]
assert not idp(lam) assert not idp(lam)
def has_extra_args(count) -> List[int]: def has_extra_args(count) -> list[int]:
return list(range(count)) return list(range(count))
assert not idp(has_extra_args) assert not idp(has_extra_args)
@ -340,10 +335,10 @@ def test_type_is_iterable() -> None:
assert not fun(None) assert not fun(None)
assert not fun(int) assert not fun(int)
assert not fun(Any) assert not fun(Any)
assert not fun(Dict[int, int]) assert not fun(dict[int, int])
assert fun(List[int]) assert fun(list[int])
assert fun(Sequence[Dict[str, str]]) assert fun(Sequence[dict[str, str]])
assert fun(Iterable[Any]) assert fun(Iterable[Any])
@ -434,7 +429,7 @@ def test_stat_iterable() -> None:
# experimental, not sure about it.. # experimental, not sure about it..
def _guess_datetime(x: Any) -> Optional[datetime]: def _guess_datetime(x: Any) -> datetime | None:
# todo hmm implement without exception.. # todo hmm implement without exception..
try: try:
d = asdict(x) d = asdict(x)

View file

@ -5,9 +5,10 @@ import sys
import tarfile import tarfile
import tempfile import tempfile
import zipfile import zipfile
from collections.abc import Generator, Sequence
from contextlib import contextmanager from contextlib import contextmanager
from pathlib import Path from pathlib import Path
from typing import Generator, List, Sequence, Tuple, Union from typing import List, Tuple, Union
from .logging import make_logger from .logging import make_logger

View file

@ -2,11 +2,11 @@
Helper 'module' for test_guess_stats Helper 'module' for test_guess_stats
""" """
from collections.abc import Iterable, Iterator, Sequence
from contextlib import contextmanager from contextlib import contextmanager
from dataclasses import dataclass from dataclasses import dataclass
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pathlib import Path from pathlib import Path
from typing import Iterable, Iterator, Sequence
@dataclass @dataclass

View file

@ -1,6 +1,8 @@
from __future__ import annotations
import os import os
from collections.abc import Iterator
from contextlib import contextmanager from contextlib import contextmanager
from typing import Iterator, Optional
import pytest import pytest
@ -15,7 +17,7 @@ skip_if_uses_optional_deps = pytest.mark.skipif(
# TODO maybe move to hpi core? # TODO maybe move to hpi core?
@contextmanager @contextmanager
def tmp_environ_set(key: str, value: Optional[str]) -> Iterator[None]: def tmp_environ_set(key: str, value: str | None) -> Iterator[None]:
prev_value = os.environ.get(key) prev_value = os.environ.get(key)
if value is None: if value is None:
os.environ.pop(key, None) os.environ.pop(key, None)

View file

@ -1,8 +1,9 @@
import json import json
import warnings import warnings
from collections.abc import Iterator
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Iterator, NamedTuple from typing import NamedTuple
from ..denylist import DenyList from ..denylist import DenyList

View file

@ -1,6 +1,6 @@
from .common import skip_if_uses_optional_deps as pytestmark from __future__ import annotations
from typing import List from .common import skip_if_uses_optional_deps as pytestmark
# TODO ugh, this is very messy.. need to sort out config overriding here # TODO ugh, this is very messy.. need to sort out config overriding here
@ -16,7 +16,7 @@ def test_cachew() -> None:
# TODO ugh. need doublewrap or something to avoid having to pass parens # TODO ugh. need doublewrap or something to avoid having to pass parens
@mcachew() @mcachew()
def cf() -> List[int]: def cf() -> list[int]:
nonlocal called nonlocal called
called += 1 called += 1
return [1, 2, 3] return [1, 2, 3]
@ -43,7 +43,7 @@ def test_cachew_dir_none() -> None:
called = 0 called = 0
@mcachew(cache_path=cache_dir() / 'ctest') @mcachew(cache_path=cache_dir() / 'ctest')
def cf() -> List[int]: def cf() -> list[int]:
nonlocal called nonlocal called
called += 1 called += 1
return [called, called, called] return [called, called, called]

View file

@ -2,8 +2,8 @@
Various tests that are checking behaviour of user config wrt to various things Various tests that are checking behaviour of user config wrt to various things
""" """
import sys
import os import os
import sys
from pathlib import Path from pathlib import Path
import pytest import pytest

View file

@ -1,5 +1,6 @@
from functools import lru_cache from collections.abc import Sequence
from typing import Dict, Sequence from functools import cache, lru_cache
from typing import Dict
import pytz import pytz
@ -43,7 +44,7 @@ def _abbr_to_timezone_map() -> Dict[str, pytz.BaseTzInfo]:
return res return res
@lru_cache(maxsize=None) @cache
def abbr_to_timezone(abbr: str) -> pytz.BaseTzInfo: def abbr_to_timezone(abbr: str) -> pytz.BaseTzInfo:
return _abbr_to_timezone_map()[abbr] return _abbr_to_timezone_map()[abbr]

View file

@ -1,10 +1,11 @@
import os import os
import pkgutil import pkgutil
import sys import sys
from collections.abc import Iterable
from itertools import chain from itertools import chain
from pathlib import Path from pathlib import Path
from types import ModuleType from types import ModuleType
from typing import Iterable, List, Optional from typing import List, Optional
from .discovery_pure import HPIModule, _is_not_module_src, has_stats, ignored from .discovery_pure import HPIModule, _is_not_module_src, has_stats, ignored

View file

@ -1,6 +1,8 @@
from __future__ import annotations
import sys import sys
from concurrent.futures import Executor, Future from concurrent.futures import Executor, Future
from typing import Any, Callable, Optional, TypeVar from typing import Any, Callable, TypeVar
from ..compat import ParamSpec from ..compat import ParamSpec
@ -15,7 +17,7 @@ class DummyExecutor(Executor):
but also want to provide an option to run the code serially (e.g. for debugging) but also want to provide an option to run the code serially (e.g. for debugging)
""" """
def __init__(self, max_workers: Optional[int] = 1) -> None: def __init__(self, max_workers: int | None = 1) -> None:
self._shutdown = False self._shutdown = False
self._max_workers = max_workers self._max_workers = max_workers

View file

@ -1,3 +1,5 @@
from __future__ import annotations
import importlib import importlib
import importlib.util import importlib.util
import sys import sys
@ -5,23 +7,22 @@ from pathlib import Path
from types import ModuleType from types import ModuleType
from typing import Optional from typing import Optional
from ..common import PathIsh
# TODO only used in tests? not sure if useful at all. # TODO only used in tests? not sure if useful at all.
def import_file(p: PathIsh, name: Optional[str] = None) -> ModuleType: def import_file(p: Path | str, name: str | None = None) -> ModuleType:
p = Path(p) p = Path(p)
if name is None: if name is None:
name = p.stem name = p.stem
spec = importlib.util.spec_from_file_location(name, p) spec = importlib.util.spec_from_file_location(name, p)
assert spec is not None, f"Fatal error; Could not create module spec from {name} {p}" assert spec is not None, f"Fatal error; Could not create module spec from {name} {p}"
foo = importlib.util.module_from_spec(spec) foo = importlib.util.module_from_spec(spec)
loader = spec.loader; assert loader is not None loader = spec.loader
assert loader is not None
loader.exec_module(foo) loader.exec_module(foo)
return foo return foo
def import_from(path: PathIsh, name: str) -> ModuleType: def import_from(path: Path | str, name: str) -> ModuleType:
path = str(path) path = str(path)
sys.path.append(path) sys.path.append(path)
try: try:
@ -30,7 +31,7 @@ def import_from(path: PathIsh, name: str) -> ModuleType:
sys.path.remove(path) sys.path.remove(path)
def import_dir(path: PathIsh, extra: str = '') -> ModuleType: def import_dir(path: Path | str, extra: str = '') -> ModuleType:
p = Path(path) p = Path(path)
if p.parts[0] == '~': if p.parts[0] == '~':
p = p.expanduser() # TODO eh. not sure about this.. p = p.expanduser() # TODO eh. not sure about this..

View file

@ -4,17 +4,13 @@ Various helpers/transforms of iterators
Ideally this should be as small as possible and we should rely on stdlib itertools or more_itertools Ideally this should be as small as possible and we should rely on stdlib itertools or more_itertools
""" """
from __future__ import annotations
import warnings import warnings
from collections.abc import Hashable from collections.abc import Hashable, Iterable, Iterator, Sized
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Callable, Callable,
Dict,
Iterable,
Iterator,
List,
Optional,
Sized,
TypeVar, TypeVar,
Union, Union,
cast, cast,
@ -23,9 +19,8 @@ from typing import (
import more_itertools import more_itertools
from decorator import decorator from decorator import decorator
from ..compat import ParamSpec
from .. import warnings as core_warnings from .. import warnings as core_warnings
from ..compat import ParamSpec
T = TypeVar('T') T = TypeVar('T')
K = TypeVar('K') K = TypeVar('K')
@ -39,7 +34,7 @@ def _identity(v: T) -> V: # type: ignore[type-var]
# ugh. nothing in more_itertools? # ugh. nothing in more_itertools?
# perhaps duplicates_everseen? but it doesn't yield non-unique elements? # perhaps duplicates_everseen? but it doesn't yield non-unique elements?
def ensure_unique(it: Iterable[T], *, key: Callable[[T], K]) -> Iterable[T]: def ensure_unique(it: Iterable[T], *, key: Callable[[T], K]) -> Iterable[T]:
key2item: Dict[K, T] = {} key2item: dict[K, T] = {}
for i in it: for i in it:
k = key(i) k = key(i)
pi = key2item.get(k, None) pi = key2item.get(k, None)
@ -72,10 +67,10 @@ def make_dict(
key: Callable[[T], K], key: Callable[[T], K],
# TODO make value optional instead? but then will need a typing override for it? # TODO make value optional instead? but then will need a typing override for it?
value: Callable[[T], V] = _identity, value: Callable[[T], V] = _identity,
) -> Dict[K, V]: ) -> dict[K, V]:
with_keys = ((key(i), i) for i in it) with_keys = ((key(i), i) for i in it)
uniques = ensure_unique(with_keys, key=lambda p: p[0]) uniques = ensure_unique(with_keys, key=lambda p: p[0])
res: Dict[K, V] = {} res: dict[K, V] = {}
for k, i in uniques: for k, i in uniques:
res[k] = i if value is None else value(i) res[k] = i if value is None else value(i)
return res return res
@ -93,8 +88,8 @@ def test_make_dict() -> None:
d = make_dict(it, key=lambda i: i % 2, value=lambda i: i) d = make_dict(it, key=lambda i: i % 2, value=lambda i: i)
# check type inference # check type inference
d2: Dict[str, int] = make_dict(it, key=lambda i: str(i)) d2: dict[str, int] = make_dict(it, key=lambda i: str(i))
d3: Dict[str, bool] = make_dict(it, key=lambda i: str(i), value=lambda i: i % 2 == 0) d3: dict[str, bool] = make_dict(it, key=lambda i: str(i), value=lambda i: i % 2 == 0)
LFP = ParamSpec('LFP') LFP = ParamSpec('LFP')
@ -102,7 +97,7 @@ LV = TypeVar('LV')
@decorator @decorator
def _listify(func: Callable[LFP, Iterable[LV]], *args: LFP.args, **kwargs: LFP.kwargs) -> List[LV]: def _listify(func: Callable[LFP, Iterable[LV]], *args: LFP.args, **kwargs: LFP.kwargs) -> list[LV]:
""" """
Wraps a function's return value in wrapper (e.g. list) Wraps a function's return value in wrapper (e.g. list)
Useful when an algorithm can be expressed more cleanly as a generator Useful when an algorithm can be expressed more cleanly as a generator
@ -115,7 +110,7 @@ def _listify(func: Callable[LFP, Iterable[LV]], *args: LFP.args, **kwargs: LFP.k
# so seems easiest to just use specialize instantiations of decorator instead # so seems easiest to just use specialize instantiations of decorator instead
if TYPE_CHECKING: if TYPE_CHECKING:
def listify(func: Callable[LFP, Iterable[LV]]) -> Callable[LFP, List[LV]]: ... # noqa: ARG001 def listify(func: Callable[LFP, Iterable[LV]]) -> Callable[LFP, list[LV]]: ... # noqa: ARG001
else: else:
listify = _listify listify = _listify
@ -130,7 +125,7 @@ def test_listify() -> None:
yield 2 yield 2
res = it() res = it()
assert_type(res, List[int]) assert_type(res, list[int])
assert res == [1, 2] assert res == [1, 2]
@ -201,24 +196,24 @@ def test_warn_if_empty_list() -> None:
ll = [1, 2, 3] ll = [1, 2, 3]
@warn_if_empty @warn_if_empty
def nonempty() -> List[int]: def nonempty() -> list[int]:
return ll return ll
with warnings.catch_warnings(record=True) as w: with warnings.catch_warnings(record=True) as w:
res1 = nonempty() res1 = nonempty()
assert len(w) == 0 assert len(w) == 0
assert_type(res1, List[int]) assert_type(res1, list[int])
assert isinstance(res1, list) assert isinstance(res1, list)
assert res1 is ll # object should be unchanged! assert res1 is ll # object should be unchanged!
@warn_if_empty @warn_if_empty
def empty() -> List[str]: def empty() -> list[str]:
return [] return []
with warnings.catch_warnings(record=True) as w: with warnings.catch_warnings(record=True) as w:
res2 = empty() res2 = empty()
assert len(w) == 1 assert len(w) == 1
assert_type(res2, List[str]) assert_type(res2, list[str])
assert isinstance(res2, list) assert isinstance(res2, list)
assert res2 == [] assert res2 == []
@ -242,7 +237,7 @@ def check_if_hashable(iterable: Iterable[_HT]) -> Iterable[_HT]:
""" """
NOTE: Despite Hashable bound, typing annotation doesn't guarantee runtime safety NOTE: Despite Hashable bound, typing annotation doesn't guarantee runtime safety
Consider hashable type X, and Y that inherits from X, but not hashable Consider hashable type X, and Y that inherits from X, but not hashable
Then l: List[X] = [Y(...)] is a valid expression, and type checks against Hashable, Then l: list[X] = [Y(...)] is a valid expression, and type checks against Hashable,
but isn't runtime hashable but isn't runtime hashable
""" """
# Sadly this doesn't work 100% correctly with dataclasses atm... # Sadly this doesn't work 100% correctly with dataclasses atm...
@ -268,28 +263,27 @@ def check_if_hashable(iterable: Iterable[_HT]) -> Iterable[_HT]:
# TODO different policies -- error/warn/ignore? # TODO different policies -- error/warn/ignore?
def test_check_if_hashable() -> None: def test_check_if_hashable() -> None:
from dataclasses import dataclass from dataclasses import dataclass
from typing import Set, Tuple
import pytest import pytest
from ..compat import assert_type from ..compat import assert_type
x1: List[int] = [1, 2] x1: list[int] = [1, 2]
r1 = check_if_hashable(x1) r1 = check_if_hashable(x1)
assert_type(r1, Iterable[int]) assert_type(r1, Iterable[int])
assert r1 is x1 assert r1 is x1
x2: Iterator[Union[int, str]] = iter((123, 'aba')) x2: Iterator[int | str] = iter((123, 'aba'))
r2 = check_if_hashable(x2) r2 = check_if_hashable(x2)
assert_type(r2, Iterable[Union[int, str]]) assert_type(r2, Iterable[Union[int, str]])
assert list(r2) == [123, 'aba'] assert list(r2) == [123, 'aba']
x3: Tuple[object, ...] = (789, 'aba') x3: tuple[object, ...] = (789, 'aba')
r3 = check_if_hashable(x3) r3 = check_if_hashable(x3)
assert_type(r3, Iterable[object]) assert_type(r3, Iterable[object])
assert r3 is x3 # object should be unchanged assert r3 is x3 # object should be unchanged
x4: List[Set[int]] = [{1, 2, 3}, {4, 5, 6}] x4: list[set[int]] = [{1, 2, 3}, {4, 5, 6}]
with pytest.raises(Exception): with pytest.raises(Exception):
# should be rejected by mypy sice set isn't Hashable, but also throw at runtime # should be rejected by mypy sice set isn't Hashable, but also throw at runtime
r4 = check_if_hashable(x4) # type: ignore[type-var] r4 = check_if_hashable(x4) # type: ignore[type-var]
@ -307,7 +301,7 @@ def test_check_if_hashable() -> None:
class X: class X:
a: int a: int
x6: List[X] = [X(a=123)] x6: list[X] = [X(a=123)]
r6 = check_if_hashable(x6) r6 = check_if_hashable(x6)
assert x6 is r6 assert x6 is r6
@ -316,7 +310,7 @@ def test_check_if_hashable() -> None:
class Y(X): class Y(X):
b: str b: str
x7: List[Y] = [Y(a=123, b='aba')] x7: list[Y] = [Y(a=123, b='aba')]
with pytest.raises(Exception): with pytest.raises(Exception):
# ideally that would also be rejected by mypy, but currently there is a bug # ideally that would also be rejected by mypy, but currently there is a bug
# which treats all dataclasses as hashable: https://github.com/python/mypy/issues/11463 # which treats all dataclasses as hashable: https://github.com/python/mypy/issues/11463
@ -331,11 +325,8 @@ _UEU = TypeVar('_UEU')
# instead of just iterator # instead of just iterator
# TODO maybe deprecated Callable support? not sure # TODO maybe deprecated Callable support? not sure
def unique_everseen( def unique_everseen(
fun: Union[ fun: Callable[[], Iterable[_UET]] | Iterable[_UET],
Callable[[], Iterable[_UET]], key: Callable[[_UET], _UEU] | None = None,
Iterable[_UET]
],
key: Optional[Callable[[_UET], _UEU]] = None,
) -> Iterator[_UET]: ) -> Iterator[_UET]:
import os import os

View file

@ -4,6 +4,7 @@ TODO ideally would be great to replace with some existing solution, or find a be
since who looks at the terminal output? since who looks at the terminal output?
E.g. would be nice to propagate the warnings in the UI (it's even a subclass of Exception!) E.g. would be nice to propagate the warnings in the UI (it's even a subclass of Exception!)
''' '''
from __future__ import annotations
import sys import sys
import warnings import warnings