From 7bfce72b7c72f470e0dcb2847c0abaf437a90a0a Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 16 Aug 2024 13:25:19 +0300 Subject: [PATCH] core: cleanup/sort imports according to `ruff check --select I` --- my/core/__init__.py | 20 +++++++++---------- my/core/__main__.py | 21 ++++++++++---------- my/core/_cpu_pool.py | 5 ++--- my/core/_deprecated/kompress.py | 15 +++++++------- my/core/cachew.py | 17 +++++++++++++--- my/core/cfg.py | 15 +++++++------- my/core/common.py | 17 ++++++++-------- my/core/compat.py | 7 +++---- my/core/core_config.py | 13 +++++++----- my/core/denylist.py | 12 +++++------ my/core/discovery_pure.py | 8 ++++---- my/core/error.py | 35 +++++++++++++++++++++++++-------- my/core/experimental.py | 2 +- my/core/freezer.py | 4 +++- my/core/hpi_compat.py | 2 +- my/core/influxdb.py | 12 +++++------ my/core/init.py | 2 +- my/core/konsume.py | 5 +++++ my/core/logging.py | 8 +++++--- my/core/orgmode.py | 8 +++++++- my/core/pandas.py | 7 +++---- my/core/preinit.py | 4 +++- my/core/query.py | 20 +++++++++++++++---- my/core/query_range.py | 14 ++++++------- my/core/serialize.py | 9 +++++---- my/core/source.py | 4 ++-- my/core/sqlite.py | 8 +++----- my/core/stats.py | 12 +++++------ my/core/structure.py | 6 ++---- my/core/tests/auto_stats.py | 2 +- my/core/tests/common.py | 3 +-- my/core/tests/denylist.py | 6 +++--- my/core/tests/sqlite.py | 4 ++-- my/core/tests/structure.py | 3 +-- my/core/tests/test_cachew.py | 3 +-- my/core/tests/test_cli.py | 2 +- my/core/tests/test_get_files.py | 8 ++++---- my/core/time.py | 2 +- my/core/types.py | 4 ++-- my/core/util.py | 15 +++++++++----- my/core/utils/concurrent.py | 6 +++--- my/core/utils/imports.py | 4 ++-- my/core/utils/itertools.py | 15 ++++++++------ my/core/warnings.py | 12 ++++++++--- my/location/common.py | 4 ++-- 45 files changed, 235 insertions(+), 170 deletions(-) diff --git a/my/core/__init__.py b/my/core/__init__.py index 247aab0..19be7fe 100644 --- a/my/core/__init__.py +++ b/my/core/__init__.py @@ -1,23 +1,21 @@ # this file only keeps the most common & critical types/utility functions from typing import TYPE_CHECKING -from .common import get_files, PathIsh, Paths -from .stats import stat, Stats +from .cfg import make_config +from .common import PathIsh, Paths, get_files +from .compat import assert_never +from .error import Res, unwrap +from .logging import ( + make_logger, +) +from .stats import Stats, stat from .types import ( Json, datetime_aware, datetime_naive, ) -from .compat import assert_never -from .utils.itertools import warn_if_empty - -from .cfg import make_config -from .error import Res, unwrap -from .logging import ( - make_logger, -) from .util import __NOT_HPI_MODULE__ - +from .utils.itertools import warn_if_empty LazyLogger = make_logger # TODO deprecate this in favor of make_logger diff --git a/my/core/__main__.py b/my/core/__main__.py index e0d5c12..276de26 100644 --- a/my/core/__main__.py +++ b/my/core/__main__.py @@ -1,17 +1,17 @@ -from contextlib import ExitStack import functools import importlib import inspect -from itertools import chain import os import shlex import shutil import sys import tempfile import traceback -from typing import Optional, Sequence, Iterable, List, Type, Any, Callable +from contextlib import ExitStack +from itertools import chain from pathlib import Path -from subprocess import check_call, run, PIPE, CompletedProcess, Popen +from subprocess import PIPE, CompletedProcess, Popen, check_call, run +from typing import Any, Callable, Iterable, List, Optional, Sequence, Type import click @@ -221,6 +221,8 @@ See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-module from .util import HPIModule, modules + + def _modules(*, all: bool=False) -> Iterable[HPIModule]: skipped = [] for m in modules(): @@ -243,9 +245,9 @@ def modules_check(*, verbose: bool, list_all: bool, quick: bool, for_modules: Li import contextlib - from .util import HPIModule - from .stats import get_stats, quick_stats from .error import warn_my_config_import_error + from .stats import get_stats, quick_stats + from .util import HPIModule mods: Iterable[HPIModule] if len(for_modules) == 0: @@ -437,7 +439,7 @@ def _ui_getchar_pick(choices: Sequence[str], prompt: str = 'Select from: ') -> i def _locate_functions_or_prompt(qualified_names: List[str], prompt: bool = True) -> Iterable[Callable[..., Any]]: - from .query import locate_qualified_function, QueryException + from .query import QueryException, locate_qualified_function from .stats import is_data_provider # if not connected to a terminal, can't prompt @@ -511,8 +513,7 @@ def query_hpi_functions( raise_exceptions: bool, drop_exceptions: bool, ) -> None: - from .query_range import select_range, RangeTuple - import my.core.error as err + from .query_range import RangeTuple, select_range # chain list of functions from user, in the order they wrote them on the CLI input_src = chain(*(f() for f in _locate_functions_or_prompt(qualified_names))) @@ -825,7 +826,7 @@ def query_cmd( hpi query --order-type datetime --after '2016-01-01' --before '2019-01-01' my.reddit.all.comments ''' - from datetime import datetime, date + from datetime import date, datetime chosen_order_type: Optional[Type] if order_type == "datetime": diff --git a/my/core/_cpu_pool.py b/my/core/_cpu_pool.py index 5ac66de..2369075 100644 --- a/my/core/_cpu_pool.py +++ b/my/core/_cpu_pool.py @@ -10,10 +10,9 @@ how many cores we want to dedicate to the DAL. Enabled by the env variable, specifying how many cores to dedicate e.g. "HPI_CPU_POOL=4 hpi query ..." """ -from concurrent.futures import ProcessPoolExecutor import os -from typing import cast, Optional - +from concurrent.futures import ProcessPoolExecutor +from typing import Optional, cast _NOT_SET = cast(ProcessPoolExecutor, object()) _INSTANCE: Optional[ProcessPoolExecutor] = _NOT_SET diff --git a/my/core/_deprecated/kompress.py b/my/core/_deprecated/kompress.py index 25b8a20..7eb9b37 100644 --- a/my/core/_deprecated/kompress.py +++ b/my/core/_deprecated/kompress.py @@ -4,13 +4,13 @@ Various helpers for compression # fmt: off from __future__ import annotations -from datetime import datetime -from functools import total_ordering import io import pathlib -from pathlib import Path import sys -from typing import Union, IO, Sequence, Any, Iterator +from datetime import datetime +from functools import total_ordering +from pathlib import Path +from typing import IO, Any, Iterator, Sequence, Union PathIsh = Union[Path, str] @@ -31,7 +31,7 @@ def is_compressed(p: Path) -> bool: def _zstd_open(path: Path, *args, **kwargs) -> IO: - import zstandard as zstd # type: ignore + import zstandard as zstd # type: ignore fh = path.open('rb') dctx = zstd.ZstdDecompressor() reader = dctx.stream_reader(fh) @@ -85,7 +85,7 @@ def kopen(path: PathIsh, *args, mode: str='rt', **kwargs) -> IO: # todo 'expected "BinaryIO"'?? return io.TextIOWrapper(ifile, encoding=encoding) elif name.endswith(Ext.lz4): - import lz4.frame # type: ignore + import lz4.frame # type: ignore return lz4.frame.open(str(pp), mode, *args, **kwargs) elif name.endswith(Ext.zstd) or name.endswith(Ext.zst): kwargs['mode'] = mode @@ -101,8 +101,8 @@ def kopen(path: PathIsh, *args, mode: str='rt', **kwargs) -> IO: return pp.open(mode, *args, **kwargs) -import typing import os +import typing if typing.TYPE_CHECKING: # otherwise mypy can't figure out that BasePath is a type alias.. @@ -147,6 +147,7 @@ def kexists(path: PathIsh, subpath: str) -> bool: import zipfile + if sys.version_info[:2] >= (3, 8): # meh... zipfile.Path is not available on 3.7 zipfile_Path = zipfile.Path diff --git a/my/core/cachew.py b/my/core/cachew.py index bcd838d..e0e7adf 100644 --- a/my/core/cachew.py +++ b/my/core/cachew.py @@ -1,11 +1,22 @@ from .internal import assert_subpackage; assert_subpackage(__name__) -from contextlib import contextmanager import logging -from pathlib import Path import sys -from typing import Optional, Iterator, cast, TYPE_CHECKING, TypeVar, Callable, overload, Union, Any, Type import warnings +from contextlib import contextmanager +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Iterator, + Optional, + Type, + TypeVar, + Union, + cast, + overload, +) import appdirs # type: ignore[import-untyped] diff --git a/my/core/cfg.py b/my/core/cfg.py index 0b59537..a71a7e3 100644 --- a/my/core/cfg.py +++ b/my/core/cfg.py @@ -1,6 +1,10 @@ from __future__ import annotations -from typing import TypeVar, Type, Callable, Dict, Any +import importlib +import re +import sys +from contextlib import ExitStack, contextmanager +from typing import Any, Callable, Dict, Iterator, Optional, Type, TypeVar Attrs = Dict[str, Any] @@ -27,8 +31,8 @@ def make_config(cls: Type[C], migration: Callable[[Attrs], Attrs]=lambda x: x) - F = TypeVar('F') -from contextlib import contextmanager -from typing import Iterator + + @contextmanager def _override_config(config: F) -> Iterator[F]: ''' @@ -46,9 +50,6 @@ def _override_config(config: F) -> Iterator[F]: delattr(config, k) -import importlib -import sys -from typing import Optional ModuleRegex = str @contextmanager def _reload_modules(modules: ModuleRegex) -> Iterator[None]: @@ -79,8 +80,6 @@ def _reload_modules(modules: ModuleRegex) -> Iterator[None]: sys.modules.pop(m, None) -from contextlib import ExitStack -import re @contextmanager def tmp_config(*, modules: Optional[ModuleRegex]=None, config=None): if modules is None: diff --git a/my/core/common.py b/my/core/common.py index f20e1d4..9a58caf 100644 --- a/my/core/common.py +++ b/my/core/common.py @@ -1,20 +1,20 @@ +import os +import warnings from glob import glob as do_glob from pathlib import Path -import os from typing import ( + TYPE_CHECKING, Callable, Iterable, List, Sequence, - TYPE_CHECKING, Tuple, TypeVar, Union, ) -import warnings -from . import warnings as core_warnings from . import compat +from . import warnings as core_warnings # some helper functions # TODO start deprecating this? soon we'd be able to use Path | str syntax which is shorter and more explicit @@ -92,7 +92,7 @@ def get_files( traceback.print_stack() if guess_compression: - from .kompress import CPath, is_compressed, ZipPath + from .kompress import CPath, ZipPath, is_compressed # NOTE: wrap is just for backwards compat with vendorized kompress # with kompress library, only is_compressed check and Cpath should be enough @@ -109,7 +109,7 @@ def get_files( return tuple(paths) -from typing import TypeVar, Callable, Generic +from typing import Callable, Generic, TypeVar _R = TypeVar('_R') @@ -133,6 +133,8 @@ class classproperty(Generic[_R]): import re + + # https://stackoverflow.com/a/295466/706389 def get_valid_filename(s: str) -> str: s = str(s).strip().replace(' ', '_') @@ -142,7 +144,6 @@ def get_valid_filename(s: str) -> str: # TODO deprecate and suggest to use one from my.core directly? not sure from .utils.itertools import unique_everseen - ### legacy imports, keeping them here for backwards compatibility ## hiding behind TYPE_CHECKING so it works in runtime ## in principle, warnings.deprecated decorator should cooperate with mypy, but doesn't look like it works atm? @@ -225,8 +226,8 @@ if not TYPE_CHECKING: from .stats import Stats from .types import ( Json, - datetime_naive, datetime_aware, + datetime_naive, ) tzdatetime = datetime_aware diff --git a/my/core/compat.py b/my/core/compat.py index 7bbe509..4372a01 100644 --- a/my/core/compat.py +++ b/my/core/compat.py @@ -6,7 +6,6 @@ If something is relevant to HPI itself, please put it in .hpi_compat instead import sys from typing import TYPE_CHECKING - if sys.version_info[:2] >= (3, 13): from warnings import deprecated else: @@ -48,7 +47,7 @@ else: # bisect_left doesn't have a 'key' parameter (which we use) # till python3.10 if sys.version_info[:2] <= (3, 9): - from typing import List, TypeVar, Any, Optional, Callable + from typing import Any, Callable, List, Optional, TypeVar X = TypeVar('X') @@ -131,6 +130,6 @@ else: if sys.version_info[:2] >= (3, 11): - from typing import assert_never, assert_type, Never + from typing import Never, assert_never, assert_type else: - from typing_extensions import assert_never, assert_type, Never + from typing_extensions import Never, assert_never, assert_type diff --git a/my/core/core_config.py b/my/core/core_config.py index 889dbf9..9036971 100644 --- a/my/core/core_config.py +++ b/my/core/core_config.py @@ -2,18 +2,18 @@ Bindings for the 'core' HPI configuration ''' +import re from dataclasses import dataclass from pathlib import Path -import re -from typing import Sequence, Optional +from typing import Optional, Sequence -from . import warnings, PathIsh +from . import PathIsh, warnings try: from my.config import core as user_config # type: ignore[attr-defined] except Exception as e: try: - from my.config import common as user_config # type: ignore[attr-defined] + from my.config import common as user_config # type: ignore[attr-defined] warnings.high("'common' config section is deprecated. Please rename it to 'core'.") except Exception as e2: # make it defensive, because it's pretty commonly used and would be annoying if it breaks hpi doctor etc. @@ -116,12 +116,15 @@ class Config(user_config): from .cfg import make_config + config = make_config(Config) ### tests start -from typing import Iterator from contextlib import contextmanager as ctx +from typing import Iterator + + @ctx def _reset_config() -> Iterator[Config]: # todo maybe have this decorator for the whole of my.config? diff --git a/my/core/denylist.py b/my/core/denylist.py index 8c18e06..7ca0ddf 100644 --- a/my/core/denylist.py +++ b/my/core/denylist.py @@ -5,19 +5,19 @@ A helper module for defining denylists for sources programmatically For docs, see doc/DENYLIST.md """ -import sys -import json import functools +import json +import sys from collections import defaultdict -from typing import TypeVar, Set, Any, Mapping, Iterator, Dict, List from pathlib import Path +from typing import Any, Dict, Iterator, List, Mapping, Set, TypeVar import click from more_itertools import seekable -from my.core.serialize import dumps -from my.core.common import PathIsh -from my.core.warnings import medium +from my.core.common import PathIsh +from my.core.serialize import dumps +from my.core.warnings import medium T = TypeVar("T") diff --git a/my/core/discovery_pure.py b/my/core/discovery_pure.py index 85b75ab..63d9922 100644 --- a/my/core/discovery_pure.py +++ b/my/core/discovery_pure.py @@ -16,11 +16,11 @@ NOT_HPI_MODULE_VAR = '__NOT_HPI_MODULE__' ### import ast -import os -from typing import Optional, Sequence, List, NamedTuple, Iterable, cast, Any -from pathlib import Path -import re import logging +import os +import re +from pathlib import Path +from typing import Any, Iterable, List, NamedTuple, Optional, Sequence, cast ''' None means that requirements weren't defined (different from empty requirements) diff --git a/my/core/error.py b/my/core/error.py index 2432a5d..c4dff07 100644 --- a/my/core/error.py +++ b/my/core/error.py @@ -3,14 +3,28 @@ Various error handling helpers See https://beepb00p.xyz/mypy-error-handling.html#kiss for more detail """ +import traceback +from datetime import datetime from itertools import tee -from typing import Union, TypeVar, Iterable, List, Tuple, Type, Optional, Callable, Any, cast, Iterator, Literal +from typing import ( + Any, + Callable, + Iterable, + Iterator, + List, + Literal, + Optional, + Tuple, + Type, + TypeVar, + Union, + cast, +) from .types import Json - T = TypeVar('T') -E = TypeVar('E', bound=Exception) # TODO make covariant? +E = TypeVar('E', bound=Exception) # TODO make covariant? ResT = Union[T, E] @@ -18,6 +32,7 @@ Res = ResT[T, Exception] ErrorPolicy = Literal["yield", "raise", "drop"] + def notnone(x: Optional[T]) -> T: assert x is not None return x @@ -29,6 +44,7 @@ def unwrap(res: Res[T]) -> T: else: return res + def drop_exceptions(itr: Iterator[Res[T]]) -> Iterator[T]: """Return non-errors from the iterable""" for o in itr: @@ -146,23 +162,23 @@ def test_sort_res_by() -> None: # helpers to associate timestamps with the errors (so something meaningful could be displayed on the plots, for example) # todo document it under 'patterns' somewhere... - # todo proper typevar? -from datetime import datetime def set_error_datetime(e: Exception, dt: Optional[datetime]) -> None: if dt is None: return e.args = e.args + (dt,) # todo not sure if should return new exception? + def attach_dt(e: Exception, *, dt: Optional[datetime]) -> Exception: set_error_datetime(e, dt) return e + # todo it might be problematic because might mess with timezones (when it's converted to string, it's converted to a shift) def extract_error_datetime(e: Exception) -> Optional[datetime]: import re - from datetime import datetime + for x in reversed(e.args): if isinstance(x, datetime): return x @@ -177,7 +193,6 @@ def extract_error_datetime(e: Exception) -> Optional[datetime]: return None -import traceback def error_to_json(e: Exception) -> Json: estr = ''.join(traceback.format_exception(Exception, e, e.__traceback__)) return {'error': estr} @@ -185,6 +200,7 @@ def error_to_json(e: Exception) -> Json: MODULE_SETUP_URL = 'https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#private-configuration-myconfig' + def warn_my_config_import_error(err: Union[ImportError, AttributeError], help_url: Optional[str] = None) -> bool: """ If the user tried to import something from my.config but it failed, @@ -193,7 +209,9 @@ def warn_my_config_import_error(err: Union[ImportError, AttributeError], help_ur Returns True if it matched a possible config error """ import re + import click + if help_url is None: help_url = MODULE_SETUP_URL if type(err) is ImportError: @@ -226,7 +244,8 @@ See {help_url} or check the corresponding module.py file for an example\ def test_datetime_errors() -> None: - import pytz + import pytz # noqa: I001 + dt_notz = datetime.now() dt_tz = datetime.now(tz=pytz.timezone('Europe/Amsterdam')) for dt in [dt_tz, dt_notz]: diff --git a/my/core/experimental.py b/my/core/experimental.py index c10ba71..1a78272 100644 --- a/my/core/experimental.py +++ b/my/core/experimental.py @@ -1,6 +1,6 @@ import sys -from typing import Any, Dict, Optional import types +from typing import Any, Dict, Optional # The idea behind this one is to support accessing "overlaid/shadowed" modules from namespace packages diff --git a/my/core/freezer.py b/my/core/freezer.py index 09ba032..e46525b 100644 --- a/my/core/freezer.py +++ b/my/core/freezer.py @@ -2,7 +2,7 @@ from .internal import assert_subpackage; assert_subpackage(__name__) import dataclasses as dcl import inspect -from typing import TypeVar, Type, Any +from typing import Any, Type, TypeVar D = TypeVar('D') @@ -22,6 +22,8 @@ def _freeze_dataclass(Orig: Type[D]): # todo need some decorator thingie? from typing import Generic + + class Freezer(Generic[D]): ''' Some magic which converts dataclass properties into fields. diff --git a/my/core/hpi_compat.py b/my/core/hpi_compat.py index 3c567d9..bad0b17 100644 --- a/my/core/hpi_compat.py +++ b/my/core/hpi_compat.py @@ -2,8 +2,8 @@ Contains various backwards compatibility/deprecation helpers relevant to HPI itself. (as opposed to .compat module which implements compatibility between python versions) """ -import os import inspect +import os import re from types import ModuleType from typing import Iterator, List, Optional, TypeVar diff --git a/my/core/influxdb.py b/my/core/influxdb.py index c4b6409..c39f6af 100644 --- a/my/core/influxdb.py +++ b/my/core/influxdb.py @@ -4,11 +4,12 @@ TODO doesn't really belong to 'core' morally, but can think of moving out later from .internal import assert_subpackage; assert_subpackage(__name__) -from typing import Iterable, Any, Optional, Dict +from typing import Any, Dict, Iterable, Optional + +import click from .logging import make_logger -from .types import asdict, Json - +from .types import Json, asdict logger = make_logger(__name__) @@ -28,7 +29,7 @@ def fill(it: Iterable[Any], *, measurement: str, reset: bool=RESET_DEFAULT, dt_c db = config.db - from influxdb import InfluxDBClient # type: ignore + from influxdb import InfluxDBClient # type: ignore client = InfluxDBClient() # todo maybe create if not exists? # client.create_database(db) @@ -106,6 +107,7 @@ def magic_fill(it, *, name: Optional[str]=None, reset: bool=RESET_DEFAULT) -> No it = it() from itertools import tee + from more_itertools import first, one it, x = tee(it) f = first(x, default=None) @@ -125,8 +127,6 @@ def magic_fill(it, *, name: Optional[str]=None, reset: bool=RESET_DEFAULT) -> No fill(it, measurement=name, reset=reset, dt_col=dtf) -import click - @click.group() def main() -> None: pass diff --git a/my/core/init.py b/my/core/init.py index bec3a9a..49148de 100644 --- a/my/core/init.py +++ b/my/core/init.py @@ -14,9 +14,9 @@ Please let me know if you are aware of a better way of dealing with this! # separate function to present namespace pollution def setup_config() -> None: - from pathlib import Path import sys import warnings + from pathlib import Path from .preinit import get_mycfg_dir mycfg_dir = get_mycfg_dir() diff --git a/my/core/konsume.py b/my/core/konsume.py index 10bea8d..ac1b100 100644 --- a/my/core/konsume.py +++ b/my/core/konsume.py @@ -94,6 +94,8 @@ class Wvalue(Zoomable): from typing import Tuple + + def _wrap(j, parent=None) -> Tuple[Zoomable, List[Zoomable]]: res: Zoomable cc: List[Zoomable] @@ -123,6 +125,7 @@ def _wrap(j, parent=None) -> Tuple[Zoomable, List[Zoomable]]: from contextlib import contextmanager from typing import Iterator + class UnconsumedError(Exception): pass @@ -146,6 +149,8 @@ Expected {c} to be fully consumed by the parser. from typing import cast + + def test_unconsumed() -> None: import pytest with pytest.raises(UnconsumedError): diff --git a/my/core/logging.py b/my/core/logging.py index 882ab12..734c1e0 100644 --- a/my/core/logging.py +++ b/my/core/logging.py @@ -1,11 +1,11 @@ from __future__ import annotations -from functools import lru_cache import logging import os import sys -from typing import Union, TYPE_CHECKING import warnings +from functools import lru_cache +from typing import TYPE_CHECKING, Union def test() -> None: @@ -222,7 +222,9 @@ def make_logger(name: str, *, level: LevelIsh = None) -> logging.Logger: # OK, when stdout is not a tty, enlighten doesn't log anything, good def get_enlighten(): # TODO could add env variable to disable enlighten for a module? - from unittest.mock import Mock # Mock to return stub so cients don't have to think about it + from unittest.mock import ( + Mock, # Mock to return stub so cients don't have to think about it + ) # for now hidden behind the flag since it's a little experimental if os.environ.get('ENLIGHTEN_ENABLE', None) is None: diff --git a/my/core/orgmode.py b/my/core/orgmode.py index 5894b23..d9a254c 100644 --- a/my/core/orgmode.py +++ b/my/core/orgmode.py @@ -2,6 +2,8 @@ Various helpers for reading org-mode data """ from datetime import datetime + + def parse_org_datetime(s: str) -> datetime: s = s.strip('[]') for fmt, cl in [ @@ -21,8 +23,10 @@ def parse_org_datetime(s: str) -> datetime: # TODO I guess want to borrow inspiration from bs4? element type <-> tag; and similar logic for find_one, find_all +from typing import Callable, Iterable, TypeVar + from orgparse import OrgNode -from typing import Iterable, TypeVar, Callable + V = TypeVar('V') def collect(n: OrgNode, cfun: Callable[[OrgNode], Iterable[V]]) -> Iterable[V]: @@ -32,6 +36,8 @@ def collect(n: OrgNode, cfun: Callable[[OrgNode], Iterable[V]]) -> Iterable[V]: from more_itertools import one from orgparse.extra import Table + + def one_table(o: OrgNode) -> Table: return one(collect(o, lambda n: (x for x in n.body_rich if isinstance(x, Table)))) diff --git a/my/core/pandas.py b/my/core/pandas.py index 5688aa3..8abbb1f 100644 --- a/my/core/pandas.py +++ b/my/core/pandas.py @@ -8,15 +8,14 @@ from __future__ import annotations import dataclasses from datetime import datetime, timezone from pprint import pformat -from typing import TYPE_CHECKING, Any, Iterable, Type, Dict, Literal, Callable, TypeVar +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, Literal, Type, TypeVar from decorator import decorator -from . import warnings, Res +from . import Res, warnings +from .error import error_to_json, extract_error_datetime from .logging import make_logger from .types import Json, asdict -from .error import error_to_json, extract_error_datetime - logger = make_logger(__name__) diff --git a/my/core/preinit.py b/my/core/preinit.py index 8c0f6a4..be5477b 100644 --- a/my/core/preinit.py +++ b/my/core/preinit.py @@ -1,11 +1,13 @@ from pathlib import Path + # todo preinit isn't really a good name? it's only in a separate file because # - it's imported from my.core.init (so we wan't to keep this file as small/reliable as possible, hence not common or something) # - we still need this function in __main__, so has to be separate from my/core/init.py def get_mycfg_dir() -> Path: - import appdirs # type: ignore[import-untyped] import os + + import appdirs # type: ignore[import-untyped] # not sure if that's necessary, i.e. could rely on PYTHONPATH instead # on the other hand, by using MY_CONFIG we are guaranteed to load it from the desired path? mvar = os.environ.get('MY_CONFIG') diff --git a/my/core/query.py b/my/core/query.py index 4d7363e..cf85b1b 100644 --- a/my/core/query.py +++ b/my/core/query.py @@ -10,16 +10,27 @@ import importlib import inspect import itertools from datetime import datetime -from typing import TypeVar, Tuple, Optional, Union, Callable, Iterable, Iterator, Dict, Any, NamedTuple, List +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + NamedTuple, + Optional, + Tuple, + TypeVar, + Union, +) import more_itertools from . import error as err -from .types import is_namedtuple from .error import Res, unwrap +from .types import is_namedtuple from .warnings import low - T = TypeVar("T") ET = Res[T] @@ -687,9 +698,10 @@ def test_raise_exceptions() -> None: def test_wrap_unsortable_with_error_and_warning() -> None: - import pytest from collections import Counter + import pytest + # by default should wrap unsortable (error) with pytest.warns(UserWarning, match=r"encountered exception"): res = list(select(_mixed_iter_errors(), order_value=lambda o: isinstance(o, datetime))) diff --git a/my/core/query_range.py b/my/core/query_range.py index 2b3a3d3..d077225 100644 --- a/my/core/query_range.py +++ b/my/core/query_range.py @@ -9,24 +9,22 @@ See the select_range function below import re import time +from datetime import date, datetime, timedelta from functools import lru_cache -from datetime import datetime, timedelta, date -from typing import Callable, Iterator, NamedTuple, Optional, Any, Type +from typing import Any, Callable, Iterator, NamedTuple, Optional, Type import more_itertools +from .compat import fromisoformat from .query import ( - QueryException, - select, + ET, OrderFunc, + QueryException, Where, _handle_generate_order_by, - ET, + select, ) -from .compat import fromisoformat - - timedelta_regex = re.compile(r"^((?P[\.\d]+?)w)?((?P[\.\d]+?)d)?((?P[\.\d]+?)h)?((?P[\.\d]+?)m)?((?P[\.\d]+?)s)?$") diff --git a/my/core/serialize.py b/my/core/serialize.py index e38bca5..b196d47 100644 --- a/my/core/serialize.py +++ b/my/core/serialize.py @@ -1,13 +1,13 @@ import datetime -from dataclasses import is_dataclass, asdict -from pathlib import Path +from dataclasses import asdict, is_dataclass from decimal import Decimal -from typing import Any, Optional, Callable, NamedTuple from functools import lru_cache +from pathlib import Path +from typing import Any, Callable, NamedTuple, Optional from .error import error_to_json -from .types import is_namedtuple from .pytest import parametrize +from .types import is_namedtuple # note: it would be nice to combine the 'asdict' and _default_encode to some function # that takes a complex python object and returns JSON-compatible fields, while still @@ -117,6 +117,7 @@ def _dumps_factory(**kwargs) -> Callable[[Any], str]: def stdlib_factory() -> Optional[Dumps]: import json + from .warnings import high high( diff --git a/my/core/source.py b/my/core/source.py index 6d0f0fd..9488ae2 100644 --- a/my/core/source.py +++ b/my/core/source.py @@ -3,9 +3,9 @@ Decorator to gracefully handle importing a data source, or warning and yielding nothing (or a default) when its not available """ -from functools import wraps -from typing import Any, Iterator, TypeVar, Callable, Optional, Iterable import warnings +from functools import wraps +from typing import Any, Callable, Iterable, Iterator, Optional, TypeVar from .warnings import medium diff --git a/my/core/sqlite.py b/my/core/sqlite.py index 4a471a9..47bd78b 100644 --- a/my/core/sqlite.py +++ b/my/core/sqlite.py @@ -1,13 +1,12 @@ from .internal import assert_subpackage; assert_subpackage(__name__) -from contextlib import contextmanager -from pathlib import Path import shutil import sqlite3 +from contextlib import contextmanager +from pathlib import Path from tempfile import TemporaryDirectory -from typing import Tuple, Any, Iterator, Callable, Optional, Union, Literal - +from typing import Any, Callable, Iterator, Literal, Optional, Tuple, Union, overload from .common import PathIsh from .compat import assert_never @@ -98,7 +97,6 @@ def sqlite_copy_and_open(db: PathIsh) -> sqlite3.Connection: # and then the return type ends up as Iterator[Tuple[str, ...]], which isn't desirable :( # a bit annoying to have this copy-pasting, but hopefully not a big issue -from typing import overload @overload def select(cols: Tuple[str ], rest: str, *, db: sqlite3.Connection) -> \ Iterator[Tuple[Any ]]: ... diff --git a/my/core/stats.py b/my/core/stats.py index d724068..08821a2 100644 --- a/my/core/stats.py +++ b/my/core/stats.py @@ -3,13 +3,13 @@ Helpers for hpi doctor/stats functionality. ''' import collections -from contextlib import contextmanager -from datetime import datetime import importlib import inspect +import typing +from contextlib import contextmanager +from datetime import datetime from pathlib import Path from types import ModuleType -import typing from typing import ( Any, Callable, @@ -26,7 +26,6 @@ from typing import ( from .types import asdict - Stats = Dict[str, Any] @@ -133,8 +132,8 @@ def test_stat() -> None: # # works with pandas dataframes - import pandas as pd import numpy as np + import pandas as pd def df() -> pd.DataFrame: dates = pd.date_range(start='2024-02-10 08:00', end='2024-02-11 16:00', freq='5h') @@ -357,7 +356,7 @@ def _stat_item(item): def _stat_iterable(it: Iterable[Any], quick: bool = False) -> Stats: - from more_itertools import ilen, take, first + from more_itertools import first, ilen, take # todo not sure if there is something in more_itertools to compute this? total = 0 @@ -448,6 +447,7 @@ def _guess_datetime(x: Any) -> Optional[datetime]: def test_guess_datetime() -> None: from dataclasses import dataclass from typing import NamedTuple + from .compat import fromisoformat dd = fromisoformat('2021-02-01T12:34:56Z') diff --git a/my/core/structure.py b/my/core/structure.py index 458440e..df25e37 100644 --- a/my/core/structure.py +++ b/my/core/structure.py @@ -1,16 +1,14 @@ +import atexit import os import shutil import tempfile import zipfile -import atexit - -from typing import Sequence, Generator, List, Union, Tuple from contextlib import contextmanager from pathlib import Path +from typing import Generator, List, Sequence, Tuple, Union from .logging import make_logger - logger = make_logger(__name__, level="info") diff --git a/my/core/tests/auto_stats.py b/my/core/tests/auto_stats.py index bf4764c..d10d4c4 100644 --- a/my/core/tests/auto_stats.py +++ b/my/core/tests/auto_stats.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path -from typing import Iterable, Sequence, Iterator +from typing import Iterable, Iterator, Sequence @dataclass diff --git a/my/core/tests/common.py b/my/core/tests/common.py index a102ad3..22a74d7 100644 --- a/my/core/tests/common.py +++ b/my/core/tests/common.py @@ -1,10 +1,9 @@ -from contextlib import contextmanager import os +from contextlib import contextmanager from typing import Iterator, Optional import pytest - V = 'HPI_TESTS_USES_OPTIONAL_DEPS' # TODO use it for serialize tests that are using simplejson/orjson? diff --git a/my/core/tests/denylist.py b/my/core/tests/denylist.py index cca757d..8016282 100644 --- a/my/core/tests/denylist.py +++ b/my/core/tests/denylist.py @@ -1,8 +1,8 @@ -from datetime import datetime import json -from pathlib import Path -from typing import NamedTuple, Iterator import warnings +from datetime import datetime +from pathlib import Path +from typing import Iterator, NamedTuple from ..denylist import DenyList diff --git a/my/core/tests/sqlite.py b/my/core/tests/sqlite.py index b3ecffe..1ad0748 100644 --- a/my/core/tests/sqlite.py +++ b/my/core/tests/sqlite.py @@ -1,7 +1,7 @@ -from concurrent.futures import ProcessPoolExecutor -from pathlib import Path import shutil import sqlite3 +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path from tempfile import TemporaryDirectory from ..sqlite import sqlite_connect_immutable, sqlite_copy_and_open diff --git a/my/core/tests/structure.py b/my/core/tests/structure.py index beb8e7f..6a94fc4 100644 --- a/my/core/tests/structure.py +++ b/my/core/tests/structure.py @@ -1,9 +1,8 @@ from pathlib import Path -from ..structure import match_structure - import pytest +from ..structure import match_structure structure_data: Path = Path(__file__).parent / "structure_data" diff --git a/my/core/tests/test_cachew.py b/my/core/tests/test_cachew.py index 5f7dd65..70ac76f 100644 --- a/my/core/tests/test_cachew.py +++ b/my/core/tests/test_cachew.py @@ -35,8 +35,7 @@ def test_cachew_dir_none() -> None: settings.ENABLE = True # by default it's off in tests (see conftest.py) - from my.core.cachew import cache_dir - from my.core.cachew import mcachew + from my.core.cachew import cache_dir, mcachew from my.core.core_config import _reset_config as reset with reset() as cc: diff --git a/my/core/tests/test_cli.py b/my/core/tests/test_cli.py index 4d847ae..1838e84 100644 --- a/my/core/tests/test_cli.py +++ b/my/core/tests/test_cli.py @@ -1,6 +1,6 @@ import os -from subprocess import check_call import sys +from subprocess import check_call def test_lists_modules() -> None: diff --git a/my/core/tests/test_get_files.py b/my/core/tests/test_get_files.py index 52e43f8..68be4d9 100644 --- a/my/core/tests/test_get_files.py +++ b/my/core/tests/test_get_files.py @@ -1,15 +1,15 @@ import os -from pathlib import Path import shutil import tempfile -from typing import TYPE_CHECKING import zipfile +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest from ..common import get_files from ..kompress import CPath, ZipPath -import pytest - # hack to replace all /tmp with 'real' tmp dir # not ideal, but makes tests more concise diff --git a/my/core/time.py b/my/core/time.py index 430b082..83a407b 100644 --- a/my/core/time.py +++ b/my/core/time.py @@ -1,5 +1,5 @@ from functools import lru_cache -from typing import Sequence, Dict +from typing import Dict, Sequence import pytz diff --git a/my/core/types.py b/my/core/types.py index c1b0add..b1cf103 100644 --- a/my/core/types.py +++ b/my/core/types.py @@ -1,13 +1,13 @@ from .internal import assert_subpackage; assert_subpackage(__name__) -from dataclasses import is_dataclass, asdict as dataclasses_asdict +from dataclasses import asdict as dataclasses_asdict +from dataclasses import is_dataclass from datetime import datetime from typing import ( Any, Dict, ) - Json = Dict[str, Any] diff --git a/my/core/util.py b/my/core/util.py index 57e41d4..b48a450 100644 --- a/my/core/util.py +++ b/my/core/util.py @@ -1,21 +1,24 @@ -from pathlib import Path -from itertools import chain import os import pkgutil import sys -from typing import List, Iterable, Optional +from itertools import chain +from pathlib import Path +from types import ModuleType +from typing import Iterable, List, Optional -from .discovery_pure import HPIModule, ignored, _is_not_module_src, has_stats +from .discovery_pure import HPIModule, _is_not_module_src, has_stats, ignored def modules() -> Iterable[HPIModule]: import my + for m in _iter_all_importables(my): yield m __NOT_HPI_MODULE__ = 'Import this to mark a python file as a helper, not an actual HPI module' from .discovery_pure import NOT_HPI_MODULE_VAR + assert NOT_HPI_MODULE_VAR in globals() # check name consistency def is_not_hpi_module(module: str) -> Optional[str]: @@ -23,6 +26,7 @@ def is_not_hpi_module(module: str) -> Optional[str]: None if a module, otherwise returns reason ''' import importlib + path: Optional[str] = None try: # TODO annoying, this can cause import of the parent module? @@ -41,7 +45,6 @@ def is_not_hpi_module(module: str) -> Optional[str]: return None -from types import ModuleType # todo reuse in readme/blog post # borrowed from https://github.com/sanitizers/octomachinery/blob/24288774d6dcf977c5033ae11311dbff89394c89/tests/circular_imports_test.py#L22-L55 def _iter_all_importables(pkg: ModuleType) -> Iterable[HPIModule]: @@ -192,6 +195,7 @@ from my.core import __NOT_HPI_MODULE__ ''') import sys + orig_path = list(sys.path) try: sys.path.insert(0, str(badp)) @@ -226,6 +230,7 @@ def stats(): ''') import sys + orig_path = list(sys.path) try: sys.path.insert(0, str(badp)) diff --git a/my/core/utils/concurrent.py b/my/core/utils/concurrent.py index cc17cda..3553cd9 100644 --- a/my/core/utils/concurrent.py +++ b/my/core/utils/concurrent.py @@ -1,10 +1,9 @@ -from concurrent.futures import Future, Executor import sys -from typing import Any, Callable, Optional, TypeVar, TYPE_CHECKING +from concurrent.futures import Executor, Future +from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar from ..compat import ParamSpec - _P = ParamSpec('_P') _T = TypeVar('_T') @@ -15,6 +14,7 @@ class DummyExecutor(Executor): This is useful if you're already using Executor for parallelising, but also want to provide an option to run the code serially (e.g. for debugging) """ + def __init__(self, max_workers: Optional[int] = 1) -> None: self._shutdown = False self._max_workers = max_workers diff --git a/my/core/utils/imports.py b/my/core/utils/imports.py index efd8e9a..4666a5e 100644 --- a/my/core/utils/imports.py +++ b/my/core/utils/imports.py @@ -1,9 +1,9 @@ import importlib import importlib.util -from pathlib import Path import sys -from typing import Optional +from pathlib import Path from types import ModuleType +from typing import Optional from ..common import PathIsh diff --git a/my/core/utils/itertools.py b/my/core/utils/itertools.py index e8802bb..023484d 100644 --- a/my/core/utils/itertools.py +++ b/my/core/utils/itertools.py @@ -4,8 +4,10 @@ Various helpers/transforms of iterators Ideally this should be as small as possible and we should rely on stdlib itertools or more_itertools """ +import warnings from collections.abc import Hashable from typing import ( + TYPE_CHECKING, Callable, Dict, Iterable, @@ -13,18 +15,16 @@ from typing import ( List, Optional, Sized, - Union, TypeVar, + Union, cast, - TYPE_CHECKING, ) -import warnings + +import more_itertools +from decorator import decorator from ..compat import ParamSpec -from decorator import decorator -import more_itertools - T = TypeVar('T') K = TypeVar('K') V = TypeVar('V') @@ -268,7 +268,9 @@ def check_if_hashable(iterable: Iterable[_HT]) -> Iterable[_HT]: def test_check_if_hashable() -> None: from dataclasses import dataclass from typing import Set, Tuple + import pytest + from ..compat import assert_type x1: List[int] = [1, 2] @@ -353,6 +355,7 @@ def unique_everseen( def test_unique_everseen() -> None: import pytest + from ..tests.common import tmp_environ_set def fun_good() -> Iterator[int]: diff --git a/my/core/warnings.py b/my/core/warnings.py index 7051f34..82e539b 100644 --- a/my/core/warnings.py +++ b/my/core/warnings.py @@ -6,8 +6,8 @@ E.g. would be nice to propagate the warnings in the UI (it's even a subclass of ''' import sys -from typing import Optional import warnings +from typing import TYPE_CHECKING, Optional import click @@ -48,5 +48,11 @@ def high(message: str, *args, **kwargs) -> None: _warn(message, *args, **kwargs) -# NOTE: deprecated -- legacy import -from warnings import warn \ No newline at end of file +if not TYPE_CHECKING: + from .compat import deprecated + + @deprecated('use warnings.warn directly instead') + def warn(*args, **kwargs): + import warnings + + return warnings.warn(*args, **kwargs) diff --git a/my/location/common.py b/my/location/common.py index 7824bef..510e005 100644 --- a/my/location/common.py +++ b/my/location/common.py @@ -41,9 +41,9 @@ def locations_to_gpx(locations: Iterable[LocationProtocol], buffer: TextIO) -> I try: import gpxpy.gpx except ImportError as ie: - from my.core.warnings import warn + from my.core.warnings import high - warn("gpxpy not installed, cannot write to gpx. 'pip install gpxpy'") + high("gpxpy not installed, cannot write to gpx. 'pip install gpxpy'") raise ie gpx = gpxpy.gpx.GPX()