diff --git a/my/core/common.py b/my/core/common.py index 0be4dae..920657a 100644 --- a/my/core/common.py +++ b/my/core/common.py @@ -65,29 +65,6 @@ def import_dir(path: PathIsh, extra: str='') -> types.ModuleType: return import_from(p.parent, p.name + extra) -# https://stackoverflow.com/a/12377059/706389 -def listify(fn=None, wrapper=list): - """ - Wraps a function's return value in wrapper (e.g. list) - Useful when an algorithm can be expressed more cleanly as a generator - """ - def listify_return(fn): - @functools.wraps(fn) - def listify_helper(*args, **kw): - return wrapper(fn(*args, **kw)) - return listify_helper - if fn is None: - return listify_return - return listify_return(fn) - - -# todo use in bluemaestro -# def dictify(fn=None, key=None, value=None): -# def md(it): -# return make_dict(it, key=key, value=value) -# return listify(fn=fn, wrapper=md) - - from .logging import setup_logger, LazyLogger @@ -628,12 +605,18 @@ if not TYPE_CHECKING: res[kk] = lst return res - @deprecated('use my.core.utils.make_dict instead') + @deprecated('use my.core.utils.itertools.make_dict instead') def make_dict(*args, **kwargs): from .utils import itertools as UI return UI.make_dict(*args, **kwargs) + @deprecated('use my.core.utils.itertools.listify instead') + def listify(*args, **kwargs): + from .utils import itertools as UI + + return UI.listify(*args, **kwargs) + # todo wrap these in deprecated decorator as well? from .cachew import mcachew # noqa: F401 diff --git a/my/core/utils/itertools.py b/my/core/utils/itertools.py index 78b91de..cab4b2c 100644 --- a/my/core/utils/itertools.py +++ b/my/core/utils/itertools.py @@ -4,8 +4,11 @@ Various helpers/transforms of iterators Ideally this should be as small as possible and we should rely on stdlib itertools or more_itertools """ -from typing import Callable, Dict, Iterable, TypeVar, cast +from typing import Callable, Dict, Iterable, Iterator, TypeVar, List, cast, TYPE_CHECKING +from ..compat import ParamSpec + +from decorator import decorator T = TypeVar('T') K = TypeVar('K') @@ -75,3 +78,39 @@ def test_make_dict() -> None: # check type inference d2: Dict[str, int] = make_dict(it, key=lambda i: str(i)) d3: Dict[str, bool] = make_dict(it, key=lambda i: str(i), value=lambda i: i % 2 == 0) + + +LFP = ParamSpec('LFP') +LV = TypeVar('LV') + + +@decorator +def _listify(func: Callable[LFP, Iterable[LV]], *args: LFP.args, **kwargs: LFP.kwargs) -> List[LV]: + """ + Wraps a function's return value in wrapper (e.g. list) + Useful when an algorithm can be expressed more cleanly as a generator + """ + return list(func(*args, **kwargs)) + + +# ugh. decorator library has stub types, but they are way too generic? +# tried implementing my own stub, but failed -- not sure if it's possible at all? +# so seems easiest to just use specialize instantiations of decorator instead +if TYPE_CHECKING: + + def listify(func: Callable[LFP, Iterable[LV]]) -> Callable[LFP, List[LV]]: ... + +else: + listify = _listify + + +def test_listify() -> None: + @listify + def it() -> Iterator[int]: + yield 1 + yield 2 + + res = it() + from typing_extensions import assert_type # TODO move to compat? + assert_type(res, List[int]) + assert res == [1, 2] diff --git a/my/rss/all.py b/my/rss/all.py index 61f9fab..b4dbdbd 100644 --- a/my/rss/all.py +++ b/my/rss/all.py @@ -1,6 +1,7 @@ ''' Unified RSS data, merged from different services I used historically ''' + # NOTE: you can comment out the sources you're not using from . import feedbin, feedly @@ -12,5 +13,5 @@ def subscriptions() -> Iterable[Subscription]: # TODO google reader? yield from compute_subscriptions( feedbin.states(), - feedly .states(), + feedly.states(), ) diff --git a/my/rss/common.py b/my/rss/common.py index f3893b7..54067d6 100644 --- a/my/rss/common.py +++ b/my/rss/common.py @@ -1,30 +1,32 @@ -# shared Rss stuff -from datetime import datetime -from typing import NamedTuple, Optional, List, Dict +from my.core import __NOT_HPI_MODULE__ + +from dataclasses import dataclass, replace +from itertools import chain +from typing import Optional, List, Dict, Iterable, Tuple, Sequence + +from my.core import warn_if_empty, datetime_aware -class Subscription(NamedTuple): +@dataclass +class Subscription: title: str url: str - id: str # TODO not sure about it... + id: str # TODO not sure about it... # eh, not all of them got reasonable 'created' time - created_at: Optional[datetime] - subscribed: bool=True + created_at: Optional[datetime_aware] + subscribed: bool = True -from typing import Iterable, Tuple, Sequence # snapshot of subscriptions at time -SubscriptionState = Tuple[datetime, Sequence[Subscription]] +SubscriptionState = Tuple[datetime_aware, Sequence[Subscription]] -from ..core import warn_if_empty @warn_if_empty def compute_subscriptions(*sources: Iterable[SubscriptionState]) -> List[Subscription]: """ Keeps track of everything I ever subscribed to. In addition, keeps track of unsubscribed as well (so you'd remember when and why you unsubscribed) """ - from itertools import chain states = list(chain.from_iterable(sources)) # TODO keep 'source'/'provider'/'service' attribute? @@ -45,7 +47,5 @@ def compute_subscriptions(*sources: Iterable[SubscriptionState]) -> List[Subscri res = [] for u, x in sorted(by_url.items()): present = u in last_urls - res.append(x._replace(subscribed=present)) + res.append(replace(x, subscribed=present)) return res - -from ..core import __NOT_HPI_MODULE__ diff --git a/my/rss/feedbin.py b/my/rss/feedbin.py index 16d4417..dc13a17 100644 --- a/my/rss/feedbin.py +++ b/my/rss/feedbin.py @@ -2,24 +2,22 @@ Feedbin RSS reader """ -from my.config import feedbin as config - +import json from pathlib import Path -from typing import Sequence +from typing import Iterator, Sequence -from my.core.common import listify, get_files +from my.core import get_files, stat, Stats from my.core.compat import fromisoformat -from .common import Subscription +from .common import Subscription, SubscriptionState + +from my.config import feedbin as config def inputs() -> Sequence[Path]: return get_files(config.export_path) -import json - -@listify -def parse_file(f: Path): +def parse_file(f: Path) -> Iterator[Subscription]: raw = json.loads(f.read_text()) for r in raw: yield Subscription( @@ -30,19 +28,14 @@ def parse_file(f: Path): ) -from typing import Iterable -from .common import SubscriptionState -def states() -> Iterable[SubscriptionState]: +def states() -> Iterator[SubscriptionState]: for f in inputs(): # TODO ugh. depends on my naming. not sure if useful? dts = f.stem.split('_')[-1] dt = fromisoformat(dts) - subs = parse_file(f) + subs = list(parse_file(f)) yield dt, subs -def stats(): - from more_itertools import ilen, last - return { - 'subscriptions': ilen(last(states())[1]) - } +def stats() -> Stats: + return stat(states) diff --git a/my/rss/feedly.py b/my/rss/feedly.py index 4611ced..55bcf9b 100644 --- a/my/rss/feedly.py +++ b/my/rss/feedly.py @@ -1,14 +1,15 @@ """ Feedly RSS reader """ + from my.config import feedly as config from datetime import datetime, timezone import json from pathlib import Path -from typing import Iterable, Sequence +from typing import Iterator, Sequence -from ..core.common import listify, get_files +from my.core import get_files from .common import Subscription, SubscriptionState @@ -16,13 +17,12 @@ def inputs() -> Sequence[Path]: return get_files(config.export_path) -@listify -def parse_file(f: Path): +def parse_file(f: Path) -> Iterator[Subscription]: raw = json.loads(f.read_text()) for r in raw: # err, some even don't have website.. rid = r['id'] - website = r.get('website', rid) # meh + website = r.get('website', rid) # meh yield Subscription( created_at=None, title=r['title'], @@ -31,9 +31,9 @@ def parse_file(f: Path): ) -def states() -> Iterable[SubscriptionState]: +def states() -> Iterator[SubscriptionState]: for f in inputs(): dts = f.stem.split('_')[-1] dt = datetime.strptime(dts, '%Y%m%d%H%M%S').replace(tzinfo=timezone.utc) - subs = parse_file(f) + subs = list(parse_file(f)) yield dt, subs