core.common: move listify to core.utils.itertools, use better typing annotations for it

also some minor refactoring of my.rss

my/core/common.py

@@ -65,29 +65,6 @@ def import_dir(path: PathIsh, extra: str='') -> types.ModuleType:
     return import_from(p.parent, p.name + extra)
 
 
-# https://stackoverflow.com/a/12377059/706389
-def listify(fn=None, wrapper=list):
-    """
-    Wraps a function's return value in wrapper (e.g. list)
-    Useful when an algorithm can be expressed more cleanly as a generator
-    """
-    def listify_return(fn):
-        @functools.wraps(fn)
-        def listify_helper(*args, **kw):
-            return wrapper(fn(*args, **kw))
-        return listify_helper
-    if fn is None:
-        return listify_return
-    return listify_return(fn)
-
-
-# todo use in bluemaestro
-# def dictify(fn=None, key=None, value=None):
-#     def md(it):
-#         return make_dict(it, key=key, value=value)
-#     return listify(fn=fn, wrapper=md)
-
-
 from .logging import setup_logger, LazyLogger
 
 
@@ -628,12 +605,18 @@ if not TYPE_CHECKING:
             res[kk] = lst
         return res
 
-    @deprecated('use my.core.utils.make_dict instead')
+    @deprecated('use my.core.utils.itertools.make_dict instead')
     def make_dict(*args, **kwargs):
         from .utils import itertools as UI
 
         return UI.make_dict(*args, **kwargs)
 
+    @deprecated('use my.core.utils.itertools.listify instead')
+    def listify(*args, **kwargs):
+        from .utils import itertools as UI
+
+        return UI.listify(*args, **kwargs)
+
     # todo wrap these in deprecated decorator as well?
     from .cachew import mcachew  # noqa: F401
 

my/core/utils/itertools.py

@@ -4,8 +4,11 @@ Various helpers/transforms of iterators
 Ideally this should be as small as possible and we should rely on stdlib itertools or more_itertools
 """
 
-from typing import Callable, Dict, Iterable, TypeVar, cast
+from typing import Callable, Dict, Iterable, Iterator, TypeVar, List, cast, TYPE_CHECKING
 
+from ..compat import ParamSpec
+
+from decorator import decorator
 
 T = TypeVar('T')
 K = TypeVar('K')
@@ -75,3 +78,39 @@ def test_make_dict() -> None:
     # check type inference
     d2: Dict[str, int] = make_dict(it, key=lambda i: str(i))
     d3: Dict[str, bool] = make_dict(it, key=lambda i: str(i), value=lambda i: i % 2 == 0)
+
+
+LFP = ParamSpec('LFP')
+LV = TypeVar('LV')
+
+
+@decorator
+def _listify(func: Callable[LFP, Iterable[LV]], *args: LFP.args, **kwargs: LFP.kwargs) -> List[LV]:
+    """
+    Wraps a function's return value in wrapper (e.g. list)
+    Useful when an algorithm can be expressed more cleanly as a generator
+    """
+    return list(func(*args, **kwargs))
+
+
+# ugh. decorator library has stub types, but they are way too generic?
+# tried implementing my own stub, but failed -- not sure if it's possible at all?
+# so seems easiest to just use specialize instantiations of decorator instead
+if TYPE_CHECKING:
+
+    def listify(func: Callable[LFP, Iterable[LV]]) -> Callable[LFP, List[LV]]: ...
+
+else:
+    listify = _listify
+
+
+def test_listify() -> None:
+    @listify
+    def it() -> Iterator[int]:
+        yield 1
+        yield 2
+
+    res = it()
+    from typing_extensions import assert_type  # TODO move to compat?
+    assert_type(res, List[int])
+    assert res == [1, 2]

my/rss/all.py

@@ -1,6 +1,7 @@
 '''
 Unified RSS data, merged from different services I used historically
 '''
+
 # NOTE: you can comment out the sources you're not using
 from . import feedbin, feedly
 
@@ -12,5 +13,5 @@ def subscriptions() -> Iterable[Subscription]:
     # TODO google reader?
     yield from compute_subscriptions(
         feedbin.states(),
-        feedly .states(),
+        feedly.states(),
     )

my/rss/common.py

@@ -1,30 +1,32 @@
-# shared Rss stuff
+from my.core import __NOT_HPI_MODULE__
-from datetime import datetime
+
-from typing import NamedTuple, Optional, List, Dict
+from dataclasses import dataclass, replace
+from itertools import chain
+from typing import Optional, List, Dict, Iterable, Tuple, Sequence
+
+from my.core import warn_if_empty, datetime_aware
 
 
-class Subscription(NamedTuple):
+@dataclass
+class Subscription:
     title: str
     url: str
-    id: str # TODO not sure about it...
+    id: str  # TODO not sure about it...
     # eh, not all of them got reasonable 'created' time
-    created_at: Optional[datetime]
+    created_at: Optional[datetime_aware]
-    subscribed: bool=True
+    subscribed: bool = True
 
-from typing import Iterable, Tuple, Sequence
 
 # snapshot of subscriptions at time
-SubscriptionState = Tuple[datetime, Sequence[Subscription]]
+SubscriptionState = Tuple[datetime_aware, Sequence[Subscription]]
 
 
-from ..core import warn_if_empty
 @warn_if_empty
 def compute_subscriptions(*sources: Iterable[SubscriptionState]) -> List[Subscription]:
     """
     Keeps track of everything I ever subscribed to.
     In addition, keeps track of unsubscribed as well (so you'd remember when and why you unsubscribed)
     """
-    from itertools import chain
     states = list(chain.from_iterable(sources))
     # TODO keep 'source'/'provider'/'service' attribute?
 
@@ -45,7 +47,5 @@ def compute_subscriptions(*sources: Iterable[SubscriptionState]) -> List[Subscri
     res = []
     for u, x in sorted(by_url.items()):
         present = u in last_urls
-        res.append(x._replace(subscribed=present))
+        res.append(replace(x, subscribed=present))
     return res
-
-from ..core import __NOT_HPI_MODULE__

my/rss/feedbin.py

@@ -2,24 +2,22 @@
 Feedbin RSS reader
 """
 
-from my.config import feedbin as config
+import json
-
 from pathlib import Path
-from typing import Sequence
+from typing import Iterator, Sequence
 
-from my.core.common import listify, get_files
+from my.core import get_files, stat, Stats
 from my.core.compat import fromisoformat
-from .common import Subscription
+from .common import Subscription, SubscriptionState
+
+from my.config import feedbin as config
 
 
 def inputs() -> Sequence[Path]:
     return get_files(config.export_path)
 
 
-import json
+def parse_file(f: Path) -> Iterator[Subscription]:
-
-@listify
-def parse_file(f: Path):
     raw = json.loads(f.read_text())
     for r in raw:
         yield Subscription(
@@ -30,19 +28,14 @@ def parse_file(f: Path):
         )
 
 
-from typing import Iterable
+def states() -> Iterator[SubscriptionState]:
-from .common import SubscriptionState
-def states() -> Iterable[SubscriptionState]:
     for f in inputs():
         # TODO ugh. depends on my naming. not sure if useful?
         dts = f.stem.split('_')[-1]
         dt = fromisoformat(dts)
-        subs = parse_file(f)
+        subs = list(parse_file(f))
         yield dt, subs
 
 
-def stats():
+def stats() -> Stats:
-    from more_itertools import ilen, last
+    return stat(states)
-    return {
-        'subscriptions': ilen(last(states())[1])
-    }

my/rss/feedly.py

@@ -1,14 +1,15 @@
 """
 Feedly RSS reader
 """
+
 from my.config import feedly as config
 
 from datetime import datetime, timezone
 import json
 from pathlib import Path
-from typing import Iterable, Sequence
+from typing import Iterator, Sequence
 
-from ..core.common import listify, get_files
+from my.core import get_files
 from .common import Subscription, SubscriptionState
 
 
@@ -16,13 +17,12 @@ def inputs() -> Sequence[Path]:
     return get_files(config.export_path)
 
 
-@listify
+def parse_file(f: Path) -> Iterator[Subscription]:
-def parse_file(f: Path):
     raw = json.loads(f.read_text())
     for r in raw:
         # err, some even don't have website..
         rid = r['id']
-        website = r.get('website', rid) # meh
+        website = r.get('website', rid)  # meh
         yield Subscription(
             created_at=None,
             title=r['title'],
@@ -31,9 +31,9 @@ def parse_file(f: Path):
         )
 
 
-def states() -> Iterable[SubscriptionState]:
+def states() -> Iterator[SubscriptionState]:
     for f in inputs():
         dts = f.stem.split('_')[-1]
         dt = datetime.strptime(dts, '%Y%m%d%H%M%S').replace(tzinfo=timezone.utc)
-        subs = parse_file(f)
+        subs = list(parse_file(f))
         yield dt, subs