twitter: use import_source and proper merging for tweets from different sources

+ use proper datetime_aware for created_at
This commit is contained in:
Dima Gerasimov 2022-02-08 20:31:41 +00:00 committed by karlicoss
parent afdf9d4334
commit b9852f45cf
6 changed files with 87 additions and 37 deletions

View file

@ -1,12 +1,21 @@
from my.core import __NOT_HPI_MODULE__
from itertools import chain
from typing import Iterator, Any
from more_itertools import unique_everseen
from ..core import warn_if_empty, __NOT_HPI_MODULE__
# TODO add proper Protocol for Tweet
Tweet = Any
from my.core import warn_if_empty, Res
@warn_if_empty
def merge_tweets(*sources):
yield from unique_everseen(
chain(*sources),
key=lambda t: t.id_str,
)
def merge_tweets(*sources: Iterator[Res[Tweet]]) -> Iterator[Res[Tweet]]:
def key(r: Res[Tweet]):
if isinstance(r, Exception):
return str(r)
else:
return r.id_str
yield from unique_everseen(chain(*sources), key=key)