fbmessenger: add all.py, merge messages from different sources

followup for https://github.com/karlicoss/HPI/pull/179
This commit is contained in:
Dima Gerasimov 2022-02-05 16:19:53 +00:00 committed by karlicoss
parent 4626c1bba6
commit f8e73134b3
4 changed files with 98 additions and 4 deletions

46
my/fbmessenger/common.py Normal file
View file

@ -0,0 +1,46 @@
from datetime import datetime
from typing import Iterator, Optional, TYPE_CHECKING
if TYPE_CHECKING:
try:
from typing import Protocol
except ImportError:
# requirement of mypy
from typing_extensions import Protocol # type: ignore[misc]
else:
Protocol = object
class Thread(Protocol):
@property
def id(self) -> str: ...
# todo hmm it doesn't like it because one from .export is just str, not Optional...
# name: Optional[str]
class Message(Protocol):
@property
def id(self) -> str: ...
@property
def dt(self) -> datetime: ...
@property
def text(self) -> Optional[str]: ...
@property
def thread(self) -> Thread: ...
from itertools import chain
from more_itertools import unique_everseen
from my.core import Res
def _merge_messages(*sources: Iterator[Res[Message]]) -> Iterator[Res[Message]]:
# todo might be nice to dump some stats for debugging, e.g. how many were overlapping?
def key(r: Res[Message]):
if isinstance(r, Exception):
return str(r)
else:
return r.id
yield from unique_everseen(chain(*sources), key=key)