initial pushshift/rexport merge implementation

This commit is contained in:
Sean Breckenridge 2021-10-28 11:28:00 -07:00
parent b54ec0d7f1
commit 5933711888
8 changed files with 259 additions and 20 deletions

26
my/core/source.py Normal file
View file

@ -0,0 +1,26 @@
from typing import Any, Iterator, TypeVar, Callable, Optional, Iterable
from my.core.warnings import warn
T = TypeVar("T")
# this is probably more generic and results in less code, but is not mypy-friendly
def import_source(factory: Callable[[], Any], default: Any) -> Any:
try:
res = factory()
return res
except ImportError: # presumable means the user hasn't installed the module
warn(f"Module {factory.__name__} could not be imported, or isn't configured propertly")
return default
# For an example of this, see the reddit.all file
def import_source_iter(factory: Callable[[], Iterator[T]], default: Optional[Iterable[T]] = None) -> Iterator[T]:
if default is None:
default = []
try:
res = factory()
yield from res
except ImportError: # presumable means the user hasn't installed the module
warn(f"Module {factory.__name__} could not be imported, or isn't configured propertly")
yield from default