misc fixes
- convert import_source to a decorator which wraps the function call in a try block - fix protocol class when not TYPE_CHECKING - add id properties to Protocols, remove attributes since protocol expects them to be settable but NT is read-only - use id to merge comments - remove type: ignore's from reddit config and just store as 'Any'
This commit is contained in:
parent
33f7f48ec5
commit
4492e00250
5 changed files with 118 additions and 71 deletions
|
@ -7,6 +7,7 @@ REQUIRES = [
|
|||
|
||||
from my.core.common import Paths
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from my.config import reddit as uconfig
|
||||
|
||||
|
@ -18,21 +19,23 @@ from my.config import reddit as uconfig
|
|||
# in the migration
|
||||
|
||||
# need to check before we subclass
|
||||
conf: Any
|
||||
|
||||
if hasattr(uconfig, "rexport"):
|
||||
# sigh... backwards compatability
|
||||
uconfig = uconfig.rexport # type: ignore[attr-defined,misc,assignment]
|
||||
conf = uconfig.rexport
|
||||
else:
|
||||
from my.core.warnings import high
|
||||
high(f"""DEPRECATED! Please modify your reddit config to look like:
|
||||
high("""DEPRECATED! Please modify your reddit config to look like:
|
||||
|
||||
class reddit:
|
||||
class rexport:
|
||||
export_path: Paths = '/path/to/rexport/data'
|
||||
""")
|
||||
conf = uconfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class reddit(uconfig):
|
||||
class reddit(conf):
|
||||
'''
|
||||
Uses [[https://github.com/karlicoss/rexport][rexport]] output.
|
||||
'''
|
||||
|
@ -81,7 +84,7 @@ def inputs() -> Sequence[Path]:
|
|||
return get_files(config.export_path)
|
||||
|
||||
|
||||
Sid = dal.Sid # str
|
||||
Uid = dal.Sid # str
|
||||
Save = dal.Save
|
||||
Comment = dal.Comment
|
||||
Submission = dal.Submission
|
||||
|
@ -161,7 +164,7 @@ def _get_bdate(bfile: Path) -> datetime:
|
|||
return bdt
|
||||
|
||||
|
||||
def _get_state(bfile: Path) -> Dict[Sid, SaveWithDt]:
|
||||
def _get_state(bfile: Path) -> Dict[Uid, SaveWithDt]:
|
||||
logger.debug('handling %s', bfile)
|
||||
|
||||
bdt = _get_bdate(bfile)
|
||||
|
@ -178,11 +181,11 @@ def _get_state(bfile: Path) -> Dict[Sid, SaveWithDt]:
|
|||
def _get_events(backups: Sequence[Path], parallel: bool=True) -> Iterator[Event]:
|
||||
# todo cachew: let it transform return type? so you don't have to write a wrapper for lists?
|
||||
|
||||
prev_saves: Mapping[Sid, SaveWithDt] = {}
|
||||
prev_saves: Mapping[Uid, SaveWithDt] = {}
|
||||
# TODO suppress first batch??
|
||||
# TODO for initial batch, treat event time as creation time
|
||||
|
||||
states: Iterable[Mapping[Sid, SaveWithDt]]
|
||||
states: Iterable[Mapping[Uid, SaveWithDt]]
|
||||
if parallel:
|
||||
with Pool() as p:
|
||||
states = p.map(_get_state, backups)
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue