fz0x1/HPI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix remaining mypy issues in reddit provider

Browse source
This commit is contained in:
Dima Gerasimov 2019-12-08 17:34:00 +00:00
parent 3d82e542fe
commit 77c8328714
1 changed files with 19 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
my/reddit.py
View file

@ -6,9 +6,15 @@ from my_configuration import paths
import my_configuration.repos.rexport.model as rexport import my_configuration.repos.rexport.model as rexport
# TODO Move this to kython.kompress?
class CPath(PosixPath): class CPath(PosixPath):
"""
Ugh. So, can't override Path because of some _flavour thing.
Path only has _accessor and _closed slots, so can't directly set .open method
_accessor.open has to return file descriptor, doesn't work for compressed stuff.
"""
def open(self, *args, **kwargs): def open(self, *args, **kwargs):
# TODO FIXME use something else instead? # TODO assert read only?
from kython import kompress from kython import kompress
return kompress.open(str(self)) return kompress.open(str(self))
@ -51,8 +57,16 @@ def reddit(suffix: str) -> str:
return 'https://reddit.com' + suffix return 'https://reddit.com' + suffix
class SaveWithDt(NamedTuple):
save: Save
backup_dt: datetime
def __getattr__(self, x):
return getattr(self.save, x)
# TODO for future events? # TODO for future events?
EventKind = Save EventKind = SaveWithDt
class Event(NamedTuple): class Event(NamedTuple):
dt: datetime dt: datetime
@ -67,14 +81,6 @@ class Event(NamedTuple):
return (self.dt, (1 if 'unfavorited' in self.text else 0)) return (self.dt, (1 if 'unfavorited' in self.text else 0))
class SaveWithDt(NamedTuple):
save: Save
backup_dt: datetime
def __getattr__(self, x):
return getattr(self.save, x)
Url = str Url = str
def _get_bdate(bfile: Path) -> datetime: def _get_bdate(bfile: Path) -> datetime:
@ -107,17 +113,18 @@ def _get_events(backups: Sequence[Path]=get_backup_files(), parallel: bool=True)
# parallel = False # NOTE: eh, not sure if still necessary? I think glumov didn't like it? # parallel = False # NOTE: eh, not sure if still necessary? I think glumov didn't like it?
logger = get_logger() logger = get_logger()
prev_saves: Mapping[Sid, Save] = {} prev_saves: Mapping[Sid, SaveWithDt] = {}
# TODO suppress first batch?? # TODO suppress first batch??
# TODO for initial batch, treat event time as creation time # TODO for initial batch, treat event time as creation time
states: Iterable[Mapping[Sid, Save]] states: Iterable[Mapping[Sid, SaveWithDt]]
if parallel: if parallel:
with Pool() as p: with Pool() as p:
states = p.map(_get_state, backups) states = p.map(_get_state, backups)
else: else:
# also make it lazy... # also make it lazy...
states = map(_get_state, backups) states = map(_get_state, backups)
# TODO mm, need to make that iterative too?
for i, (bfile, saves) in enumerate(zip(backups, states)): for i, (bfile, saves) in enumerate(zip(backups, states)):
bdt = _get_bdate(bfile) bdt = _get_bdate(bfile)