fz0x1/HPI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

make parallel processing optional

Browse source
This commit is contained in:
Dima Gerasimov 2019-03-17 15:09:24 +00:00
parent b429ef23f6
commit 1ad2a5bffa
1 changed files with 8 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
reddit/__init__.py
View file

@ -120,7 +120,7 @@ def get_state(bfile: Path) -> Dict[Sid, Save]:
return OrderedDict() return OrderedDict()
def get_events(all_=True) -> List[Event]: def get_events(all_=True, parallel=True) -> List[Event]:
backups = _get_backups(all_=all_) backups = _get_backups(all_=all_)
assert len(backups) > 0 assert len(backups) > 0
@ -129,8 +129,13 @@ def get_events(all_=True) -> List[Event]:
# TODO suppress first batch?? # TODO suppress first batch??
# TODO for initial batch, treat event time as creation time # TODO for initial batch, treat event time as creation time
with Pool() as p: states: Iterable[Dict[Sid, Save]]
states = p.map(get_state, backups) if parallel:
with Pool() as p:
states = p.map(get_state, backups)
else:
# also make it lazy...
states = map(get_state, backups)
RE = re.compile(r'reddit-(\d{14})') RE = re.compile(r'reddit-(\d{14})')
for i, (b, saves) in enumerate(zip(backups, states)): # TODO when date... for i, (b, saves) in enumerate(zip(backups, states)): # TODO when date...