fz0x1/HPI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use multiprocessing to speed up unpacking

Browse source
This commit is contained in:
Dima Gerasimov 2019-03-16 13:25:33 +00:00
parent ea06df9be0
commit b429ef23f6
1 changed files with 5 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
reddit/__init__.py
View file

@ -7,6 +7,7 @@ import pytz
import re import re
from datetime import datetime from datetime import datetime
import logging import logging
from multiprocessing import Pool
from kython import kompress, cproperty, make_dict from kython import kompress, cproperty, make_dict
@ -128,14 +129,16 @@ def get_events(all_=True) -> List[Event]:
# TODO suppress first batch?? # TODO suppress first batch??
# TODO for initial batch, treat event time as creation time # TODO for initial batch, treat event time as creation time
with Pool() as p:
states = p.map(get_state, backups)
RE = re.compile(r'reddit-(\d{14})') RE = re.compile(r'reddit-(\d{14})')
for i, b in enumerate(backups): # TODO when date... for i, (b, saves) in enumerate(zip(backups, states)): # TODO when date...
match = RE.search(b.stem) match = RE.search(b.stem)
assert match is not None assert match is not None
btime = pytz.utc.localize(datetime.strptime(match.group(1), "%Y%m%d%H%M%S")) btime = pytz.utc.localize(datetime.strptime(match.group(1), "%Y%m%d%H%M%S"))
first = i == 0 first = i == 0
saves = get_state(b)
def etime(dt: datetime): def etime(dt: datetime):
if first: if first: