From b429ef23f6050ef9aa526825ca95fc7997121f01 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 16 Mar 2019 13:25:33 +0000 Subject: [PATCH] use multiprocessing to speed up unpacking --- reddit/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/reddit/__init__.py b/reddit/__init__.py index f57e342..4e6e10d 100755 --- a/reddit/__init__.py +++ b/reddit/__init__.py @@ -7,6 +7,7 @@ import pytz import re from datetime import datetime import logging +from multiprocessing import Pool from kython import kompress, cproperty, make_dict @@ -128,14 +129,16 @@ def get_events(all_=True) -> List[Event]: # TODO suppress first batch?? # TODO for initial batch, treat event time as creation time + with Pool() as p: + states = p.map(get_state, backups) + RE = re.compile(r'reddit-(\d{14})') - for i, b in enumerate(backups): # TODO when date... + for i, (b, saves) in enumerate(zip(backups, states)): # TODO when date... match = RE.search(b.stem) assert match is not None btime = pytz.utc.localize(datetime.strptime(match.group(1), "%Y%m%d%H%M%S")) first = i == 0 - saves = get_state(b) def etime(dt: datetime): if first: