From c2351d530bcf1b7d3aae52993990fc01a3ab375c Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 4 Apr 2019 21:33:40 +0100 Subject: [PATCH 1/6] initial topcoder provider --- coding/topcoder.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 coding/topcoder.py diff --git a/coding/topcoder.py b/coding/topcoder.py new file mode 100644 index 0000000..a4e3f76 --- /dev/null +++ b/coding/topcoder.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +from datetime import datetime +from typing import NamedTuple +from pathlib import Path +import json +from typing import Dict, Iterator + +from kython import cproperty +from kython.konsume import dell, zoom, keq, akeq +from kython.kerror import Res, ytry + + +def get_latest(): + last = max(Path('/L/Dropbox/data/topcoder').glob('*.json')) + return json.loads(last.read_text()) + + +class Competition(NamedTuple): + json: Dict[str, str] + + @cproperty + def contest(self) -> str: + return self.json['challengeName'] + + @cproperty + def when(self) -> str: + return self.json['date'] + + # TODO rating/placement/percentile?? + + @classmethod + def make(cls, json) -> Iterator[Res['Competition']]: + yield cls(json=json) + yield from ytry(lambda: akeq(json, 'challengeName', 'percentile', 'rating', 'placement', 'date')) + + +def iter_data() -> Iterator[Res[Competition]]: + j = get_latest() + dell(j, 'id', 'version') + + j = zoom(j, 'result') + dell(j, 'success', 'status', 'metadata') + + j = zoom(j, 'content') + + dell(j, 'handle', 'handleLower', 'userId', 'createdAt', 'updatedAt', 'createdBy', 'updatedBy') + + dell(j, 'DEVELOP') # TODO handle it?? + j = zoom(j, 'DATA_SCIENCE') + + mar, srm = zoom(j, 'MARATHON_MATCH', 'SRM') + + mar = zoom(mar, 'history') + srm = zoom(srm, 'history') + # TODO right, I guess I could rely on pylint for unused variables?? + + for c in mar + srm: + dell(c, 'challengeId') + yield from Competition.make(json=c) + + +def get_data(): + return list(sorted(iter_data(), key=Competition.when)) + + +def main(): + for d in iter_data(): + print(d) + + +if __name__ == '__main__': + main() From 911449986cf633e7f12365ab62dea2b5c2e9c9bb Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 4 Apr 2019 21:55:46 +0100 Subject: [PATCH 2/6] refine topcoder provider --- coding/topcoder.py | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/coding/topcoder.py b/coding/topcoder.py index a4e3f76..8bea8b5 100644 --- a/coding/topcoder.py +++ b/coding/topcoder.py @@ -3,11 +3,11 @@ from datetime import datetime from typing import NamedTuple from pathlib import Path import json -from typing import Dict, Iterator +from typing import Dict, Iterator, Any -from kython import cproperty +from kython import cproperty, fget from kython.konsume import dell, zoom, keq, akeq -from kython.kerror import Res, ytry +from kython.kerror import Res, ytry, unwrap def get_latest(): @@ -16,22 +16,36 @@ def get_latest(): class Competition(NamedTuple): - json: Dict[str, str] + json: Dict[str, Any] + + @cproperty + def uid(self) -> str: + return self.contest + + def __hash__(self): + return hash(self.json['challengeId']) @cproperty def contest(self) -> str: return self.json['challengeName'] @cproperty - def when(self) -> str: - return self.json['date'] + def when(self) -> datetime: + ds = self.json['date'] + return datetime.strptime(ds, '%Y-%m-%dT%H:%M:%S.%fZ') - # TODO rating/placement/percentile?? + @cproperty + def percentile(self) -> float: + return self.json['percentile'] + + @cproperty + def summary(self) -> str: + return f'participated in {self.contest}: {self.percentile:.0f}' @classmethod def make(cls, json) -> Iterator[Res['Competition']]: yield cls(json=json) - yield from ytry(lambda: akeq(json, 'challengeName', 'percentile', 'rating', 'placement', 'date')) + yield from ytry(lambda: akeq(json, 'challengeId', 'challengeName', 'percentile', 'rating', 'placement', 'date')) def iter_data() -> Iterator[Res[Competition]]: @@ -55,17 +69,20 @@ def iter_data() -> Iterator[Res[Competition]]: # TODO right, I guess I could rely on pylint for unused variables?? for c in mar + srm: - dell(c, 'challengeId') yield from Competition.make(json=c) def get_data(): - return list(sorted(iter_data(), key=Competition.when)) + return list(sorted(iter_data(), key=fget(Competition.when))) def main(): for d in iter_data(): - print(d) + try: + d = unwrap(d) + print(d.summary) + except Exception as e: + print(f'ERROR! {d}') if __name__ == '__main__': From 5709d5e4660acd09081f212405cba5076cd49792 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 4 Apr 2019 22:30:54 +0100 Subject: [PATCH 3/6] codeforces provider --- coding/codeforces.py | 108 +++++++++++++++++++++++++++++++++++++++++++ coding/topcoder.py | 11 +++-- 2 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 coding/codeforces.py diff --git a/coding/codeforces.py b/coding/codeforces.py new file mode 100644 index 0000000..7ab96f0 --- /dev/null +++ b/coding/codeforces.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +from datetime import datetime +from typing import NamedTuple +from pathlib import Path +import json +from typing import Dict, Iterator, Any + +from kython import cproperty, fget +from kython.konsume import dell, zoom, keq, akeq +from kython.kerror import Res, ytry, unwrap +from kython.kdatetime import as_utc + + +_BDIR = Path('/L/Dropbox/data/codeforces') + + +Cid = int + +class Contest(NamedTuple): + cid: Cid + when: datetime + + @classmethod + def make(cls, j) -> 'Contest': + return cls( + cid=j['id'], + when=as_utc(j['startTimeSeconds']), + ) + +Cmap = Dict[Cid, Contest] + +def get_contests() -> Cmap: + last = max(_BDIR.glob('allcontests*.json')) + j = json.loads(last.read_text()) + d = {} + for c in j['result']: + cc = Contest.make(c) + d[cc.cid] = cc + return d + + +def get_latest(): + last = max(_BDIR.glob('codeforces*.json')) + return json.loads(last.read_text()) + + +class Competition(NamedTuple): + json: Dict[str, Any] + cmap: Cmap + + @cproperty + def uid(self) -> str: + return self.contest_id + + @property + def contest_id(self): + return self.json['contestId'] + + def __hash__(self): + return hash(self.contest_id) + + @cproperty + def when(self) -> datetime: + return self.cmap[self.uid].when + + @cproperty + def contest(self) -> str: + return self.json['contestName'] + + @cproperty + def summary(self) -> str: + return f'participated in {self.contest}' # TODO + + @classmethod + def make(cls, cmap, json) -> Iterator[Res['Competition']]: + yield cls(cmap=cmap, json=json) + yield from ytry(lambda: akeq(json, 'contestId', 'contestName', 'rank', 'oldRating', 'newRating')) + + +def iter_data() -> Iterator[Res[Competition]]: + cmap = get_contests() + + j = get_latest() + dell(j, 'status') + + j = zoom(j, 'result') + + for c in j: + dell(c, 'handle', 'ratingUpdateTimeSeconds') + yield from Competition.make(cmap=cmap, json=c) + + +def get_data(): + return list(sorted(iter_data(), key=fget(Competition.when))) + +def main(): + for d in iter_data(): + try: + d = unwrap(d) + except Exception as e: + print(f'ERROR! {d}') + else: + print(f'{d.when}: {d.summary}') + + + +if __name__ == '__main__': + main() diff --git a/coding/topcoder.py b/coding/topcoder.py index 8bea8b5..efe16b2 100644 --- a/coding/topcoder.py +++ b/coding/topcoder.py @@ -20,10 +20,14 @@ class Competition(NamedTuple): @cproperty def uid(self) -> str: - return self.contest + return self.contest_id + + @property + def contest_id(self) -> str: + return self.json['challengeId'] def __hash__(self): - return hash(self.json['challengeId']) + return hash(self.contest_id) @cproperty def contest(self) -> str: @@ -80,9 +84,10 @@ def main(): for d in iter_data(): try: d = unwrap(d) - print(d.summary) except Exception as e: print(f'ERROR! {d}') + else: + print(d.summary) if __name__ == '__main__': From 07736f7e1e4c69ab3c7810dee04e12cfbdef9c2f Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 4 May 2019 00:16:22 +0100 Subject: [PATCH 4/6] reuse konsume in codeforces --- coding/codeforces.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/coding/codeforces.py b/coding/codeforces.py index 7ab96f0..3e1dd82 100644 --- a/coding/codeforces.py +++ b/coding/codeforces.py @@ -6,7 +6,7 @@ import json from typing import Dict, Iterator, Any from kython import cproperty, fget -from kython.konsume import dell, zoom, keq, akeq +from kython.konsume import zoom, ignore from kython.kerror import Res, ytry, unwrap from kython.kdatetime import as_utc @@ -45,17 +45,14 @@ def get_latest(): class Competition(NamedTuple): - json: Dict[str, Any] + contest_id: str + contest: str cmap: Cmap @cproperty def uid(self) -> str: return self.contest_id - @property - def contest_id(self): - return self.json['contestId'] - def __hash__(self): return hash(self.contest_id) @@ -63,31 +60,36 @@ class Competition(NamedTuple): def when(self) -> datetime: return self.cmap[self.uid].when - @cproperty - def contest(self) -> str: - return self.json['contestName'] - @cproperty def summary(self) -> str: return f'participated in {self.contest}' # TODO @classmethod def make(cls, cmap, json) -> Iterator[Res['Competition']]: - yield cls(cmap=cmap, json=json) - yield from ytry(lambda: akeq(json, 'contestId', 'contestName', 'rank', 'oldRating', 'newRating')) - + # TODO try here?? + contest_id = json['contestId'].zoom().value + contest = json['contestName'].zoom().value + yield cls( + contest_id=contest_id, + contest=contest, + cmap=cmap, + ) + # TODO ytry??? + ignore(json, 'rank', 'oldRating', 'newRating') +from kython.konsume import wrap def iter_data() -> Iterator[Res[Competition]]: cmap = get_contests() - j = get_latest() - dell(j, 'status') + with wrap(get_latest()) as j: + j['status'].ignore() + res = j['result'].zoom() - j = zoom(j, 'result') - - for c in j: - dell(c, 'handle', 'ratingUpdateTimeSeconds') - yield from Competition.make(cmap=cmap, json=c) + for c in list(res): # TODO maybe we want 'iter' method?? + ignore(c, 'handle', 'ratingUpdateTimeSeconds') + yield from Competition.make(cmap=cmap, json=c) + c.consume() + # TODO maybe if they are all empty, no need to consume?? def get_data(): From 2b45b2db1dbf464ce7d38d05057bfd174b9de736 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 4 May 2019 00:29:22 +0100 Subject: [PATCH 5/6] migrate to new konsume --- coding/topcoder.py | 63 +++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/coding/topcoder.py b/coding/topcoder.py index efe16b2..4162429 100644 --- a/coding/topcoder.py +++ b/coding/topcoder.py @@ -6,7 +6,7 @@ import json from typing import Dict, Iterator, Any from kython import cproperty, fget -from kython.konsume import dell, zoom, keq, akeq +from kython.konsume import zoom, wrap, ignore from kython.kerror import Res, ytry, unwrap @@ -16,31 +16,21 @@ def get_latest(): class Competition(NamedTuple): - json: Dict[str, Any] + contest_id: str + contest: str + percentile: float + dates: str @cproperty def uid(self) -> str: return self.contest_id - @property - def contest_id(self) -> str: - return self.json['challengeId'] - def __hash__(self): return hash(self.contest_id) - @cproperty - def contest(self) -> str: - return self.json['challengeName'] - @cproperty def when(self) -> datetime: - ds = self.json['date'] - return datetime.strptime(ds, '%Y-%m-%dT%H:%M:%S.%fZ') - - @cproperty - def percentile(self) -> float: - return self.json['percentile'] + return datetime.strptime(self.dates, '%Y-%m-%dT%H:%M:%S.%fZ') @cproperty def summary(self) -> str: @@ -48,32 +38,41 @@ class Competition(NamedTuple): @classmethod def make(cls, json) -> Iterator[Res['Competition']]: - yield cls(json=json) - yield from ytry(lambda: akeq(json, 'challengeId', 'challengeName', 'percentile', 'rating', 'placement', 'date')) + ignore(json, 'rating', 'placement') + cid = json['challengeId'].zoom().value + cname = json['challengeName'].zoom().value + percentile = json['percentile'].zoom().value + dates = json['date'].zoom().value + yield cls( + contest_id=cid, + contest=cname, + percentile=percentile, + dates=dates, + ) def iter_data() -> Iterator[Res[Competition]]: - j = get_latest() - dell(j, 'id', 'version') + with wrap(get_latest()) as j: + ignore(j, 'id', 'version') - j = zoom(j, 'result') - dell(j, 'success', 'status', 'metadata') + res = j['result'].zoom() + ignore(res, 'success', 'status', 'metadata') - j = zoom(j, 'content') + cont = res['content'].zoom() + ignore(cont, 'handle', 'handleLower', 'userId', 'createdAt', 'updatedAt', 'createdBy', 'updatedBy') - dell(j, 'handle', 'handleLower', 'userId', 'createdAt', 'updatedAt', 'createdBy', 'updatedBy') + cont['DEVELOP'].ignore() # TODO FIXME handle it?? + ds = cont['DATA_SCIENCE'].zoom() - dell(j, 'DEVELOP') # TODO handle it?? - j = zoom(j, 'DATA_SCIENCE') + mar, srm = zoom(ds, 'MARATHON_MATCH', 'SRM') - mar, srm = zoom(j, 'MARATHON_MATCH', 'SRM') - - mar = zoom(mar, 'history') - srm = zoom(srm, 'history') + mar = mar['history'].zoom() + srm = srm['history'].zoom() # TODO right, I guess I could rely on pylint for unused variables?? - for c in mar + srm: - yield from Competition.make(json=c) + for c in mar + srm: + yield from Competition.make(json=c) + c.consume() def get_data(): From 859ff19a85266faadc54e5649cc296c938a83974 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 8 May 2019 21:19:51 +0100 Subject: [PATCH 6/6] fix various paths --- coding/codeforces.py | 7 ++++++- coding/topcoder.py | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/coding/codeforces.py b/coding/codeforces.py index 3e1dd82..c6b9378 100644 --- a/coding/codeforces.py +++ b/coding/codeforces.py @@ -11,7 +11,7 @@ from kython.kerror import Res, ytry, unwrap from kython.kdatetime import as_utc -_BDIR = Path('/L/Dropbox/data/codeforces') +_BDIR = Path('/L/zzz_syncthing/data/codeforces') Cid = int @@ -95,6 +95,11 @@ def iter_data() -> Iterator[Res[Competition]]: def get_data(): return list(sorted(iter_data(), key=fget(Competition.when))) + +def test(): + assert len(get_data()) > 10 + + def main(): for d in iter_data(): try: diff --git a/coding/topcoder.py b/coding/topcoder.py index 4162429..d41c2ae 100644 --- a/coding/topcoder.py +++ b/coding/topcoder.py @@ -11,7 +11,7 @@ from kython.kerror import Res, ytry, unwrap def get_latest(): - last = max(Path('/L/Dropbox/data/topcoder').glob('*.json')) + last = max(Path('/L/zzz_syncthing/data/topcoder').glob('*.json')) return json.loads(last.read_text()) @@ -79,6 +79,9 @@ def get_data(): return list(sorted(iter_data(), key=fget(Competition.when))) +def test(): + assert len(get_data()) > 10 + def main(): for d in iter_data(): try: