use my.config instead of mycfg; minor cleanups and docstrings

This commit is contained in:
Dima Gerasimov 2020-04-12 00:18:48 +01:00
parent f31ff71e6f
commit 9fe5c8e670
37 changed files with 142 additions and 110 deletions

View file

@ -1,4 +1,8 @@
#!/usr/bin/env python3
from .. import init
from my.config import codeforces as config
from datetime import datetime
from typing import NamedTuple
from pathlib import Path
@ -31,8 +35,7 @@ Cmap = Dict[Cid, Contest]
def get_contests() -> Cmap:
from mycfg import paths
last = max(get_files(paths.codeforces.export_path, 'allcontests*.json'))
last = max(get_files(config.export_path, 'allcontests*.json'))
j = json.loads(last.read_text())
d = {}
for c in j['result']:
@ -77,8 +80,7 @@ class Competition(NamedTuple):
def iter_data() -> Iterator[Res[Competition]]:
cmap = get_contests()
from mycfg import paths
last = max(get_files(paths.codeforces.export_path, 'codeforces*.json'))
last = max(get_files(config.export_path, 'codeforces*.json'))
with wrap(json.loads(last.read_text())) as j:
j['status'].ignore()

View file

@ -7,7 +7,7 @@ from datetime import datetime, timezone
from typing import List, NamedTuple, Optional, Dict, Any, Iterator, Set
from ..common import PathIsh, LazyLogger, mcachew
from mycfg import commits as config
from my.config import commits as config
# pip3 install gitpython
import git # type: ignore

View file

@ -2,6 +2,8 @@
Github events and their metadata: comments/issues/pull requests
"""
from .. import init
from typing import Dict, List, Union, Any, NamedTuple, Tuple, Optional, Iterator, TypeVar, Set
from datetime import datetime
import json
@ -14,8 +16,8 @@ from ..kython.kompress import CPath
from ..common import get_files, mcachew
from ..error import Res
from mycfg import paths
import mycfg.repos.ghexport.dal as ghexport
from my.config import github as config
import my.config.repos.ghexport.dal as ghexport
logger = LazyLogger('my.github')
@ -79,7 +81,7 @@ def _get_summary(e) -> Tuple[str, Optional[str], Optional[str]]:
def get_dal():
sources = get_files(paths.github.export_dir, glob='*.json*')
sources = get_files(config.export_dir, glob='*.json*')
sources = list(map(CPath, sources)) # TODO maybe move it to get_files? e.g. compressed=True arg?
return ghexport.DAL(sources)
@ -178,7 +180,7 @@ def iter_gdpr_events() -> Iterator[Res[Event]]:
"""
Parses events from GDPR export (https://github.com/settings/admin)
"""
files = list(sorted(paths.github.gdpr_dir.glob('*.json')))
files = list(sorted(config.gdpr_dir.glob('*.json')))
handler_map = {
'schema' : None,
'issue_events_': None, # eh, doesn't seem to have any useful bodies
@ -215,7 +217,7 @@ def iter_gdpr_events() -> Iterator[Res[Event]]:
# TODO hmm. not good, need to be lazier?...
@mcachew(paths.github.cache_dir, hashf=lambda dal: dal.sources)
@mcachew(config.cache_dir, hashf=lambda dal: dal.sources)
def iter_backup_events(dal=get_dal()) -> Iterator[Event]:
for d in dal.events():
yield _parse_event(d)

View file

@ -1,4 +1,8 @@
#!/usr/bin/env python3
from .. import init
from my.config import topcoder as config
from datetime import datetime
from typing import NamedTuple
from pathlib import Path
@ -15,8 +19,7 @@ from kython.konsume import zoom, wrap, ignore
# TODO json type??
def _get_latest() -> Dict:
from mycfg import paths
pp = max(get_files(paths.topcoder.export_path, glob='*.json'))
pp = max(get_files(config.export_path, glob='*.json'))
return json.loads(pp.read_text())