fix zstd handling and github wrapper
This commit is contained in:
parent
77d557e172
commit
3912ef2460
3 changed files with 15 additions and 14 deletions
|
@ -4,10 +4,9 @@ Github events and their metadata: comments/issues/pull requests
|
||||||
|
|
||||||
from .. import init
|
from .. import init
|
||||||
|
|
||||||
from typing import Dict, List, Union, Any, NamedTuple, Tuple, Optional, Iterator, TypeVar, Set
|
from typing import Dict, Any, NamedTuple, Tuple, Optional, Iterator, TypeVar, Set
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytz
|
import pytz
|
||||||
|
|
||||||
|
@ -95,7 +94,7 @@ def _get_summary(e) -> Tuple[str, Optional[str], Optional[str]]:
|
||||||
|
|
||||||
|
|
||||||
def inputs():
|
def inputs():
|
||||||
return get_files(config.export_dir, glob='*.json*')
|
return get_files(config.export_dir)
|
||||||
|
|
||||||
|
|
||||||
def _dal():
|
def _dal():
|
||||||
|
@ -198,7 +197,8 @@ def iter_gdpr_events() -> Iterator[Res[Event]]:
|
||||||
"""
|
"""
|
||||||
Parses events from GDPR export (https://github.com/settings/admin)
|
Parses events from GDPR export (https://github.com/settings/admin)
|
||||||
"""
|
"""
|
||||||
files = list(sorted(config.gdpr_dir.glob('*.json')))
|
# TODO allow using archive here?
|
||||||
|
files = get_files(config.gdpr_dir, glob='*.json')
|
||||||
handler_map = {
|
handler_map = {
|
||||||
'schema' : None,
|
'schema' : None,
|
||||||
'issue_events_': None, # eh, doesn't seem to have any useful bodies
|
'issue_events_': None, # eh, doesn't seem to have any useful bodies
|
||||||
|
@ -264,10 +264,3 @@ def get_events():
|
||||||
# from github.Event import Event as GEvent # type: ignore
|
# from github.Event import Event as GEvent # type: ignore
|
||||||
# # see https://github.com/PyGithub/PyGithub/blob/master/github/GithubObject.py::GithubObject.__init__
|
# # see https://github.com/PyGithub/PyGithub/blob/master/github/GithubObject.py::GithubObject.__init__
|
||||||
# e = GEvent(None, None, raw_event, True)
|
# e = GEvent(None, None, raw_event, True)
|
||||||
|
|
||||||
|
|
||||||
def test():
|
|
||||||
events = get_events()
|
|
||||||
assert len(events) > 100
|
|
||||||
for e in events:
|
|
||||||
print(e)
|
|
||||||
|
|
|
@ -9,12 +9,12 @@ import io
|
||||||
PathIsh = Union[Path, str]
|
PathIsh = Union[Path, str]
|
||||||
|
|
||||||
|
|
||||||
def _zstd_open(path: Path, *args, **kwargs):
|
def _zstd_open(path: Path, *args, **kwargs) -> IO[str]:
|
||||||
import zstandard as zstd # type: ignore
|
import zstandard as zstd # type: ignore
|
||||||
fh = path.open(*args, **kwargs)
|
fh = path.open('rb')
|
||||||
dctx = zstd.ZstdDecompressor()
|
dctx = zstd.ZstdDecompressor()
|
||||||
reader = dctx.stream_reader(fh)
|
reader = dctx.stream_reader(fh)
|
||||||
return reader
|
return io.TextIOWrapper(reader, **kwargs) # meh
|
||||||
|
|
||||||
|
|
||||||
# TODO returns protocol that we can call 'read' against?
|
# TODO returns protocol that we can call 'read' against?
|
||||||
|
|
8
tests/github.py
Normal file
8
tests/github.py
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from my.coding.github import get_events
|
||||||
|
|
||||||
|
def test():
|
||||||
|
events = get_events()
|
||||||
|
assert len(events) > 100
|
||||||
|
for e in events:
|
||||||
|
print(e)
|
Loading…
Add table
Reference in a new issue