diff --git a/my/github/common.py b/my/github/common.py index 737c652..6114045 100644 --- a/my/github/common.py +++ b/my/github/common.py @@ -1,6 +1,9 @@ """ Github events and their metadata: comments/issues/pull requests """ +from ..core import __NOT_HPI_MODULE__ + + from datetime import datetime from typing import Optional, NamedTuple, Iterable, Set, Tuple @@ -48,4 +51,12 @@ def parse_dt(s: str) -> datetime: return pytz.utc.localize(datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ')) -from ..core import __NOT_HPI_MODULE__ +# experimental way of supportint event ids... not sure +class EventIds: + @staticmethod + def repo_created(*, dts: str, name: str, ref_type: str, ref: Optional[str]) -> str: + return f'{dts}_repocreated_{name}_{ref_type}_{ref}' + + @staticmethod + def pr(*, dts: str, action: str, url: str) -> str: + return f'{dts}_pr{action}_{url}' diff --git a/my/github/gdpr.py b/my/github/gdpr.py index 8d4fbf8..fe8a64b 100644 --- a/my/github/gdpr.py +++ b/my/github/gdpr.py @@ -8,7 +8,7 @@ from typing import Iterable, Dict, Any from ..core.error import Res from ..core import get_files -from .common import Event, parse_dt +from .common import Event, parse_dt, EventIds # TODO later, use a separate user config? (github_gdpr) from my.config import github as user_config @@ -87,11 +87,14 @@ def _parse_common(d: Dict) -> Dict: def _parse_repository(d: Dict) -> Event: pref = 'https://github.com/' url = d['url'] + dts = d['created_at'] + rt = d['type'] assert url.startswith(pref); name = url[len(pref):] + eid = EventIds.repo_created(dts=dts, name=name, ref_type=rt, ref=None) return Event( # type: ignore[misc] **_parse_common(d), summary='created ' + name, - eid='created_' + name, # TODO ?? + eid=eid, ) @@ -119,6 +122,7 @@ def _parse_issue(d: Dict) -> Event: def _parse_pull_request(d: Dict) -> Event: + dts = d['created_at'] url = d['url'] title = d['title'] is_bot = "[bot]" in d["user"] @@ -127,7 +131,7 @@ def _parse_pull_request(d: Dict) -> Event: # TODO distinguish incoming/outgoing? # TODO action? opened/closed?? summary=f'opened PR {title}', - eid='pull_request_' + url, + eid=EventIds.pr(dts=dts, action='opened', url=url), is_bot=is_bot, ) @@ -146,6 +150,7 @@ def _parse_project(d: Dict) -> Event: is_bot=is_bot, ) + def _parse_release(d: Dict) -> Event: tag = d['tag_name'] return Event( # type: ignore[misc] diff --git a/my/github/ghexport.py b/my/github/ghexport.py index cfa7b99..c9ba7ea 100644 --- a/my/github/ghexport.py +++ b/my/github/ghexport.py @@ -46,7 +46,7 @@ from typing import Tuple, Dict, Sequence, Optional from my.core import get_files, Path, LazyLogger from my.core.common import mcachew -from .common import Event, parse_dt, Results +from .common import Event, parse_dt, Results, EventIds logger = LazyLogger(__name__) @@ -61,9 +61,16 @@ def _dal() -> dal.DAL: return dal.DAL(sources) -# todo cachew: hmm. not good, need to be lazier?... @mcachew(depends_on=lambda: inputs()) def events() -> Results: + from my.core.common import ensure_unique + key = lambda e: object() if isinstance(e, Exception) else e.eid + # crap. sometimes API events can be repeated with exactly the same payload and different id + # yield from ensure_unique(_events(), key=key) + yield from _events() + + +def _events() -> Results: dal = _dal() for d in dal.events(): if isinstance(d, Exception): @@ -93,6 +100,7 @@ EventId = str Body = str def _get_summary(e) -> Tuple[str, Optional[Link], Optional[EventId], Optional[Body]]: # TODO would be nice to give access to raw event within timeline + dts = e['created_at'] eid = e['id'] tp = e['type'] pl = e['payload'] @@ -119,10 +127,13 @@ def _get_summary(e) -> Tuple[str, Optional[Link], Optional[EventId], Optional[Bo what = mapping[tp] rt = pl['ref_type'] ref = pl['ref'] - # TODO link to branch? only contains weird API link though - # TODO hmm. include timestamp instead? + if what == 'created': + # FIXME should handle delection?... + eid = EventIds.repo_created(dts=dts, name=rname, ref_type=rt, ref=ref) + mref = '' if ref is None else ' ' + ref + # todo link to branch? only contains weird API link though # TODO combine automatically instead - return f"{rname}: {what} {rt} {ref}", None, f'{rname}_{what}_{rt}_{ref}_{eid}', None + return f"{rname}: {what} {rt}{mref}", None, eid, None elif tp == 'PullRequestEvent': pr = pl['pull_request'] title = pr['title'] @@ -130,7 +141,8 @@ def _get_summary(e) -> Tuple[str, Optional[Link], Optional[EventId], Optional[Bo link = pr['html_url'] body = pr['body'] action = pl['action'] - return f"{rname}: {action} PR: {title}", link, f'{rname}_{action}_pr_{link}', body + eid = EventIds.pr(dts=dts, action=action, url=link) + return f"{rname}: {action} PR: {title}", link, eid, body elif tp == 'PullRequestReviewEvent': pr = pl['pull_request'] title = pr['title']