From 571cb48aea2dcdd5c6641d535812711c126dfc7e Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 11 Dec 2020 03:40:31 +0000 Subject: [PATCH] core: add modules_ast for more robust module collection --- my/body/exercise/cross_trainer.py | 2 +- my/core/util.py | 38 +++++++++++++++++++++++++++++++ my/github/common.py | 5 ++-- my/roamresearch.py | 2 +- my/stackexchange/stexport.py | 2 +- my/time/tz/main.py | 2 +- my/time/tz/via_location.py | 2 +- my/twitter/twint.py | 4 +--- my/vk/favorites.py | 5 +--- my/vk/vk_messages_backup.py | 2 +- 10 files changed, 48 insertions(+), 16 deletions(-) diff --git a/my/body/exercise/cross_trainer.py b/my/body/exercise/cross_trainer.py index fd92666..58c32b2 100644 --- a/my/body/exercise/cross_trainer.py +++ b/my/body/exercise/cross_trainer.py @@ -1,5 +1,5 @@ ''' -My cross trainer exercise data, arbitrated between differen sources (mainly, Endomondo and various manual plaintext notes) +My cross trainer exercise data, arbitrated from different sources (mainly, Endomondo and manual text notes) This is probably too specific to my needs, so later I will move it away to a personal 'layer'. For now it's worth keeping it here as an example and perhaps utility functions might be useful for other HPI modules. diff --git a/my/core/util.py b/my/core/util.py index f9a5e72..1277e58 100644 --- a/my/core/util.py +++ b/my/core/util.py @@ -11,6 +11,8 @@ from typing import List, Iterable, NamedTuple, Optional class HPIModule(NamedTuple): name: str skip_reason: Optional[str] + doc: Optional[str] = None + file: Optional[Path] = None def modules() -> Iterable[HPIModule]: @@ -167,6 +169,42 @@ def _walk_packages(path=None, prefix='', onerror=None) -> Iterable[HPIModule]: path = [p for p in path if not seen(p)] yield from _walk_packages(path, mname+'.', onerror) + +def modules_via_ast() -> Iterable[HPIModule]: + ''' + Experimental version, which isn't importing the modules, making it more robust and safe. + ''' + import ast + + my_root = Path(__file__).absolute().parent.parent + for f in sorted(my_root.rglob('*.py')): + if f.is_symlink(): + continue # meh + mp = f.relative_to(my_root.parent) + if mp.name == '__init__.py': + mp = mp.parent + m = str(mp.with_suffix('')).replace('/', '.') + if ignored(m): + continue + a = ast.parse(f.read_text()) + NM = '__NOT_HPI_MODULE__' + is_not_module = any( + getattr(node, 'name', None) == NM # direct definition + or + any(getattr(n, 'name', None) == NM for n in getattr(node, 'names', [])) # import from + for node in a.body) + if is_not_module: + continue + doc = ast.get_docstring(a, clean=False) + + yield HPIModule( + name=m, + skip_reason=None, + doc=doc, + file=f.relative_to(my_root.parent), # todo not sure if should be relative + ) + + # deprecate? def get_modules() -> List[HPIModule]: return list(modules()) diff --git a/my/github/common.py b/my/github/common.py index e37bd83..737c652 100644 --- a/my/github/common.py +++ b/my/github/common.py @@ -47,6 +47,5 @@ def parse_dt(s: str) -> datetime: # TODO isoformat? return pytz.utc.localize(datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ')) -# TODO not sure -# def get_events() -> Iterable[Res[Event]]: -# return sort_res_by(events(), key=lambda e: e.dt) + +from ..core import __NOT_HPI_MODULE__ diff --git a/my/roamresearch.py b/my/roamresearch.py index e9be8f7..20a4391 100644 --- a/my/roamresearch.py +++ b/my/roamresearch.py @@ -9,7 +9,7 @@ from typing import NamedTuple, Iterator, List, Optional import pytz -from .common import get_files, LazyLogger, Json +from .core import get_files, LazyLogger, Json from my.config import roamresearch as config diff --git a/my/stackexchange/stexport.py b/my/stackexchange/stexport.py index 8dfee73..6286c83 100644 --- a/my/stackexchange/stexport.py +++ b/my/stackexchange/stexport.py @@ -1,5 +1,5 @@ ''' -Stackexchange data +Stackexchange data (uses API via [[https://github.com/karlicoss/stexport][stexport]]) ''' REQUIRES = [ 'git+https://github.com/karlicoss/stexport', diff --git a/my/time/tz/main.py b/my/time/tz/main.py index 50ebe05..624d7aa 100644 --- a/my/time/tz/main.py +++ b/my/time/tz/main.py @@ -1,5 +1,5 @@ ''' -Timezone data provider +Timezone data provider, used to localize timezone-unaware timestamps for other modules ''' from datetime import datetime from ...core.common import tzdatetime diff --git a/my/time/tz/via_location.py b/my/time/tz/via_location.py index 4b04d9f..ae4f4eb 100644 --- a/my/time/tz/via_location.py +++ b/my/time/tz/via_location.py @@ -1,5 +1,5 @@ ''' -Timezone data provider, useful for localizing UTC-only/timezone unaware dates. +Timezone data provider, guesses timezone based on location data (e.g. GPS) ''' REQUIRES = [ # for determining timezone by coordinate diff --git a/my/twitter/twint.py b/my/twitter/twint.py index 53240eb..77a6859 100644 --- a/my/twitter/twint.py +++ b/my/twitter/twint.py @@ -1,7 +1,5 @@ """ -Twitter data (tweets and favorites). - -Uses [[https://github.com/twintproject/twint][Twint]] data export. +Twitter data (tweets and favorites). Uses [[https://github.com/twintproject/twint][Twint]] data export. Requirements: =pip3 install --user dataset= """ diff --git a/my/vk/favorites.py b/my/vk/favorites.py index ad64e03..e6ccbf3 100644 --- a/my/vk/favorites.py +++ b/my/vk/favorites.py @@ -1,7 +1,4 @@ -''' -todo: uses my private export script? -''' - +# todo: uses my private export script? from datetime import datetime import json from typing import NamedTuple, Iterable, Sequence, Optional diff --git a/my/vk/vk_messages_backup.py b/my/vk/vk_messages_backup.py index 63f1d8e..b933757 100644 --- a/my/vk/vk_messages_backup.py +++ b/my/vk/vk_messages_backup.py @@ -1,5 +1,5 @@ ''' -Handles VK data exported by https://github.com/Totktonada/vk_messages_backup +VK data (exported by [[https://github.com/Totktonada/vk_messages_backup][Totktonada/vk_messages_backup]]) ''' # note: could reuse the original repo, but little point I guess since VK closed their API