diff --git a/.ci/run b/.ci/run index 7656575..fe2719e 100755 --- a/.ci/run +++ b/.ci/run @@ -11,8 +11,6 @@ if ! command -v sudo; then } fi -# --parallel-live to show outputs while it's running -tox_cmd='run-parallel --parallel-live' if [ -n "${CI-}" ]; then # install OS specific stuff here case "$OSTYPE" in @@ -22,8 +20,7 @@ if [ -n "${CI-}" ]; then ;; cygwin* | msys* | win*) # windows - # ugh. parallel stuff seems super flaky under windows, some random failures, "file used by other process" and crap like that - tox_cmd='run' + : ;; *) # must be linux? @@ -40,9 +37,5 @@ if ! command -v python3 &> /dev/null; then PY_BIN="python" fi - -# TODO hmm for some reason installing uv with pip and then running -# "$PY_BIN" -m uv tool fails with missing setuptools error?? -# just uvx directly works, but it's not present in PATH... -"$PY_BIN" -m pip install --user pipx -"$PY_BIN" -m pipx run uv tool run --with=tox-uv tox $tox_cmd "$@" +"$PY_BIN" -m pip install --user tox +"$PY_BIN" -m tox --parallel --parallel-live "$@" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 111d0e9..f49c6b5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,20 +21,19 @@ on: jobs: build: strategy: - fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] exclude: [ # windows runners are pretty scarce, so let's only run lowest and highest python version + {platform: windows-latest, python-version: '3.9' }, {platform: windows-latest, python-version: '3.10'}, {platform: windows-latest, python-version: '3.11'}, - {platform: windows-latest, python-version: '3.12'}, # same, macos is a bit too slow and ubuntu covers python quirks well + {platform: macos-latest , python-version: '3.9' }, {platform: macos-latest , python-version: '3.10' }, {platform: macos-latest , python-version: '3.11' }, - {platform: macos-latest , python-version: '3.12' }, ] runs-on: ${{ matrix.platform }} @@ -46,11 +45,11 @@ jobs: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 with: submodules: recursive fetch-depth: 0 # nicer to have all git history when debugging/for tests @@ -62,15 +61,13 @@ jobs: - run: bash .ci/run - if: matrix.platform == 'ubuntu-latest' # no need to compute coverage for other platforms - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: - include-hidden-files: true name: .coverage.mypy-misc_${{ matrix.platform }}_${{ matrix.python-version }} path: .coverage.mypy-misc/ - if: matrix.platform == 'ubuntu-latest' # no need to compute coverage for other platforms - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: - include-hidden-files: true name: .coverage.mypy-core_${{ matrix.platform }}_${{ matrix.python-version }} path: .coverage.mypy-core/ @@ -82,11 +79,11 @@ jobs: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.8' - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 with: submodules: recursive diff --git a/.gitignore b/.gitignore index 65ba630..19c3380 100644 --- a/.gitignore +++ b/.gitignore @@ -155,9 +155,6 @@ celerybeat-schedule .dmypy.json dmypy.json -# linters -.ruff_cache/ - # Pyre type checker .pyre/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d60ef35..3dd19df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ General/my.core changes: - e81dddddf083ffd81aa7e2b715bd34f59949479c properly resolve class properties in make_config + add test Modules: -- some initial work on filling **InfluxDB** with HPI data +- some innitial work on filling **InfluxDB** with HPI data - pinboard - 42399f6250d9901d93dcedcfe05f7857babcf834: **breaking backwards compatibility**, use pinbexport module directly diff --git a/README.org b/README.org index 79621a5..c065a0c 100644 --- a/README.org +++ b/README.org @@ -723,10 +723,10 @@ If you want to write modules for personal use but don't want to merge them into Other HPI Repositories: -- [[https://github.com/purarue/HPI][purarue/HPI]] +- [[https://github.com/seanbreckenridge/HPI][seanbreckenridge/HPI]] - [[https://github.com/madelinecameron/hpi][madelinecameron/HPI]] -If you want to create your own to create your own modules/override something here, you can use the [[https://github.com/purarue/HPI-template][template]]. +If you want to create your own to create your own modules/override something here, you can use the [[https://github.com/seanbreckenridge/HPI-template][template]]. * Related links :PROPERTIES: diff --git a/conftest.py b/conftest.py deleted file mode 100644 index b959cfa..0000000 --- a/conftest.py +++ /dev/null @@ -1,47 +0,0 @@ -# this is a hack to monkey patch pytest so it handles tests inside namespace packages without __init__.py properly -# without it, pytest can't discover the package root for some reason -# also see https://github.com/karlicoss/pytest_namespace_pkgs for more - -import os -import pathlib -from typing import Optional - -import _pytest.main -import _pytest.pathlib - -# we consider all dirs in repo/ to be namespace packages -root_dir = pathlib.Path(__file__).absolute().parent.resolve() # / 'src' -assert root_dir.exists(), root_dir - -# TODO assert it contains package name?? maybe get it via setuptools.. - -namespace_pkg_dirs = [str(d) for d in root_dir.iterdir() if d.is_dir()] - -# resolve_package_path is called from _pytest.pathlib.import_path -# takes a full abs path to the test file and needs to return the path to the 'root' package on the filesystem -resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path -def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]: - result = path # search from the test file upwards - for parent in result.parents: - if str(parent) in namespace_pkg_dirs: - return parent - if os.name == 'nt': - # ??? for some reason on windows it is trying to call this against conftest? but not on linux/osx - if path.name == 'conftest.py': - return resolve_pkg_path_orig(path) - raise RuntimeError("Couldn't determine path for ", path) -_pytest.pathlib.resolve_package_path = resolve_package_path - - -# without patching, the orig function returns just a package name for some reason -# (I think it's used as a sort of fallback) -# so we need to point it at the absolute path properly -# not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure.. -search_pypath_orig = _pytest.main.search_pypath -def search_pypath(module_name: str) -> str: - mpath = root_dir / module_name.replace('.', os.sep) - if not mpath.is_dir(): - mpath = mpath.with_suffix('.py') - assert mpath.exists(), mpath # just in case - return str(mpath) -_pytest.main.search_pypath = search_pypath diff --git a/doc/DENYLIST.md b/doc/DENYLIST.md index 3d8dea0..440715c 100644 --- a/doc/DENYLIST.md +++ b/doc/DENYLIST.md @@ -76,7 +76,7 @@ This would typically be used in an overridden `all.py` file, or in a one-off scr which you may want to filter out some items from a source, progressively adding more items to the denylist as you go. -A potential `my/ip/all.py` file might look like (Sidenote: `discord` module from [here](https://github.com/purarue/HPI)): +A potential `my/ip/all.py` file might look like (Sidenote: `discord` module from [here](https://github.com/seanbreckenridge/HPI)): ```python from typing import Iterator @@ -119,9 +119,9 @@ python3 -c 'from my.ip import all; all.deny.deny_cli(all.ips())' To edit the `all.py`, you could either: - install it as editable (`python3 -m pip install --user -e ./HPI`), and then edit the file directly -- or, create a namespace package, which splits the package across multiple directories. For info on that see [`MODULE_DESIGN`](https://github.com/karlicoss/HPI/blob/master/doc/MODULE_DESIGN.org#namespace-packages), [`reorder_editable`](https://github.com/purarue/reorder_editable), and possibly the [`HPI-template`](https://github.com/purarue/HPI-template) to create your own HPI namespace package to create your own `all.py` file. +- or, create a namespace package, which splits the package across multiple directories. For info on that see [`MODULE_DESIGN`](https://github.com/karlicoss/HPI/blob/master/doc/MODULE_DESIGN.org#namespace-packages), [`reorder_editable`](https://github.com/seanbreckenridge/reorder_editable), and possibly the [`HPI-template`](https://github.com/seanbreckenridge/HPI-template) to create your own HPI namespace package to create your own `all.py` file. -For a real example of this see, [purarue/HPI-personal](https://github.com/purarue/HPI-personal/blob/master/my/ip/all.py) +For a real example of this see, [seanbreckenridge/HPI-personal](https://github.com/seanbreckenridge/HPI-personal/blob/master/my/ip/all.py) Sidenote: the reason why we want to specifically override the all.py and not just create a script that filters out the items you're diff --git a/doc/MODULES.org b/doc/MODULES.org index 347d88d..9f48024 100644 --- a/doc/MODULES.org +++ b/doc/MODULES.org @@ -76,7 +76,7 @@ The config snippets below are meant to be modified accordingly and *pasted into You don't have to set up all modules at once, it's recommended to do it gradually, to get the feel of how HPI works. -For an extensive/complex example, you can check out ~@purarue~'s [[https://github.com/purarue/dotfiles/blob/master/.config/my/my/config/__init__.py][config]] +For an extensive/complex example, you can check out ~@seanbreckenridge~'s [[https://github.com/seanbreckenridge/dotfiles/blob/master/.config/my/my/config/__init__.py][config]] # Nested Configurations before the doc generation using the block below ** [[file:../my/reddit][my.reddit]] @@ -96,7 +96,7 @@ For an extensive/complex example, you can check out ~@purarue~'s [[https://githu class pushshift: ''' - Uses [[https://github.com/purarue/pushshift_comment_export][pushshift]] to get access to old comments + Uses [[https://github.com/seanbreckenridge/pushshift_comment_export][pushshift]] to get access to old comments ''' # path[s]/glob to the exported JSON data @@ -106,7 +106,7 @@ For an extensive/complex example, you can check out ~@purarue~'s [[https://githu ** [[file:../my/browser/][my.browser]] - Parses browser history using [[http://github.com/purarue/browserexport][browserexport]] + Parses browser history using [[http://github.com/seanbreckenridge/browserexport][browserexport]] #+begin_src python class browser: @@ -132,7 +132,7 @@ For an extensive/complex example, you can check out ~@purarue~'s [[https://githu You might also be able to use [[file:../my/location/via_ip.py][my.location.via_ip]] which uses =my.ip.all= to provide geolocation data for an IPs (though no IPs are provided from any - of the sources here). For an example of usage, see [[https://github.com/purarue/HPI/tree/master/my/ip][here]] + of the sources here). For an example of usage, see [[https://github.com/seanbreckenridge/HPI/tree/master/my/ip][here]] #+begin_src python class location: @@ -256,9 +256,9 @@ for cls, p in modules: ** [[file:../my/google/takeout/parser.py][my.google.takeout.parser]] - Parses Google Takeout using [[https://github.com/purarue/google_takeout_parser][google_takeout_parser]] + Parses Google Takeout using [[https://github.com/seanbreckenridge/google_takeout_parser][google_takeout_parser]] - See [[https://github.com/purarue/google_takeout_parser][google_takeout_parser]] for more information about how to export and organize your takeouts + See [[https://github.com/seanbreckenridge/google_takeout_parser][google_takeout_parser]] for more information about how to export and organize your takeouts If the =DISABLE_TAKEOUT_CACHE= environment variable is set, this won't cache individual exports in =~/.cache/google_takeout_parser= diff --git a/doc/MODULE_DESIGN.org b/doc/MODULE_DESIGN.org index 442dbf2..7aedf2f 100644 --- a/doc/MODULE_DESIGN.org +++ b/doc/MODULE_DESIGN.org @@ -67,7 +67,7 @@ If you want to disable a source, you have a few options. ... that suppresses the warning message and lets you use ~my.location.all~ without having to change any lines of code -Another benefit is that all the custom sources/data is localized to the ~all.py~ file, so a user can override the ~all.py~ (see the sections below on ~namespace packages~) file in their own HPI repository, adding additional sources without having to maintain a fork and patching in changes as things eventually change. For a 'real world' example of that, see [[https://github.com/purarue/HPI#partially-in-usewith-overrides][purarue]]s location and ip modules. +Another benefit is that all the custom sources/data is localized to the ~all.py~ file, so a user can override the ~all.py~ (see the sections below on ~namespace packages~) file in their own HPI repository, adding additional sources without having to maintain a fork and patching in changes as things eventually change. For a 'real world' example of that, see [[https://github.com/seanbreckenridge/HPI#partially-in-usewith-overrides][seanbreckenridge]]s location and ip modules. This is of course not required for personal or single file modules, its just the pattern that seems to have the least amount of friction for the user, while being extendable, and without using a bulky plugin system to let users add additional sources. @@ -208,13 +208,13 @@ Where ~lastfm.py~ is your version of ~my.lastfm~, which you've copied from this Then, running ~python3 -m pip install -e .~ in that directory would install that as part of the namespace package, and assuming (see below for possible issues) this appears on ~sys.path~ before the upstream repository, your ~lastfm.py~ file overrides the upstream. Adding more files, like ~my.some_new_module~ into that directory immediately updates the global ~my~ package -- allowing you to quickly add new modules without having to re-install. -If you install both directories as editable packages (which has the benefit of any changes you making in either repository immediately updating the globally installed ~my~ package), there are some concerns with which editable install appears on your ~sys.path~ first. If you wanted your modules to override the upstream modules, yours would have to appear on the ~sys.path~ first (this is the same reason that =custom_lastfm_overlay= must be at the front of your ~PYTHONPATH~). For more details and examples on dealing with editable namespace packages in the context of HPI, see the [[https://github.com/purarue/reorder_editable][reorder_editable]] repository. +If you install both directories as editable packages (which has the benefit of any changes you making in either repository immediately updating the globally installed ~my~ package), there are some concerns with which editable install appears on your ~sys.path~ first. If you wanted your modules to override the upstream modules, yours would have to appear on the ~sys.path~ first (this is the same reason that =custom_lastfm_overlay= must be at the front of your ~PYTHONPATH~). For more details and examples on dealing with editable namespace packages in the context of HPI, see the [[https://github.com/seanbreckenridge/reorder_editable][reorder_editable]] repository. There is no limit to how many directories you could install into a single namespace package, which could be a possible way for people to install additional HPI modules, without worrying about the module count here becoming too large to manage. -There are some other users [[https://github.com/hpi/hpi][who have begun publishing their own modules]] as namespace packages, which you could potentially install and use, in addition to this repository, if any of those interest you. If you want to create your own you can use the [[https://github.com/purarue/HPI-template][template]] to get started. +There are some other users [[https://github.com/hpi/hpi][who have begun publishing their own modules]] as namespace packages, which you could potentially install and use, in addition to this repository, if any of those interest you. If you want to create your own you can use the [[https://github.com/seanbreckenridge/HPI-template][template]] to get started. -Though, enabling this many modules may make ~hpi doctor~ look pretty busy. You can explicitly choose to enable/disable modules with a list of modules/regexes in your [[https://github.com/karlicoss/HPI/blob/f559e7cb899107538e6c6bbcf7576780604697ef/my/core/core_config.py#L24-L55][core config]], see [[https://github.com/purarue/dotfiles/blob/a1a77c581de31bd55a6af3d11b8af588614a207e/.config/my/my/config/__init__.py#L42-L72][here]] for an example. +Though, enabling this many modules may make ~hpi doctor~ look pretty busy. You can explicitly choose to enable/disable modules with a list of modules/regexes in your [[https://github.com/karlicoss/HPI/blob/f559e7cb899107538e6c6bbcf7576780604697ef/my/core/core_config.py#L24-L55][core config]], see [[https://github.com/seanbreckenridge/dotfiles/blob/a1a77c581de31bd55a6af3d11b8af588614a207e/.config/my/my/config/__init__.py#L42-L72][here]] for an example. You may use the other modules or [[https://github.com/karlicoss/hpi-personal-overlay][my overlay]] as reference, but python packaging is already a complicated issue, before adding complexities like namespace packages and editable installs on top of it... If you're having trouble extending HPI in this fashion, you can open an issue here, preferably with a link to your code/repository and/or ~setup.py~ you're trying to use. diff --git a/doc/OVERLAYS.org b/doc/OVERLAYS.org deleted file mode 100644 index 7bafa48..0000000 --- a/doc/OVERLAYS.org +++ /dev/null @@ -1,322 +0,0 @@ -NOTE this kinda overlaps with [[file:MODULE_DESIGN.org][the module design doc]], should be unified in the future. - -Relevant discussion about overlays: https://github.com/karlicoss/HPI/issues/102 - -# This is describing TODO -# TODO goals -# - overrides -# - proper mypy support -# - TODO reusing parent modules? - -# You can see them TODO in overlays dir - -Consider a toy package/module structure with minimal code, without any actual data parsing, just for demonstration purposes. - -- =main= package structure - # TODO do links - - - =my/twitter/gdpr.py= - Extracts Twitter data from GDPR archive. - - =my/twitter/all.py= - Merges twitter data from multiple sources (only =gdpr= in this case), so data consumers are agnostic of specific data sources used. - This will be overridden by =overlay=. - - =my/twitter/common.py= - Contains helper function to merge data, so they can be reused by overlay's =all.py=. - - =my/reddit.py= - Extracts Reddit data -- this won't be overridden by the overlay, we just keep it for demonstration purposes. - -- =overlay= package structure - - - =my/twitter/talon.py= - Extracts Twitter data from Talon android app. - - =my/twitter/all.py= - Override for =all.py= from =main= package -- it merges together data from =gpdr= and =talon= modules. - -# TODO mention resolution? reorder_editable - -* Installing (editable install) - -NOTE: this was tested with =python 3.10= and =pip 23.3.2=. - -To install, we run: - -: pip3 install --user -e overlay/ -: pip3 install --user -e main/ - -# TODO mention non-editable installs (this bit will still work with non-editable install) - -As a result, we get: - -: pip3 list | grep hpi -: hpi-main 0.0.0 /project/main/src -: hpi-overlay 0.0.0 /project/overlay/src - -: cat ~/.local/lib/python3.10/site-packages/easy-install.pth -: /project/overlay/src -: /project/main/src - -(the order above is important, so =overlay= takes precedence over =main= TODO link) - -Verify the setup: - -: $ python3 -c 'import my; print(my.__path__)' -: _NamespacePath(['/project/overlay/src/my', '/project/main/src/my']) - -This basically means that modules will be searched in both paths, with overlay taking precedence. - -** Installing with =--use-pep517= - -See here for discussion https://github.com/purarue/reorder_editable/issues/2, but TLDR it should work similarly. - -* Testing runtime behaviour (editable install) - -: $ python3 -c 'import my.reddit as R; print(R.upvotes())' -: [main] my.reddit hello -: ['reddit upvote1', 'reddit upvote2'] - -Just as expected here, =my.reddit= is imported from the =main= package, since it doesn't exist in =overlay=. - -Let's theck twitter now: - -: $ python3 -c 'import my.twitter.all as T; print(T.tweets())' -: [overlay] my.twitter.all hello -: [main] my.twitter.common hello -: [main] my.twitter.gdpr hello -: [overlay] my.twitter.talon hello -: ['gdpr tweet 1', 'gdpr tweet 2', 'talon tweet 1', 'talon tweet 2'] - -As expected, =my.twitter.all= was imported from the =overlay=. -As you can see it's merged data from =gdpr= (from =main= package) and =talon= (from =overlay= package). - -So far so good, let's see how it works with mypy. - -* Mypy support (editable install) - -To check that mypy works as expected I injected some statements in modules that have no impact on runtime, -but should trigger mypy, like this =trigger_mypy_error: str = 123=: - -Let's run it: - -: $ mypy --namespace-packages --strict -p my -: overlay/src/my/twitter/talon.py:9: error: Incompatible types in assignment (expression has type "int", variable has type "str") -: [assignment] -: trigger_mypy_error: str = 123 -: ^ -: Found 1 error in 1 file (checked 4 source files) - -Hmm, this did find the statement in the =overlay=, but missed everything from =main= (e.g. =reddit.py= and =gdpr.py= should have also triggered the check). - -First, let's check which sources mypy is processing: - -: $ mypy --namespace-packages --strict -p my -v 2>&1 | grep BuildSource -: LOG: Found source: BuildSource(path='/project/overlay/src/my', module='my', has_text=False, base_dir=None) -: LOG: Found source: BuildSource(path='/project/overlay/src/my/twitter', module='my.twitter', has_text=False, base_dir=None) -: LOG: Found source: BuildSource(path='/project/overlay/src/my/twitter/all.py', module='my.twitter.all', has_text=False, base_dir=None) -: LOG: Found source: BuildSource(path='/project/overlay/src/my/twitter/talon.py', module='my.twitter.talon', has_text=False, base_dir=None) - -So seems like mypy is not processing anything from =main= package at all? - -At this point I cloned mypy, put a breakpoint, and found out this is the culprit: https://github.com/python/mypy/blob/1dd8e7fe654991b01bd80ef7f1f675d9e3910c3a/mypy/modulefinder.py#L288 - -This basically returns the first path where it finds =my= package, which happens to be the overlay in this case. -So everything else is ignored? - -It even seems to have a test for a similar usecase, which is quite sad. -https://github.com/python/mypy/blob/1dd8e7fe654991b01bd80ef7f1f675d9e3910c3a/mypy/test/testmodulefinder.py#L64-L71 - -For now, I opened an issue in mypy repository https://github.com/python/mypy/issues/16683 - -But ok, maybe mypy treats =main= as an external package somehow but still type checks it properly? -Let's see what's going on with imports: - -: $ mypy --namespace-packages --strict -p my --follow-imports=error -: overlay/src/my/twitter/talon.py:9: error: Incompatible types in assignment (expression has type "int", variable has type "str") -: [assignment] -: trigger_mypy_error: str = 123 -: ^ -: overlay/src/my/twitter/all.py:3: error: Import of "my.twitter.common" ignored [misc] -: from .common import merge -: ^ -: overlay/src/my/twitter/all.py:6: error: Import of "my.twitter.gdpr" ignored [misc] -: from . import gdpr -: ^ -: overlay/src/my/twitter/all.py:6: note: (Using --follow-imports=error, module not passed on command line) -: overlay/src/my/twitter/all.py: note: In function "tweets": -: overlay/src/my/twitter/all.py:8: error: Returning Any from function declared to return "List[str]" [no-any-return] -: return merge(gdpr, talon) -: ^ -: Found 4 errors in 2 files (checked 4 source files) - -Nope -- looks like it's completely unawareof =main=, and what's worst, by default (without tweaking =--follow-imports=), these errors would be suppressed. - -What if we check =my.twitter= directly? - -: $ mypy --namespace-packages --strict -p my.twitter --follow-imports=error -: overlay/src/my/twitter/talon.py:9: error: Incompatible types in assignment (expression has type "int", variable has type "str") -: [assignment] -: trigger_mypy_error: str = 123 -: ^~~ -: overlay/src/my/twitter: error: Ancestor package "my" ignored [misc] -: overlay/src/my/twitter: note: (Using --follow-imports=error, submodule passed on command line) -: overlay/src/my/twitter/all.py:3: error: Import of "my.twitter.common" ignored [misc] -: from .common import merge -: ^ -: overlay/src/my/twitter/all.py:3: note: (Using --follow-imports=error, module not passed on command line) -: overlay/src/my/twitter/all.py:6: error: Import of "my.twitter.gdpr" ignored [misc] -: from . import gdpr -: ^ -: overlay/src/my/twitter/all.py: note: In function "tweets": -: overlay/src/my/twitter/all.py:8: error: Returning Any from function declared to return "list[str]" [no-any-return] -: return merge(gdpr, talon) -: ^~~~~~~~~~~~~~~~~~~~~~~~~ -: Found 5 errors in 3 files (checked 3 source files) - -Now we're also getting =error: Ancestor package "my" ignored [misc]= .. not ideal. - -* What if we don't install at all? -Instead of editable install let's try running mypy directly over source files - -First let's only check =main= package: - -: $ MYPYPATH=main/src mypy --namespace-packages --strict -p my -: main/src/my/twitter/gdpr.py:9: error: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] -: trigger_mypy_error: str = 123 -: ^~~ -: main/src/my/reddit.py:11: error: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] -: trigger_mypy_error: str = 123 -: ^~~ -: Found 2 errors in 2 files (checked 6 source files) - -As expected, it found both errors. - -Now with overlay as well: - -: $ MYPYPATH=overlay/src:main/src mypy --namespace-packages --strict -p my -: overlay/src/my/twitter/all.py:6: note: In module imported here: -: main/src/my/twitter/gdpr.py:9: error: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] -: trigger_mypy_error: str = 123 -: ^~~ -: overlay/src/my/twitter/talon.py:9: error: Incompatible types in assignment (expression has type "int", variable has type "str") -: [assignment] -: trigger_mypy_error: str = 123 -: ^~~ -: Found 2 errors in 2 files (checked 4 source files) - -Interesting enough, this is slightly better than the editable install (it detected error in =gdpr.py= as well). -But still no =reddit.py= error. - -TODO possibly worth submitting to mypy issue tracker as well... - -Overall it seems that properly type checking HPI setup as a whole is kinda problematic, especially if the modules actually override/extend base modules. - -* Modifying (monkey patching) original module in the overlay -Let's say we want to modify/monkey patch =my.twitter.talon= module from =main=, for example, convert "gdpr" to uppercase, i.e. =tweet.replace('gdpr', 'GDPR')=. - -# TODO see overlay2/ - -I think our options are: - -- symlink to the 'parent' packages, e.g. =main= in the case - - Alternatively, somehow install =main= under a different name/alias (managed by pip). - - This is discussed here: https://github.com/karlicoss/HPI/issues/102 - - The main upside is that it's relatively simple and (sort of works with mypy). - - There are a few big downsides: - - creates a parallel package hierarchy (to the one maintained by pip), symlinks will need to be carefully managed manually - - This may not be such a huge deal if you don't have too many overlays. - However this results in problems if you're trying to switch between two different HPI checkouts (e.g. stable and development). If you have symlinks into "stable" from the overlay then stable modules will sometimes be picked up when you're expecting "development" package. - - - symlinks pointing outside of the source tree might cause pip install to go into infinite loop - - - it modifies the package name - - This may potentially result in some confusing behaviours. - - One thing I noticed for example is that cachew caches might get duplicated. - - - it might not work in all cases or might result in recursive imports - -- do not shadow the original module - - Basically instead of shadowing via namespace package mechanism and creating identically named module, - create some sort of hook that would patch the original =my.twitter.talon= module from =main=. - - The downside is that it's a bit unclear where to do that, we need some sort of entry point? - - - it could be some global dynamic hook defined in the overlay, and then executed from =my.core= - - However, it's a bit intrusive, and unclear how to handle errors. E.g. what if we're monkey patching a module that we weren't intending to use, don't have dependencies installed and it's crashing? - - Perhaps core could support something like =_hook= in each of HPI's modules? - Note that it can't be =my.twitter.all=, since we might want to override =.all= itself. - - The downside is is this probably not going to work well with =tmp_config= and such -- we'll need to somehow execute the hook again on reloading the module? - - - ideally we'd have something that integrates with =importlib= and executed automatically when module is imported? - - TODO explore these: - - - https://stackoverflow.com/questions/43571737/how-to-implement-an-import-hook-that-can-modify-the-source-code-on-the-fly-using - - https://github.com/brettlangdon/importhook - - This one is pretty intrusive, and has some issues, e.g. https://github.com/brettlangdon/importhook/issues/4 - - Let's try it: - : $ PYTHONPATH=overlay3/src:main/src python3 -c 'import my.twitter._hook; import my.twitter.all as M; print(M.tweets())' - : [main] my.twitter.all hello - : [main] my.twitter.common hello - : [main] my.twitter.gdpr hello - : EXECUTING IMPORT HOOK! - : ['GDPR tweet 1', 'GDPR tweet 2'] - - Ok it worked, and seems pretty neat. - However sadly it doesn't work with =tmp_config= (TODO add a proper demo?) - Not sure if it's more of an issue with =tmp_config= implementation (which is very hacky), or =importhook= itself? - - In addition, still the question is where to put the hook itself, but in that case even a global one could be fine. - - - define hook in =my/twitter/__init__.py= - - Basically, use =extend_path= to make it behave like a namespace package, but in addition, patch original =my.twitter.talon=? - - : $ cat overlay2/src/my/twitter/__init__.py - : print(f'[overlay2] {__name__} hello') - : - : from pkgutil import extend_path - : __path__ = extend_path(__path__, __name__) - : - : def hack_gdpr_module() -> None: - : from . import gdpr - : tweets_orig = gdpr.tweets - : def tweets_patched(): - : return [t.replace('gdpr', 'GDPR') for t in tweets_orig()] - : gdpr.tweets = tweets_patched - : - : hack_gdpr_module() - - This actually seems to work?? - - : PYTHONPATH=overlay2/src:main/src python3 -c 'import my.twitter.all as M; print(M.tweets())' - : [overlay2] my.twitter hello - : [main] my.twitter.gdpr hello - : [main] my.twitter.all hello - : [main] my.twitter.common hello - : ['GDPR tweet 1', 'GDPR tweet 2'] - - However, this doesn't stack, i.e. if the 'parent' overlay had its own =__init__.py=, it won't get called. - -- shadow the original module and temporarily modify =__path__= before importing the same module from the parent overlay - - This approach is implemented in =my.core.experimental.import_original_module= - - TODO demonstrate it properly, but I think that also works in a 'chain' of overlays - - Seems like that option is the most promising so far, albeit very hacky. - -Note that none of these options work well with mypy (since it's all dynamic hackery), even if you disregard the issues described in the previous sections. - -# TODO .pkg files? somewhat interesting... https://github.com/python/cpython/blob/3.12/Lib/pkgutil.py#L395-L410 diff --git a/doc/QUERY.md b/doc/QUERY.md index 9a5d9d3..b672dff 100644 --- a/doc/QUERY.md +++ b/doc/QUERY.md @@ -97,9 +97,9 @@ By default, this just returns the items in the order they were returned by the f hpi query my.coding.commits.commits --order-key committed_dt --limit 1 --reverse --output pprint --stream Commit(committed_dt=datetime.datetime(2023, 4, 14, 23, 9, 1, tzinfo=datetime.timezone(datetime.timedelta(days=-1, seconds=61200))), authored_dt=datetime.datetime(2023, 4, 14, 23, 4, 1, tzinfo=datetime.timezone(datetime.timedelta(days=-1, seconds=61200))), - message='sources.smscalls: propagate errors if there are breaking ' + message='sources.smscalls: propogate errors if there are breaking ' 'schema changes', - repo='/home/username/Repos/promnesia-fork', + repo='/home/sean/Repos/promnesia-fork', sha='22a434fca9a28df9b0915ccf16368df129d2c9ce', ref='refs/heads/smscalls-handle-result') ``` @@ -195,7 +195,7 @@ To preview, you can use something like [`qgis`](https://qgis.org/en/site/) or fo chicago trip -(Sidenote: this is [`@purarue`](https://github.com/purarue/)s locations, on a trip to Chicago) +(Sidenote: this is [`@seanbreckenridge`](https://github.com/seanbreckenridge/)s locations, on a trip to Chicago) ## Python reference @@ -301,4 +301,4 @@ The `hpi query` command is a CLI wrapper around the code in [`query.py`](../my/c If you specify a range, drop_unsorted is forced to be True ``` -Those can be imported and accept any sort of iterator, `hpi query` just defaults to the output of functions here. As an example, see [`listens`](https://github.com/purarue/HPI-personal/blob/master/scripts/listens) which just passes an generator (iterator) as the first argument to `query_range` +Those can be imported and accept any sort of iterator, `hpi query` just defaults to the output of functions here. As an example, see [`listens`](https://github.com/seanbreckenridge/HPI-personal/blob/master/scripts/listens) which just passes an generator (iterator) as the first argument to `query_range` diff --git a/doc/SETUP.org b/doc/SETUP.org index ee9571c..0fced62 100644 --- a/doc/SETUP.org +++ b/doc/SETUP.org @@ -387,7 +387,7 @@ But there is an extra caveat: rexport is already coming with nice [[https://gith Several other HPI modules are following a similar pattern: hypothesis, instapaper, pinboard, kobo, etc. -Since the [[https://github.com/karlicoss/rexport#api-limitations][reddit API has limited results]], you can use [[https://github.com/purarue/pushshift_comment_export][my.reddit.pushshift]] to access older reddit comments, which both then get merged into =my.reddit.all.comments= +Since the [[https://github.com/karlicoss/rexport#api-limitations][reddit API has limited results]], you can use [[https://github.com/seanbreckenridge/pushshift_comment_export][my.reddit.pushshift]] to access older reddit comments, which both then get merged into =my.reddit.all.comments= ** Twitter diff --git a/doc/overlays/install_packages.sh b/doc/overlays/install_packages.sh deleted file mode 100755 index 5853e08..0000000 --- a/doc/overlays/install_packages.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -eux -pip3 install --user "$@" -e main/ -pip3 install --user "$@" -e overlay/ diff --git a/doc/overlays/main/setup.py b/doc/overlays/main/setup.py deleted file mode 100644 index 51ac55c..0000000 --- a/doc/overlays/main/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import setup, find_namespace_packages # type: ignore - - -def main() -> None: - pkgs = find_namespace_packages('src') - pkg = min(pkgs) - setup( - name='hpi-main', - zip_safe=False, - packages=pkgs, - package_dir={'': 'src'}, - package_data={pkg: ['py.typed']}, - ) - - -if __name__ == '__main__': - main() diff --git a/doc/overlays/main/src/my/py.typed b/doc/overlays/main/src/my/py.typed deleted file mode 100644 index e69de29..0000000 diff --git a/doc/overlays/main/src/my/reddit.py b/doc/overlays/main/src/my/reddit.py deleted file mode 100644 index ae4e481..0000000 --- a/doc/overlays/main/src/my/reddit.py +++ /dev/null @@ -1,11 +0,0 @@ -print(f'[main] {__name__} hello') - - -def upvotes() -> list[str]: - return [ - 'reddit upvote1', - 'reddit upvote2', - ] - - -trigger_mypy_error: str = 123 diff --git a/doc/overlays/main/src/my/twitter/all.py b/doc/overlays/main/src/my/twitter/all.py deleted file mode 100644 index feb9fce..0000000 --- a/doc/overlays/main/src/my/twitter/all.py +++ /dev/null @@ -1,7 +0,0 @@ -print(f'[main] {__name__} hello') - -from .common import merge - -def tweets() -> list[str]: - from . import gdpr - return merge(gdpr) diff --git a/doc/overlays/main/src/my/twitter/common.py b/doc/overlays/main/src/my/twitter/common.py deleted file mode 100644 index 4121b5b..0000000 --- a/doc/overlays/main/src/my/twitter/common.py +++ /dev/null @@ -1,11 +0,0 @@ -print(f'[main] {__name__} hello') - -from typing import Protocol - -class Source(Protocol): - def tweets(self) -> list[str]: - ... - -def merge(*sources: Source) -> list[str]: - from itertools import chain - return list(chain.from_iterable(src.tweets() for src in sources)) diff --git a/doc/overlays/main/src/my/twitter/gdpr.py b/doc/overlays/main/src/my/twitter/gdpr.py deleted file mode 100644 index 22ea220..0000000 --- a/doc/overlays/main/src/my/twitter/gdpr.py +++ /dev/null @@ -1,9 +0,0 @@ -print(f'[main] {__name__} hello') - -def tweets() -> list[str]: - return [ - 'gdpr tweet 1', - 'gdpr tweet 2', - ] - -trigger_mypy_error: str = 123 diff --git a/doc/overlays/overlay/setup.py b/doc/overlays/overlay/setup.py deleted file mode 100644 index aaaa244..0000000 --- a/doc/overlays/overlay/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import setup, find_namespace_packages # type: ignore - - -def main() -> None: - pkgs = find_namespace_packages('src') - pkg = min(pkgs) - setup( - name='hpi-overlay', - zip_safe=False, - packages=pkgs, - package_dir={'': 'src'}, - package_data={pkg: ['py.typed']}, - ) - - -if __name__ == '__main__': - main() diff --git a/doc/overlays/overlay/src/my/py.typed b/doc/overlays/overlay/src/my/py.typed deleted file mode 100644 index e69de29..0000000 diff --git a/doc/overlays/overlay/src/my/twitter/all.py b/doc/overlays/overlay/src/my/twitter/all.py deleted file mode 100644 index 895d84b..0000000 --- a/doc/overlays/overlay/src/my/twitter/all.py +++ /dev/null @@ -1,8 +0,0 @@ -print(f'[overlay] {__name__} hello') - -from .common import merge - -def tweets() -> list[str]: - from . import gdpr - from . import talon - return merge(gdpr, talon) diff --git a/doc/overlays/overlay/src/my/twitter/talon.py b/doc/overlays/overlay/src/my/twitter/talon.py deleted file mode 100644 index 782236c..0000000 --- a/doc/overlays/overlay/src/my/twitter/talon.py +++ /dev/null @@ -1,9 +0,0 @@ -print(f'[overlay] {__name__} hello') - -def tweets() -> list[str]: - return [ - 'talon tweet 1', - 'talon tweet 2', - ] - -trigger_mypy_error: str = 123 diff --git a/doc/overlays/overlay2/setup.py b/doc/overlays/overlay2/setup.py deleted file mode 100644 index e34e7de..0000000 --- a/doc/overlays/overlay2/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import setup, find_namespace_packages # type: ignore - - -def main() -> None: - pkgs = find_namespace_packages('src') - pkg = min(pkgs) - setup( - name='hpi-overlay2', - zip_safe=False, - packages=pkgs, - package_dir={'': 'src'}, - package_data={pkg: ['py.typed']}, - ) - - -if __name__ == '__main__': - main() diff --git a/doc/overlays/overlay2/src/my/py.typed b/doc/overlays/overlay2/src/my/py.typed deleted file mode 100644 index e69de29..0000000 diff --git a/doc/overlays/overlay2/src/my/twitter/__init__.py b/doc/overlays/overlay2/src/my/twitter/__init__.py deleted file mode 100644 index 9c5674f..0000000 --- a/doc/overlays/overlay2/src/my/twitter/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -print(f'[overlay2] {__name__} hello') - -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) - -def hack_gdpr_module() -> None: - from . import gdpr - tweets_orig = gdpr.tweets - def tweets_patched(): - return [t.replace('gdpr', 'GDPR') for t in tweets_orig()] - gdpr.tweets = tweets_patched - -hack_gdpr_module() diff --git a/doc/overlays/overlay3/setup.py b/doc/overlays/overlay3/setup.py deleted file mode 100644 index 106a115..0000000 --- a/doc/overlays/overlay3/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import setup, find_namespace_packages # type: ignore - - -def main() -> None: - pkgs = find_namespace_packages('src') - pkg = min(pkgs) - setup( - name='hpi-overlay3', - zip_safe=False, - packages=pkgs, - package_dir={'': 'src'}, - package_data={pkg: ['py.typed']}, - ) - - -if __name__ == '__main__': - main() diff --git a/doc/overlays/overlay3/src/my/py.typed b/doc/overlays/overlay3/src/my/py.typed deleted file mode 100644 index e69de29..0000000 diff --git a/doc/overlays/overlay3/src/my/twitter/_hook.py b/doc/overlays/overlay3/src/my/twitter/_hook.py deleted file mode 100644 index ce249ae..0000000 --- a/doc/overlays/overlay3/src/my/twitter/_hook.py +++ /dev/null @@ -1,9 +0,0 @@ -import importhook - -@importhook.on_import('my.twitter.gdpr') -def on_import(gdpr): - print("EXECUTING IMPORT HOOK!") - tweets_orig = gdpr.tweets - def tweets_patched(): - return [t.replace('gdpr', 'GDPR') for t in tweets_orig()] - gdpr.tweets = tweets_patched diff --git a/misc/.flake8-karlicoss b/misc/.flake8-karlicoss index 5933253..3c98b96 100644 --- a/misc/.flake8-karlicoss +++ b/misc/.flake8-karlicoss @@ -32,6 +32,6 @@ ignore = # # as a reference: -# https://github.com/purarue/cookiecutter-template/blob/master/%7B%7Bcookiecutter.module_name%7D%7D/setup.cfg +# https://github.com/seanbreckenridge/cookiecutter-template/blob/master/%7B%7Bcookiecutter.module_name%7D%7D/setup.cfg # and this https://github.com/karlicoss/HPI/pull/151 # find ./my | entr flake8 --ignore=E402,E501,E741,W503,E266,E302,E305,E203,E261,E252,E251,E221,W291,E225,E303,E702,E202,F841,E731,E306,E127 E722,E231 my | grep -v __NOT_HPI_MODULE__ diff --git a/my/arbtt.py b/my/arbtt.py index 5d4bf8e..5683515 100644 --- a/my/arbtt.py +++ b/my/arbtt.py @@ -2,22 +2,19 @@ [[https://github.com/nomeata/arbtt#arbtt-the-automatic-rule-based-time-tracker][Arbtt]] time tracking ''' -from __future__ import annotations - REQUIRES = ['ijson', 'cffi'] # NOTE likely also needs libyajl2 from apt or elsewhere? -from collections.abc import Iterable, Sequence -from dataclasses import dataclass from pathlib import Path +from typing import Sequence, Iterable, List, Optional def inputs() -> Sequence[Path]: try: from my.config import arbtt as user_config except ImportError: - from my.core.warnings import low + from .core.warnings import low low("Couldn't find 'arbtt' config section, falling back to the default capture.log (usually in HOME dir). Add 'arbtt' section with logfiles = '' to suppress this warning.") return [] else: @@ -25,9 +22,8 @@ def inputs() -> Sequence[Path]: return get_files(user_config.logfiles) - -from my.core import Json, PathIsh, datetime_aware -from my.core.compat import fromisoformat +from .core import dataclass, Json, PathIsh, datetime_aware +from .core.common import isoparse @dataclass @@ -43,7 +39,6 @@ class Entry: @property def dt(self) -> datetime_aware: # contains utc already - # TODO after python>=3.11, could just use fromisoformat ds = self.json['date'] elen = 27 lds = len(ds) @@ -51,13 +46,13 @@ class Entry: # ugh. sometimes contains less that 6 decimal points ds = ds[:-1] + '0' * (elen - lds) + 'Z' elif lds > elen: - # and sometimes more... + # ahd sometimes more... ds = ds[:elen - 1] + 'Z' - return fromisoformat(ds) + return isoparse(ds) @property - def active(self) -> str | None: + def active(self) -> Optional[str]: # NOTE: WIP, might change this in the future... ait = (w for w in self.json['windows'] if w['active']) a = next(ait, None) @@ -76,18 +71,17 @@ class Entry: def entries() -> Iterable[Entry]: inps = list(inputs()) - base: list[PathIsh] = ['arbtt-dump', '--format=json'] + base: List[PathIsh] = ['arbtt-dump', '--format=json'] - cmds: list[list[PathIsh]] + cmds: List[List[PathIsh]] if len(inps) == 0: cmds = [base] # rely on default else: # otherwise, 'merge' them - cmds = [[*base, '--logfile', f] for f in inps] + cmds = [base + ['--logfile', f] for f in inps] - from subprocess import PIPE, Popen - - import ijson.backends.yajl2_cffi as ijson # type: ignore + import ijson.backends.yajl2_cffi as ijson # type: ignore + from subprocess import Popen, PIPE for cmd in cmds: with Popen(cmd, stdout=PIPE) as p: out = p.stdout; assert out is not None @@ -96,8 +90,8 @@ def entries() -> Iterable[Entry]: def fill_influxdb() -> None: - from .core.freezer import Freezer from .core.influxdb import magic_fill + from .core.freezer import Freezer freezer = Freezer(Entry) fit = (freezer.freeze(e) for e in entries()) # TODO crap, influxdb doesn't like None https://github.com/influxdata/influxdb/issues/7722 @@ -109,8 +103,6 @@ def fill_influxdb() -> None: magic_fill(fit, name=f'{entries.__module__}:{entries.__name__}') -from .core import Stats, stat - - +from .core import stat, Stats def stats() -> Stats: return stat(entries) diff --git a/my/bluemaestro.py b/my/bluemaestro.py index 8c739f0..8f05aac 100644 --- a/my/bluemaestro.py +++ b/my/bluemaestro.py @@ -1,63 +1,39 @@ +#!/usr/bin/python3 """ [[https://bluemaestro.com/products/product-details/bluetooth-environmental-monitor-and-logger][Bluemaestro]] temperature/humidity/pressure monitor """ -from __future__ import annotations - # todo most of it belongs to DAL... but considering so few people use it I didn't bother for now -import re -import sqlite3 -from abc import abstractmethod -from collections.abc import Iterable, Sequence from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path -from typing import Protocol +import re +import sqlite3 +from typing import Iterable, Sequence, Set, Optional import pytz from my.core import ( - Paths, - Res, - Stats, get_files, make_logger, + Res, stat, - unwrap, + Stats, + influxdb, ) -from my.core.cachew import mcachew +from my.core.common import mcachew +from my.core.error import unwrap from my.core.pandas import DataFrameT, as_dataframe from my.core.sqlite import sqlite_connect_immutable - -class config(Protocol): - @property - @abstractmethod - def export_path(self) -> Paths: - raise NotImplementedError - - @property - def tz(self) -> pytz.BaseTzInfo: - # fixme: later, rely on the timezone provider - # NOTE: the timezone should be set with respect to the export date!!! - return pytz.timezone('Europe/London') - # TODO when I change tz, check the diff - - -def make_config() -> config: - from my.config import bluemaestro as user_config - - class combined_config(user_config, config): ... - - return combined_config() +from my.config import bluemaestro as config logger = make_logger(__name__) def inputs() -> Sequence[Path]: - cfg = make_config() - return get_files(cfg.export_path) + return get_files(config.export_path) Celsius = float @@ -74,6 +50,12 @@ class Measurement: dewpoint: Celsius +# fixme: later, rely on the timezone provider +# NOTE: the timezone should be set with respect to the export date!!! +tz = pytz.timezone('Europe/London') +# TODO when I change tz, check the diff + + def is_bad_table(name: str) -> bool: # todo hmm would be nice to have a hook that can patch any module up to delegate = getattr(config, 'is_bad_table', None) @@ -82,31 +64,28 @@ def is_bad_table(name: str) -> bool: @mcachew(depends_on=inputs) def measurements() -> Iterable[Res[Measurement]]: - cfg = make_config() - tz = cfg.tz - # todo ideally this would be via arguments... but needs to be lazy paths = inputs() total = len(paths) width = len(str(total)) - last: datetime | None = None + last: Optional[datetime] = None # tables are immutable, so can save on processing.. - processed_tables: set[str] = set() + processed_tables: Set[str] = set() for idx, path in enumerate(paths): logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}') tot = 0 new = 0 # todo assert increasing timestamp? with sqlite_connect_immutable(path) as db: - db_dt: datetime | None = None + db_dt: Optional[datetime] = None try: datas = db.execute( f'SELECT "{path.name}" as name, Time, Temperature, Humidity, Pressure, Dewpoint FROM data ORDER BY log_index' ) oldfmt = True - [(db_dts,)] = db.execute('SELECT last_download FROM info') + db_dts = list(db.execute('SELECT last_download FROM info'))[0][0] if db_dts == 'N/A': # ??? happens for 20180923-20180928 continue @@ -139,7 +118,7 @@ def measurements() -> Iterable[Res[Measurement]]: processed_tables |= set(log_tables) # todo use later? - frequencies = [list(db.execute(f'SELECT interval from {t.replace("_log", "_meta")}'))[0][0] for t in log_tables] # noqa: RUF015 + frequencies = [list(db.execute(f'SELECT interval from {t.replace("_log", "_meta")}'))[0][0] for t in log_tables] # todo could just filter out the older datapoints?? dunno. @@ -155,7 +134,7 @@ def measurements() -> Iterable[Res[Measurement]]: oldfmt = False db_dt = None - for (name, tsc, temp, hum, pres, dewp) in datas: + for i, (name, tsc, temp, hum, pres, dewp) in enumerate(datas): if is_bad_table(name): continue @@ -232,8 +211,6 @@ def dataframe() -> DataFrameT: def fill_influxdb() -> None: - from my.core import influxdb - influxdb.fill(measurements(), measurement=__name__) diff --git a/my/body/blood.py b/my/body/blood.py index 867568c..fb035eb 100644 --- a/my/body/blood.py +++ b/my/body/blood.py @@ -2,42 +2,41 @@ Blood tracking (manual org-mode entries) """ -from __future__ import annotations - -from collections.abc import Iterable from datetime import datetime -from typing import NamedTuple - -import orgparse -import pandas as pd - -from my.config import blood as config # type: ignore[attr-defined] +from typing import Iterable, NamedTuple, Optional from ..core.error import Res -from ..core.orgmode import one_table, parse_org_datetime +from ..core.orgmode import parse_org_datetime, one_table + + +import pandas as pd +import orgparse + + +from my.config import blood as config # type: ignore[attr-defined] class Entry(NamedTuple): dt: datetime - ketones : float | None=None - glucose : float | None=None + ketones : Optional[float]=None + glucose : Optional[float]=None - vitamin_d : float | None=None - vitamin_b12 : float | None=None + vitamin_d : Optional[float]=None + vitamin_b12 : Optional[float]=None - hdl : float | None=None - ldl : float | None=None - triglycerides: float | None=None + hdl : Optional[float]=None + ldl : Optional[float]=None + triglycerides: Optional[float]=None - source : str | None=None - extra : str | None=None + source : Optional[str]=None + extra : Optional[str]=None Result = Res[Entry] -def try_float(s: str) -> float | None: +def try_float(s: str) -> Optional[float]: l = s.split() if len(l) == 0: return None @@ -106,7 +105,6 @@ def blood_tests_data() -> Iterable[Result]: def data() -> Iterable[Result]: from itertools import chain - from ..core.error import sort_res_by datas = chain(glucose_ketones_data(), blood_tests_data()) return sort_res_by(datas, key=lambda e: e.dt) diff --git a/my/body/exercise/all.py b/my/body/exercise/all.py index d0df747..e86a5af 100644 --- a/my/body/exercise/all.py +++ b/my/body/exercise/all.py @@ -7,10 +7,10 @@ from ...core.pandas import DataFrameT, check_dataframe @check_dataframe def dataframe() -> DataFrameT: # this should be somehow more flexible... - import pandas as pd - from ...endomondo import dataframe as EDF - from ...runnerup import dataframe as RDF + from ...runnerup import dataframe as RDF + + import pandas as pd return pd.concat([ EDF(), RDF(), diff --git a/my/body/exercise/cardio.py b/my/body/exercise/cardio.py index d8a6afd..083b972 100644 --- a/my/body/exercise/cardio.py +++ b/my/body/exercise/cardio.py @@ -3,6 +3,7 @@ Cardio data, filtered from various data sources ''' from ...core.pandas import DataFrameT, check_dataframe + CARDIO = { 'Running', 'Running, treadmill', diff --git a/my/body/exercise/cross_trainer.py b/my/body/exercise/cross_trainer.py index 30f96f9..d073f43 100644 --- a/my/body/exercise/cross_trainer.py +++ b/my/body/exercise/cross_trainer.py @@ -5,18 +5,16 @@ This is probably too specific to my needs, so later I will move it away to a per For now it's worth keeping it here as an example and perhaps utility functions might be useful for other HPI modules. ''' -from __future__ import annotations - from datetime import datetime, timedelta +from typing import Optional -import pytz +from ...core.pandas import DataFrameT, check_dataframe as cdf +from ...core.orgmode import collect, Table, parse_org_datetime, TypedTable from my.config import exercise as config -from ...core.orgmode import Table, TypedTable, collect, parse_org_datetime -from ...core.pandas import DataFrameT -from ...core.pandas import check_dataframe as cdf +import pytz # FIXME how to attach it properly? tz = pytz.timezone('Europe/London') @@ -107,7 +105,7 @@ def dataframe() -> DataFrameT: rows = [] idxs = [] # type: ignore[var-annotated] NO_ENDOMONDO = 'no endomondo matches' - for _i, row in mdf.iterrows(): + for i, row in mdf.iterrows(): rd = row.to_dict() mdate = row['date'] if pd.isna(mdate): @@ -116,7 +114,7 @@ def dataframe() -> DataFrameT: rows.append(rd) # presumably has an error set continue - idx: int | None + idx: Optional[int] close = edf[edf['start_time'].apply(lambda t: pd_date_diff(t, mdate)).abs() < _DELTA] if len(close) == 0: idx = None @@ -165,9 +163,7 @@ def dataframe() -> DataFrameT: # TODO wtf?? where is speed coming from?? -from ...core import Stats, stat - - +from ...core import stat, Stats def stats() -> Stats: return stat(cross_trainer_data) diff --git a/my/body/sleep/common.py b/my/body/sleep/common.py index fc288e5..7bc1021 100644 --- a/my/body/sleep/common.py +++ b/my/body/sleep/common.py @@ -1,6 +1,5 @@ -from ...core import Stats, stat -from ...core.pandas import DataFrameT -from ...core.pandas import check_dataframe as cdf +from ...core import stat, Stats +from ...core.pandas import DataFrameT, check_dataframe as cdf class Combine: @@ -8,7 +7,7 @@ class Combine: self.modules = modules @cdf - def dataframe(self, *, with_temperature: bool=True) -> DataFrameT: + def dataframe(self, with_temperature: bool=True) -> DataFrameT: import pandas as pd # todo include 'source'? df = pd.concat([m.dataframe() for m in self.modules]) @@ -18,21 +17,15 @@ class Combine: bdf = BM.dataframe() temp = bdf['temp'] - # sort index and drop nans, otherwise indexing with [start: end] gonna complain - temp = pd.Series( - temp.values, - index=pd.to_datetime(temp.index, utc=True) - ).sort_index() - temp = temp.loc[temp.index.dropna()] - def calc_avg_temperature(row): start = row['sleep_start'] end = row['sleep_end'] if pd.isna(start) or pd.isna(end): return None + between = (start <= temp.index) & (temp.index <= end) # on no temp data, returns nan, ok - return temp[start: end].mean() + return temp[between].mean() df['avg_temp'] = df.apply(calc_avg_temperature, axis=1) return df diff --git a/my/body/sleep/main.py b/my/body/sleep/main.py index 2460e03..29b12a7 100644 --- a/my/body/sleep/main.py +++ b/my/body/sleep/main.py @@ -1,6 +1,7 @@ -from ... import emfit, jawbone -from .common import Combine +from ... import jawbone +from ... import emfit +from .common import Combine _combined = Combine([ jawbone, emfit, diff --git a/my/body/weight.py b/my/body/weight.py index d5478ef..def3e87 100644 --- a/my/body/weight.py +++ b/my/body/weight.py @@ -2,29 +2,21 @@ Weight data (manually logged) ''' -from collections.abc import Iterator -from dataclasses import dataclass from datetime import datetime -from typing import Any +from typing import NamedTuple, Iterator -from my import orgmode -from my.core import make_logger -from my.core.error import Res, extract_error_datetime, set_error_datetime +from ..core import LazyLogger +from ..core.error import Res, set_error_datetime, extract_error_datetime -config = Any +from .. import orgmode + +from my.config import weight as config # type: ignore[attr-defined] -def make_config() -> config: - from my.config import weight as user_config # type: ignore[attr-defined] - - return user_config() +log = LazyLogger('my.body.weight') -log = make_logger(__name__) - - -@dataclass -class Entry: +class Entry(NamedTuple): dt: datetime value: float # TODO comment?? @@ -34,8 +26,6 @@ Result = Res[Entry] def from_orgmode() -> Iterator[Result]: - cfg = make_config() - orgs = orgmode.query() for o in orgmode.query().all(): if 'weight' not in o.tags: @@ -56,8 +46,8 @@ def from_orgmode() -> Iterator[Result]: yield e continue # FIXME use timezone provider - created = cfg.default_timezone.localize(created) - assert created is not None # ??? somehow mypy wasn't happy? + created = config.default_timezone.localize(created) + assert created is not None #??? somehow mypy wasn't happy? yield Entry( dt=created, value=w, @@ -67,23 +57,21 @@ def from_orgmode() -> Iterator[Result]: def make_dataframe(data: Iterator[Result]): import pandas as pd - def it(): for e in data: if isinstance(e, Exception): dt = extract_error_datetime(e) yield { - 'dt': dt, + 'dt' : dt, 'error': str(e), } else: yield { - 'dt': e.dt, + 'dt' : e.dt, 'weight': e.value, } - df = pd.DataFrame(it()) - df = df.set_index('dt') + df.set_index('dt', inplace=True) # TODO not sure about UTC?? df.index = pd.to_datetime(df.index, utc=True) return df @@ -93,7 +81,6 @@ def dataframe(): entries = from_orgmode() return make_dataframe(entries) - # TODO move to a submodule? e.g. my.body.weight.orgmode? # so there could be more sources # not sure about my.body thing though diff --git a/my/books/kobo.py b/my/books/kobo.py index 40b7ed7..2a469d0 100644 --- a/my/books/kobo.py +++ b/my/books/kobo.py @@ -1,6 +1,7 @@ -from my.core import warnings +from ..core import warnings warnings.high('my.books.kobo is deprecated! Please use my.kobo instead!') -from my.core.util import __NOT_HPI_MODULE__ -from my.kobo import * +from ..core.util import __NOT_HPI_MODULE__ + +from ..kobo import * # type: ignore[no-redef] diff --git a/my/browser/active_browser.py b/my/browser/active_browser.py index 1686fc5..601182a 100644 --- a/my/browser/active_browser.py +++ b/my/browser/active_browser.py @@ -1,13 +1,12 @@ """ -Parses active browser history by backing it up with [[http://github.com/purarue/sqlite_backup][sqlite_backup]] +Parses active browser history by backing it up with [[http://github.com/seanbreckenridge/sqlite_backup][sqlite_backup]] """ REQUIRES = ["browserexport", "sqlite_backup"] -from dataclasses import dataclass from my.config import browser as user_config -from my.core import Paths +from my.core import Paths, dataclass @dataclass @@ -19,18 +18,16 @@ class config(user_config.active_browser): export_path: Paths -from collections.abc import Iterator, Sequence from pathlib import Path +from typing import Sequence, Iterator -from browserexport.merge import Visit, read_visits +from my.core import get_files, Stats, make_logger +from browserexport.merge import read_visits, Visit from sqlite_backup import sqlite_backup -from my.core import Stats, get_files, make_logger - logger = make_logger(__name__) from .common import _patch_browserexport_logs - _patch_browserexport_logs(logger.level) diff --git a/my/browser/all.py b/my/browser/all.py index feb973a..a7d12b4 100644 --- a/my/browser/all.py +++ b/my/browser/all.py @@ -1,9 +1,9 @@ -from collections.abc import Iterator - -from browserexport.merge import Visit, merge_visits +from typing import Iterator from my.core import Stats from my.core.source import import_source +from browserexport.merge import merge_visits, Visit + src_export = import_source(module_name="my.browser.export") src_active = import_source(module_name="my.browser.active_browser") diff --git a/my/browser/export.py b/my/browser/export.py index 52ade0e..46a4217 100644 --- a/my/browser/export.py +++ b/my/browser/export.py @@ -1,36 +1,31 @@ """ -Parses browser history using [[http://github.com/purarue/browserexport][browserexport]] +Parses browser history using [[http://github.com/seanbreckenridge/browserexport][browserexport]] """ REQUIRES = ["browserexport"] -from collections.abc import Iterator, Sequence -from dataclasses import dataclass -from pathlib import Path - -from browserexport.merge import Visit, read_and_merge - -from my.core import ( - Paths, - Stats, - get_files, - make_logger, - stat, -) -from my.core.cachew import mcachew - -from .common import _patch_browserexport_logs - -import my.config # isort: skip +from my.config import browser as user_config +from my.core import Paths, dataclass @dataclass -class config(my.config.browser.export): +class config(user_config.export): # path[s]/glob to your backed up browser history sqlite files export_path: Paths -logger = make_logger(__name__) +from pathlib import Path +from typing import Iterator, Sequence, List + +from my.core import Stats, get_files, LazyLogger +from my.core.common import mcachew + +from browserexport.merge import read_and_merge, Visit + +from .common import _patch_browserexport_logs + + +logger = LazyLogger(__name__, level="warning") _patch_browserexport_logs(logger.level) @@ -39,10 +34,16 @@ def inputs() -> Sequence[Path]: return get_files(config.export_path) -@mcachew(depends_on=inputs, logger=logger) +def _cachew_depends_on() -> List[str]: + return [str(f) for f in inputs()] + + +@mcachew(depends_on=_cachew_depends_on, logger=logger) def history() -> Iterator[Visit]: yield from read_and_merge(inputs()) def stats() -> Stats: + from my.core import stat + return {**stat(history)} diff --git a/my/bumble/android.py b/my/bumble/android.py index 3f9fa13..86c9d1e 100644 --- a/my/bumble/android.py +++ b/my/bumble/android.py @@ -3,24 +3,24 @@ Bumble data from Android app database (in =/data/data/com.bumble.app/databases/C """ from __future__ import annotations -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime -from pathlib import Path +from typing import Iterator, Sequence, Optional, Dict from more_itertools import unique_everseen -from my.core import Paths, get_files - -from my.config import bumble as user_config # isort: skip +from my.config import bumble as user_config +from ..core import Paths @dataclass class config(user_config.android): # paths[s]/glob to the exported sqlite databases export_path: Paths +from ..core import get_files +from pathlib import Path def inputs() -> Sequence[Path]: return get_files(config.export_path) @@ -43,23 +43,20 @@ class _BaseMessage: @dataclass(unsafe_hash=True) class _Message(_BaseMessage): conversation_id: str - reply_to_id: str | None + reply_to_id: Optional[str] @dataclass(unsafe_hash=True) class Message(_BaseMessage): person: Person - reply_to: Message | None + reply_to: Optional[Message] import json -import sqlite3 from typing import Union - -from my.core.compat import assert_never - -from ..core import Res -from ..core.sqlite import select, sqlite_connect_immutable +from ..core import Res, assert_never +import sqlite3 +from ..core.sqlite import sqlite_connect_immutable, select EntitiesRes = Res[Union[Person, _Message]] @@ -109,11 +106,10 @@ def _handle_db(db: sqlite3.Connection) -> Iterator[EntitiesRes]: def _key(r: EntitiesRes): if isinstance(r, _Message): - if '/hidden?' in r.text: + if '&srv_width=' in r.text: # ugh. seems that image URLs change all the time in the db? # can't access them without login anyway # so use a different key for such messages - # todo maybe normalize text instead? since it's gonna always trigger diffs down the line return (r.id, r.created) return r @@ -122,8 +118,8 @@ _UNKNOWN_PERSON = "UNKNOWN_PERSON" def messages() -> Iterator[Res[Message]]: - id2person: dict[str, Person] = {} - id2msg: dict[str, Message] = {} + id2person: Dict[str, Person] = {} + id2msg: Dict[str, Message] = {} for x in unique_everseen(_entities(), key=_key): if isinstance(x, Exception): yield x diff --git a/my/calendar/holidays.py b/my/calendar/holidays.py index 522672e..6fa3560 100644 --- a/my/calendar/holidays.py +++ b/my/calendar/holidays.py @@ -9,18 +9,16 @@ from datetime import date, datetime, timedelta from functools import lru_cache from typing import Union -from my.core import Stats -from my.core.time import zone_to_countrycode +from ..core.time import zone_to_countrycode @lru_cache(1) def _calendar(): - from workalendar.registry import registry # type: ignore - + from workalendar.registry import registry # type: ignore # todo switch to using time.tz.main once _get_tz stabilizes? from ..time.tz import via_location as LTZ # TODO would be nice to do it dynamically depending on the past timezones... - tz = LTZ.get_tz(datetime.now()) + tz = LTZ._get_tz(datetime.now()) assert tz is not None zone = tz.zone; assert zone is not None code = zone_to_countrycode(zone) @@ -48,6 +46,7 @@ def is_workday(d: DateIsh) -> bool: return not is_holiday(d) +from ..core.common import Stats def stats() -> Stats: # meh, but not sure what would be a better test? res = {} diff --git a/my/cfg.py b/my/cfg.py index 9331e8a..e4020b4 100644 --- a/my/cfg.py +++ b/my/cfg.py @@ -1,6 +1,7 @@ import my.config as config from .core import __NOT_HPI_MODULE__ + from .core import warnings as W # still used in Promnesia, maybe in dashboard? diff --git a/my/codeforces.py b/my/codeforces.py deleted file mode 100644 index 9c6b7c9..0000000 --- a/my/codeforces.py +++ /dev/null @@ -1,78 +0,0 @@ -import json -from collections.abc import Iterator, Sequence -from dataclasses import dataclass -from datetime import datetime, timezone -from functools import cached_property -from pathlib import Path - -from my.config import codeforces as config # type: ignore[attr-defined] -from my.core import Res, datetime_aware, get_files - - -def inputs() -> Sequence[Path]: - return get_files(config.export_path) - - -ContestId = int - - -@dataclass -class Contest: - contest_id: ContestId - when: datetime_aware - name: str - - -@dataclass -class Competition: - contest: Contest - old_rating: int - new_rating: int - - @cached_property - def when(self) -> datetime_aware: - return self.contest.when - - -# todo not sure if parser is the best name? hmm -class Parser: - def __init__(self, *, inputs: Sequence[Path]) -> None: - self.inputs = inputs - self.contests: dict[ContestId, Contest] = {} - - def _parse_allcontests(self, p: Path) -> Iterator[Contest]: - j = json.loads(p.read_text()) - for c in j['result']: - yield Contest( - contest_id=c['id'], - when=datetime.fromtimestamp(c['startTimeSeconds'], tz=timezone.utc), - name=c['name'], - ) - - def _parse_competitions(self, p: Path) -> Iterator[Competition]: - j = json.loads(p.read_text()) - for c in j['result']: - contest_id = c['contestId'] - contest = self.contests[contest_id] - yield Competition( - contest=contest, - old_rating=c['oldRating'], - new_rating=c['newRating'], - ) - - def parse(self) -> Iterator[Res[Competition]]: - for path in inputs(): - if 'allcontests' in path.name: - # these contain information about all CF contests along with useful metadata - for contest in self._parse_allcontests(path): - # TODO some method to assert on mismatch if it exists? not sure - self.contests[contest.contest_id] = contest - elif 'codeforces' in path.name: - # these contain only contests the user participated in - yield from self._parse_competitions(path) - else: - raise RuntimeError(f"shouldn't happen: {path.name}") - - -def data() -> Iterator[Res[Competition]]: - return Parser(inputs=inputs()).parse() diff --git a/my/coding/codeforces.py b/my/coding/codeforces.py new file mode 100644 index 0000000..a8b0f65 --- /dev/null +++ b/my/coding/codeforces.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +from my.config import codeforces as config # type: ignore[attr-defined] + + +from datetime import datetime, timezone +from functools import cached_property +import json +from typing import NamedTuple, Dict, Iterator + + +from ..core import get_files, Res, unwrap +from ..core.konsume import ignore, wrap + + +Cid = int + +class Contest(NamedTuple): + cid: Cid + when: datetime + + @classmethod + def make(cls, j) -> 'Contest': + return cls( + cid=j['id'], + when=datetime.fromtimestamp(j['startTimeSeconds'], tz=timezone.utc), + ) + +Cmap = Dict[Cid, Contest] + + +def get_contests() -> Cmap: + last = max(get_files(config.export_path, 'allcontests*.json')) + j = json.loads(last.read_text()) + d = {} + for c in j['result']: + cc = Contest.make(c) + d[cc.cid] = cc + return d + + +class Competition(NamedTuple): + contest_id: Cid + contest: str + cmap: Cmap + + @cached_property + def uid(self) -> Cid: + return self.contest_id + + def __hash__(self): + return hash(self.contest_id) + + @cached_property + def when(self) -> datetime: + return self.cmap[self.uid].when + + @cached_property + def summary(self) -> str: + return f'participated in {self.contest}' # TODO + + @classmethod + def make(cls, cmap, json) -> Iterator[Res['Competition']]: + # TODO try here?? + contest_id = json['contestId'].zoom().value + contest = json['contestName'].zoom().value + yield cls( + contest_id=contest_id, + contest=contest, + cmap=cmap, + ) + # TODO ytry??? + ignore(json, 'rank', 'oldRating', 'newRating') + + +def iter_data() -> Iterator[Res[Competition]]: + cmap = get_contests() + last = max(get_files(config.export_path, 'codeforces*.json')) + + with wrap(json.loads(last.read_text())) as j: + j['status'].ignore() + res = j['result'].zoom() + + for c in list(res): # TODO maybe we want 'iter' method?? + ignore(c, 'handle', 'ratingUpdateTimeSeconds') + yield from Competition.make(cmap=cmap, json=c) + c.consume() + # TODO maybe if they are all empty, no need to consume?? + + +def get_data(): + return list(sorted(iter_data(), key=Competition.when.fget)) diff --git a/my/coding/commits.py b/my/coding/commits.py index fe17dee..67ee77d 100644 --- a/my/coding/commits.py +++ b/my/coding/commits.py @@ -1,32 +1,30 @@ """ Git commits data for repositories on your filesystem """ - -from __future__ import annotations - REQUIRES = [ 'gitpython', ] -import shutil -from collections.abc import Iterator, Sequence -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Optional, cast -from my.core import LazyLogger, PathIsh, make_config -from my.core.cachew import cache_dir, mcachew +import shutil +from pathlib import Path +from datetime import datetime, timezone +from dataclasses import dataclass, field +from typing import List, Optional, Iterator, Set, Sequence, cast + + +from my.core import PathIsh, LazyLogger, make_config +from my.core.cachew import cache_dir +from my.core.common import mcachew from my.core.warnings import high -from my.config import commits as user_config # isort: skip - +from my.config import commits as user_config @dataclass class commits_cfg(user_config): roots: Sequence[PathIsh] = field(default_factory=list) - emails: Sequence[str] | None = None - names: Sequence[str] | None = None + emails: Optional[Sequence[str]] = None + names: Optional[Sequence[str]] = None # experiment to make it lazy? @@ -43,6 +41,7 @@ def config() -> commits_cfg: import git from git.repo.fun import is_git_dir + log = LazyLogger(__name__, level='info') @@ -95,7 +94,7 @@ def _git_root(git_dir: PathIsh) -> Path: return gd # must be bare -def _repo_commits_aux(gr: git.Repo, rev: str, emitted: set[str]) -> Iterator[Commit]: +def _repo_commits_aux(gr: git.Repo, rev: str, emitted: Set[str]) -> Iterator[Commit]: # without path might not handle pull heads properly for c in gr.iter_commits(rev=rev): if not by_me(c): @@ -122,7 +121,7 @@ def _repo_commits_aux(gr: git.Repo, rev: str, emitted: set[str]) -> Iterator[Com def repo_commits(repo: PathIsh): gr = git.Repo(str(repo)) - emitted: set[str] = set() + emitted: Set[str] = set() for r in gr.references: yield from _repo_commits_aux(gr=gr, rev=r.path, emitted=emitted) @@ -138,61 +137,61 @@ def canonical_name(repo: Path) -> str: # else: # rname = r.name # if 'backups/github' in repo: - # pass # TODO + # pass # TODO def _fd_path() -> str: # todo move it to core - fd_path: str | None = shutil.which("fdfind") or shutil.which("fd-find") or shutil.which("fd") + fd_path: Optional[str] = shutil.which("fdfind") or shutil.which("fd-find") or shutil.which("fd") if fd_path is None: high("my.coding.commits requires 'fd' to be installed, See https://github.com/sharkdp/fd#installation") assert fd_path is not None return fd_path -def git_repos_in(roots: list[Path]) -> list[Path]: +def git_repos_in(roots: List[Path]) -> List[Path]: from subprocess import check_output outputs = check_output([ _fd_path(), # '--follow', # right, not so sure about follow... make configurable? '--hidden', - '--no-ignore', # otherwise doesn't go inside .git directory (from fd v9) '--full-path', '--type', 'f', '/HEAD', # judging by is_git_dir, it should always be here.. *roots, ]).decode('utf8').splitlines() - candidates = {Path(o).resolve().absolute().parent for o in outputs} + candidates = set(Path(o).resolve().absolute().parent for o in outputs) # exclude stuff within .git dirs (can happen for submodules?) candidates = {c for c in candidates if '.git' not in c.parts[:-1]} candidates = {c for c in candidates if is_git_dir(c)} - repos = sorted(map(_git_root, candidates)) + repos = list(sorted(map(_git_root, candidates))) return repos -def repos() -> list[Path]: +def repos() -> List[Path]: return git_repos_in(list(map(Path, config().roots))) # returns modification time for an index to use as hash function def _repo_depends_on(_repo: Path) -> int: - for pp in [ + for pp in { ".git/FETCH_HEAD", ".git/HEAD", "FETCH_HEAD", # bare "HEAD", # bare - ]: + }: ff = _repo / pp if ff.exists(): return int(ff.stat().st_mtime) - raise RuntimeError(f"Could not find a FETCH_HEAD/HEAD file in {_repo}") + else: + raise RuntimeError(f"Could not find a FETCH_HEAD/HEAD file in {_repo}") -def _commits(_repos: list[Path]) -> Iterator[Commit]: +def _commits(_repos: List[Path]) -> Iterator[Commit]: for r in _repos: yield from _cached_commits(r) diff --git a/my/coding/github.py b/my/coding/github.py index c495554..9358b04 100644 --- a/my/coding/github.py +++ b/my/coding/github.py @@ -1,12 +1,9 @@ -from typing import TYPE_CHECKING +import warnings -from my.core import warnings - -warnings.high('my.coding.github is deprecated! Please use my.github.all instead!') +warnings.warn('my.coding.github is deprecated! Please use my.github.all instead!') # todo why aren't DeprecationWarning shown by default?? -if not TYPE_CHECKING: - from ..github.all import events, get_events # noqa: F401 +from ..github.all import events, get_events - # todo deprecate properly - iter_events = events +# todo deprecate properly +iter_events = events diff --git a/my/coding/topcoder.py b/my/coding/topcoder.py new file mode 100644 index 0000000..96bcdf7 --- /dev/null +++ b/my/coding/topcoder.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +from my.config import topcoder as config # type: ignore[attr-defined] + + +from datetime import datetime +from functools import cached_property +import json +from typing import NamedTuple, Dict, Iterator + + +from ..core import get_files, Res, unwrap, Json +from ..core.error import Res, unwrap +from ..core.konsume import zoom, wrap, ignore + + +def _get_latest() -> Json: + pp = max(get_files(config.export_path)) + return json.loads(pp.read_text()) + + +class Competition(NamedTuple): + contest_id: str + contest: str + percentile: float + dates: str + + @cached_property + def uid(self) -> str: + return self.contest_id + + def __hash__(self): + return hash(self.contest_id) + + @cached_property + def when(self) -> datetime: + return datetime.strptime(self.dates, '%Y-%m-%dT%H:%M:%S.%fZ') + + @cached_property + def summary(self) -> str: + return f'participated in {self.contest}: {self.percentile:.0f}' + + @classmethod + def make(cls, json) -> Iterator[Res['Competition']]: + ignore(json, 'rating', 'placement') + cid = json['challengeId'].zoom().value + cname = json['challengeName'].zoom().value + percentile = json['percentile'].zoom().value + dates = json['date'].zoom().value + yield cls( + contest_id=cid, + contest=cname, + percentile=percentile, + dates=dates, + ) + + +def iter_data() -> Iterator[Res[Competition]]: + with wrap(_get_latest()) as j: + ignore(j, 'id', 'version') + + res = j['result'].zoom() + ignore(res, 'success', 'status', 'metadata') + + cont = res['content'].zoom() + ignore(cont, 'handle', 'handleLower', 'userId', 'createdAt', 'updatedAt', 'createdBy', 'updatedBy') + + cont['DEVELOP'].ignore() # TODO handle it?? + ds = cont['DATA_SCIENCE'].zoom() + + mar, srm = zoom(ds, 'MARATHON_MATCH', 'SRM') + + mar = mar['history'].zoom() + srm = srm['history'].zoom() + # TODO right, I guess I could rely on pylint for unused variables?? + + for c in mar + srm: + yield from Competition.make(json=c) + c.consume() + + +def get_data(): + return list(sorted(iter_data(), key=Competition.when.fget)) + diff --git a/my/common.py b/my/common.py index 22e9487..1b56fb5 100644 --- a/my/common.py +++ b/my/common.py @@ -1,6 +1,6 @@ from .core.warnings import high - high("DEPRECATED! Please use my.core.common instead.") from .core import __NOT_HPI_MODULE__ + from .core.common import * diff --git a/my/config.py b/my/config.py index 301bf49..9cc9c11 100644 --- a/my/config.py +++ b/my/config.py @@ -9,18 +9,17 @@ This file is used for: - mypy: this file provides some type annotations - for loading the actual user config ''' - -from __future__ import annotations - #### NOTE: you won't need this line VVVV in your personal config -from my.core import init # noqa: F401 # isort: skip +from my.core import init ### from datetime import tzinfo from pathlib import Path +from typing import List -from my.core import PathIsh, Paths + +from my.core import Paths, PathIsh class hypothesis: @@ -69,23 +68,17 @@ class pinboard: export_dir: Paths = '' class google: - class maps: - class android: - export_path: Paths = '' - takeout_path: Paths = '' -from collections.abc import Sequence -from datetime import date, datetime, timedelta -from typing import Union - +from typing import Sequence, Union, Tuple +from datetime import datetime, date, timedelta DateIsh = Union[datetime, date, str] -LatLon = tuple[float, float] +LatLon = Tuple[float, float] class location: # todo ugh, need to think about it... mypy wants the type here to be general, otherwise it can't deduce # and we can't import the types from the module itself, otherwise would be circular. common module? - home: LatLon | Sequence[tuple[DateIsh, LatLon]] = (1.0, -1.0) + home: Union[LatLon, Sequence[Tuple[DateIsh, LatLon]]] = (1.0, -1.0) home_accuracy = 30_000.0 class via_ip: @@ -106,8 +99,6 @@ class location: from typing import Literal - - class time: class tz: policy: Literal['keep', 'convert', 'throw'] @@ -126,9 +117,10 @@ class arbtt: logfiles: Paths +from typing import Optional class commits: - emails: Sequence[str] | None - names: Sequence[str] | None + emails: Optional[Sequence[str]] + names: Optional[Sequence[str]] roots: Sequence[PathIsh] @@ -154,8 +146,8 @@ class tinder: class instagram: class android: export_path: Paths - username: str | None - full_name: str | None + username: Optional[str] + full_name: Optional[str] class gdpr: export_path: Paths @@ -173,7 +165,7 @@ class materialistic: class fbmessenger: class fbmessengerexport: export_db: PathIsh - facebook_id: str | None + facebook_id: Optional[str] class android: export_path: Paths @@ -185,8 +177,6 @@ class twitter_archive: class twitter: class talon: export_path: Paths - class android: - export_path: Paths class twint: @@ -251,7 +241,7 @@ class runnerup: class emfit: export_path: Path timezone: tzinfo - excluded_sids: list[str] + excluded_sids: List[str] class foursquare: @@ -274,13 +264,8 @@ class roamresearch: class whatsapp: class android: export_path: Paths - my_user_id: str | None + my_user_id: Optional[str] class harmonic: export_path: Paths - - -class monzo: - class monzoexport: - export_path: Paths diff --git a/my/core/__init__.py b/my/core/__init__.py index a8a41f4..d753760 100644 --- a/my/core/__init__.py +++ b/my/core/__init__.py @@ -1,53 +1,40 @@ # this file only keeps the most common & critical types/utility functions -from typing import TYPE_CHECKING +from .common import get_files, PathIsh, Paths +from .common import Json +from .common import warn_if_empty +from .common import stat, Stats +from .common import datetime_naive, datetime_aware +from .common import assert_never from .cfg import make_config -from .common import PathIsh, Paths, get_files -from .compat import assert_never -from .error import Res, notnone, unwrap -from .logging import ( - make_logger, -) -from .stats import Stats, stat -from .types import ( - Json, - datetime_aware, - datetime_naive, -) +from .error import Res, unwrap +from .logging import make_logger, LazyLogger from .util import __NOT_HPI_MODULE__ -from .utils.itertools import warn_if_empty - -LazyLogger = make_logger # TODO deprecate this in favor of make_logger -if not TYPE_CHECKING: - # we used to keep these here for brevity, but feels like it only adds confusion, - # e.g. suggest that we perhaps somehow modify builtin behaviour or whatever - # so best to prefer explicit behaviour - from dataclasses import dataclass - from pathlib import Path +# just for brevity in modules +# todo not sure about these.. maybe best to rely on regular imports.. perhaps compare? +from dataclasses import dataclass +from pathlib import Path __all__ = [ - '__NOT_HPI_MODULE__', + 'get_files', 'PathIsh', 'Paths', 'Json', - 'LazyLogger', # legacy import - 'Path', - 'PathIsh', - 'Paths', - 'Res', - 'Stats', - 'assert_never', # TODO maybe deprecate from use in my.core? will be in stdlib soon - 'dataclass', - 'datetime_aware', - 'datetime_naive', - 'get_files', - 'make_config', 'make_logger', - 'notnone', - 'stat', - 'unwrap', + 'LazyLogger', # legacy import 'warn_if_empty', + 'stat', 'Stats', + 'datetime_aware', 'datetime_naive', + 'assert_never', + + 'make_config', + + '__NOT_HPI_MODULE__', + + 'Res', 'unwrap', + + 'dataclass', 'Path', ] @@ -55,7 +42,7 @@ __all__ = [ # you could put _init_hook.py next to your private my/config # that way you can configure logging/warnings/env variables on every HPI import try: - import my._init_hook # type: ignore[import-not-found] # noqa: F401 + import my._init_hook # type: ignore[import-not-found] except: pass ## diff --git a/my/core/__main__.py b/my/core/__main__.py index 7e2d8f9..ca88513 100644 --- a/my/core/__main__.py +++ b/my/core/__main__.py @@ -1,26 +1,23 @@ -from __future__ import annotations - +from contextlib import ExitStack import functools import importlib import inspect +from itertools import chain import os import shlex import shutil import sys import tempfile import traceback -from collections.abc import Iterable, Sequence -from contextlib import ExitStack -from itertools import chain +from typing import Optional, Sequence, Iterable, List, Type, Any, Callable from pathlib import Path -from subprocess import PIPE, CompletedProcess, Popen, check_call, run -from typing import Any, Callable +from subprocess import check_call, run, PIPE, CompletedProcess, Popen import click -@functools.lru_cache -def mypy_cmd() -> Sequence[str] | None: +@functools.lru_cache() +def mypy_cmd() -> Optional[Sequence[str]]: try: # preferably, use mypy from current python env import mypy # noqa: F401 fine not to use it @@ -35,7 +32,7 @@ def mypy_cmd() -> Sequence[str] | None: return None -def run_mypy(cfg_path: Path) -> CompletedProcess | None: +def run_mypy(cfg_path: Path) -> Optional[CompletedProcess]: # todo dunno maybe use the same mypy config in repository? # I'd need to install mypy.ini then?? env = {**os.environ} @@ -46,7 +43,7 @@ def run_mypy(cfg_path: Path) -> CompletedProcess | None: cmd = mypy_cmd() if cmd is None: return None - mres = run([ # noqa: UP022,PLW1510 + mres = run([ *cmd, '--namespace-packages', '--color-output', # not sure if works?? @@ -66,27 +63,21 @@ def eprint(x: str) -> None: # err=True prints to stderr click.echo(x, err=True) - def indent(x: str) -> str: - # todo use textwrap.indent? return ''.join(' ' + l for l in x.splitlines(keepends=True)) -OK = '✅' +OK = '✅' OFF = '🔲' - def info(x: str) -> None: eprint(OK + ' ' + x) - def error(x: str) -> None: eprint('❌ ' + x) - def warning(x: str) -> None: - eprint('❗ ' + x) # todo yellow? - + eprint('❗ ' + x) # todo yellow? def tb(e: Exception) -> None: tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__)) @@ -95,7 +86,6 @@ def tb(e: Exception) -> None: def config_create() -> None: from .preinit import get_mycfg_dir - mycfg_dir = get_mycfg_dir() created = False @@ -104,8 +94,7 @@ def config_create() -> None: my_config = mycfg_dir / 'my' / 'config' / '__init__.py' my_config.parent.mkdir(parents=True) - my_config.write_text( - ''' + my_config.write_text(''' ### HPI personal config ## see # https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules @@ -128,8 +117,7 @@ class example: ### you can insert your own configuration below ### but feel free to delete the stuff above if you don't need ti -'''.lstrip() - ) +'''.lstrip()) info(f'created empty config: {my_config}') created = True else: @@ -142,13 +130,12 @@ class example: # todo return the config as a result? def config_ok() -> bool: - errors: list[Exception] = [] + errors: List[Exception] = [] # at this point 'my' should already be imported, so doesn't hurt to extract paths from it import my - try: - paths: list[str] = list(my.__path__) + paths: List[str] = list(my.__path__) except Exception as e: errors.append(e) error('failed to determine module import path') @@ -158,23 +145,19 @@ def config_ok() -> bool: # first try doing as much as possible without actually importing my.config from .preinit import get_mycfg_dir - cfg_path = get_mycfg_dir() # alternative is importing my.config and then getting cfg_path from its __file__/__path__ # not sure which is better tbh ## check we're not using stub config import my.core - try: core_pkg_path = str(Path(my.core.__path__[0]).parent) if str(cfg_path).startswith(core_pkg_path): - error( - f''' + error(f''' Seems that the stub config is used ({cfg_path}). This is likely not going to work. See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules for more information -'''.strip() - ) +'''.strip()) errors.append(RuntimeError('bad config path')) except Exception as e: errors.append(e) @@ -188,6 +171,8 @@ See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-module # use a temporary directory, useful because # - compileall ignores -B, so always craps with .pyc files (annoyng on RO filesystems) # - compileall isn't following symlinks, just silently ignores them + # note: ugh, annoying that copytree requires a non-existing dir before 3.8. + # once we have min version 3.8, can use dirs_exist_ok=True param tdir = Path(td) / 'cfg' # NOTE: compileall still returns code 0 if the path doesn't exist.. # but in our case hopefully it's not an issue @@ -196,7 +181,7 @@ See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-module try: # this will resolve symlinks when copying # should be under try/catch since might fail if some symlinks are missing - shutil.copytree(cfg_path, tdir, dirs_exist_ok=True) + shutil.copytree(cfg_path, tdir) check_call(cmd) info('syntax check: ' + ' '.join(cmd)) except Exception as e: @@ -206,7 +191,7 @@ See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-module ## check types mypy_res = run_mypy(cfg_path) - if mypy_res is not None: # has mypy + if mypy_res is not None: # has mypy rc = mypy_res.returncode if rc == 0: info('mypy check : success') @@ -229,16 +214,14 @@ See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-module if len(errors) > 0: error(f'config check: {len(errors)} errors') return False - - # note: shouldn't exit here, might run something else - info('config check: success!') - return True + else: + # note: shouldn't exit here, might run something else + info('config check: success!') + return True from .util import HPIModule, modules - - -def _modules(*, all: bool = False) -> Iterable[HPIModule]: +def _modules(*, all: bool=False) -> Iterable[HPIModule]: skipped = [] for m in modules(): if not all and m.skip_reason is not None: @@ -249,7 +232,7 @@ def _modules(*, all: bool = False) -> Iterable[HPIModule]: warning(f'Skipped {len(skipped)} modules: {skipped}. Pass --all if you want to see them.') -def modules_check(*, verbose: bool, list_all: bool, quick: bool, for_modules: list[str]) -> None: +def modules_check(*, verbose: bool, list_all: bool, quick: bool, for_modules: List[str]) -> None: if len(for_modules) > 0: # if you're checking specific modules, show errors # hopefully makes sense? @@ -260,9 +243,10 @@ def modules_check(*, verbose: bool, list_all: bool, quick: bool, for_modules: li import contextlib + from .common import quick_stats + from .util import get_stats, HPIModule + from .stats import guess_stats from .error import warn_my_config_import_error - from .stats import get_stats, quick_stats - from .util import HPIModule mods: Iterable[HPIModule] if len(for_modules) == 0: @@ -273,7 +257,7 @@ def modules_check(*, verbose: bool, list_all: bool, quick: bool, for_modules: li # todo add a --all argument to disregard is_active check? for mr in mods: skip = mr.skip_reason - m = mr.name + m = mr.name if skip is not None: eprint(f'{OFF} {click.style("SKIP", fg="yellow")}: {m:<50} {skip}') continue @@ -292,8 +276,11 @@ def modules_check(*, verbose: bool, list_all: bool, quick: bool, for_modules: li continue info(f'{click.style("OK", fg="green")} : {m:<50}') - # TODO add hpi 'stats'? instead of doctor? not sure - stats = get_stats(m, guess=True) + # first try explicitly defined stats function: + stats = get_stats(m) + if stats is None: + # then try guessing.. not sure if should log somehow? + stats = guess_stats(m, quick=quick) if stats is None: eprint(" - no 'stats' function, can't check the data") @@ -304,7 +291,6 @@ def modules_check(*, verbose: bool, list_all: bool, quick: bool, for_modules: li try: kwargs = {} - # todo hmm why wouldn't they be callable?? if callable(stats) and 'quick' in inspect.signature(stats).parameters: kwargs['quick'] = quick with quick_context: @@ -323,8 +309,8 @@ def list_modules(*, list_all: bool) -> None: tabulate_warnings() for mr in _modules(all=list_all): - m = mr.name - sr = mr.skip_reason + m = mr.name + sr = mr.skip_reason if sr is None: pre = OK suf = '' @@ -340,20 +326,17 @@ def tabulate_warnings() -> None: Helper to avoid visual noise in hpi modules/doctor ''' import warnings - orig = warnings.formatwarning def override(*args, **kwargs) -> str: res = orig(*args, **kwargs) return ''.join(' ' + x for x in res.splitlines(keepends=True)) - warnings.formatwarning = override # TODO loggers as well? def _requires(modules: Sequence[str]) -> Sequence[str]: from .discovery_pure import module_by_name - mods = [module_by_name(module) for module in modules] res = [] for mod in mods: @@ -380,7 +363,7 @@ def module_requires(*, module: Sequence[str]) -> None: click.echo(x) -def module_install(*, user: bool, module: Sequence[str], parallel: bool = False, break_system_packages: bool = False) -> None: +def module_install(*, user: bool, module: Sequence[str], parallel: bool=False, break_system_packages: bool=False) -> None: if isinstance(module, str): # legacy behavior, used to take a since argument module = [module] @@ -391,9 +374,8 @@ def module_install(*, user: bool, module: Sequence[str], parallel: bool = False, warning('requirements list is empty, no need to install anything') return - use_uv = 'HPI_MODULE_INSTALL_USE_UV' in os.environ pre_cmd = [ - sys.executable, '-m', *(['uv'] if use_uv else []), 'pip', + sys.executable, '-m', 'pip', 'install', *(['--user'] if user else []), # todo maybe instead, forward all the remaining args to pip? *(['--break-system-packages'] if break_system_packages else []), # https://peps.python.org/pep-0668/ @@ -411,7 +393,7 @@ def module_install(*, user: bool, module: Sequence[str], parallel: bool = False, # I think it only helps for pypi artifacts (not git!), # and only if they weren't cached for r in requirements: - cmds.append([*pre_cmd, r]) + cmds.append(pre_cmd + [r]) else: if parallel: warning('parallel install is not supported on this platform, installing sequentially...') @@ -457,8 +439,8 @@ def _ui_getchar_pick(choices: Sequence[str], prompt: str = 'Select from: ') -> i return result_map[ch] -def _locate_functions_or_prompt(qualified_names: list[str], *, prompt: bool = True) -> Iterable[Callable[..., Any]]: - from .query import QueryException, locate_qualified_function +def _locate_functions_or_prompt(qualified_names: List[str], prompt: bool = True) -> Iterable[Callable[..., Any]]: + from .query import locate_qualified_function, QueryException from .stats import is_data_provider # if not connected to a terminal, can't prompt @@ -475,9 +457,9 @@ def _locate_functions_or_prompt(qualified_names: list[str], *, prompt: bool = Tr # user to select a 'data provider' like function try: mod = importlib.import_module(qualname) - except Exception as ie: + except Exception: eprint(f"During fallback, importing '{qualname}' as module failed") - raise qr_err from ie + raise qr_err # find data providers in this module data_providers = [f for _, f in inspect.getmembers(mod, inspect.isfunction) if is_data_provider(f)] @@ -506,9 +488,8 @@ def _locate_functions_or_prompt(qualified_names: list[str], *, prompt: bool = Tr def _warn_exceptions(exc: Exception) -> None: - from my.core import make_logger - - logger = make_logger('CLI', level='warning') + from my.core.common import LazyLogger + logger = LazyLogger('CLI', level='warning') logger.exception(f'hpi query: {exc}') @@ -519,28 +500,26 @@ def query_hpi_functions( *, output: str = 'json', stream: bool = False, - qualified_names: list[str], - order_key: str | None, - order_by_value_type: type | None, + qualified_names: List[str], + order_key: Optional[str], + order_by_value_type: Optional[Type], after: Any, before: Any, within: Any, reverse: bool = False, - limit: int | None, + limit: Optional[int], drop_unsorted: bool, wrap_unsorted: bool, warn_exceptions: bool, raise_exceptions: bool, drop_exceptions: bool, ) -> None: - from .query_range import RangeTuple, select_range + from .query_range import select_range, RangeTuple + import my.core.error as err # chain list of functions from user, in the order they wrote them on the CLI input_src = chain(*(f() for f in _locate_functions_or_prompt(qualified_names))) - # NOTE: if passing just one function to this which returns a single namedtuple/dataclass, - # using both --order-key and --order-type will often be faster as it does not need to - # duplicate the iterator in memory, or try to find the --order-type type on each object before sorting res = select_range( input_src, order_key=order_key, @@ -553,8 +532,7 @@ def query_hpi_functions( warn_exceptions=warn_exceptions, warn_func=_warn_exceptions, raise_exceptions=raise_exceptions, - drop_exceptions=drop_exceptions, - ) + drop_exceptions=drop_exceptions) if output == 'json': from .serialize import dumps @@ -588,7 +566,7 @@ def query_hpi_functions( # can ignore the mypy warning here, locations_to_gpx yields any errors # if you didnt pass it something that matches the LocationProtocol - for exc in locations_to_gpx(res, sys.stdout): # type: ignore[arg-type] + for exc in locations_to_gpx(res, sys.stdout): # type: ignore[arg-type] if warn_exceptions: _warn_exceptions(exc) elif raise_exceptions: @@ -601,11 +579,10 @@ def query_hpi_functions( # output == 'repl' eprint(f"\nInteract with the results by using the {click.style('res', fg='green')} variable\n") try: - import IPython # type: ignore[import,unused-ignore] + import IPython # type: ignore[import] except ModuleNotFoundError: eprint("'repl' typically uses ipython, install it with 'python3 -m pip install ipython'. falling back to stdlib...") import code - code.interact(local=locals()) else: IPython.embed() @@ -613,7 +590,7 @@ def query_hpi_functions( @click.group() @click.option("--debug", is_flag=True, default=False, help="Show debug logs") -def main(*, debug: bool) -> None: +def main(debug: bool) -> None: ''' Human Programming Interface @@ -639,19 +616,20 @@ def main(*, debug: bool) -> None: # to run things at the end (would need to use a callback or pass context) # https://click.palletsprojects.com/en/7.x/commands/#nested-handling-and-contexts - tdir = Path(tempfile.gettempdir()) / 'hpi_temp_dir' - tdir.mkdir(exist_ok=True) + tdir: str = os.path.join(tempfile.gettempdir(), 'hpi_temp_dir') + if not os.path.exists(tdir): + os.makedirs(tdir) os.chdir(tdir) @functools.lru_cache(maxsize=1) -def _all_mod_names() -> list[str]: +def _all_mod_names() -> List[str]: """Should include all modules, in case user is trying to diagnose issues""" # sort this, so that the order doesn't change while tabbing through return sorted([m.name for m in modules()]) -def _module_autocomplete(ctx: click.Context, args: Sequence[str], incomplete: str) -> list[str]: +def _module_autocomplete(ctx: click.Context, args: Sequence[str], incomplete: str) -> List[str]: return [m for m in _all_mod_names() if m.startswith(incomplete)] @@ -661,7 +639,7 @@ def _module_autocomplete(ctx: click.Context, args: Sequence[str], incomplete: st @click.option('-q', '--quick', is_flag=True, help='Only run partial checks (first 100 items)') @click.option('-S', '--skip-config-check', 'skip_conf', is_flag=True, help='Skip configuration check') @click.argument('MODULE', nargs=-1, required=False, shell_complete=_module_autocomplete) -def doctor_cmd(*, verbose: bool, list_all: bool, quick: bool, skip_conf: bool, module: Sequence[str]) -> None: +def doctor_cmd(verbose: bool, list_all: bool, quick: bool, skip_conf: bool, module: Sequence[str]) -> None: ''' Run various checks @@ -695,7 +673,7 @@ def config_create_cmd() -> None: @main.command(name='modules', short_help='list available modules') @click.option('--all', 'list_all', is_flag=True, help='List all modules, including disabled') -def module_cmd(*, list_all: bool) -> None: +def module_cmd(list_all: bool) -> None: '''List available modules''' list_modules(list_all=list_all) @@ -708,7 +686,7 @@ def module_grp() -> None: @module_grp.command(name='requires', short_help='print module reqs') @click.argument('MODULES', shell_complete=_module_autocomplete, nargs=-1, required=True) -def module_requires_cmd(*, modules: Sequence[str]) -> None: +def module_requires_cmd(modules: Sequence[str]) -> None: ''' Print MODULES requirements @@ -725,7 +703,7 @@ def module_requires_cmd(*, modules: Sequence[str]) -> None: is_flag=True, help='Bypass PEP 668 and install dependencies into the system-wide python package directory.') @click.argument('MODULES', shell_complete=_module_autocomplete, nargs=-1, required=True) -def module_install_cmd(*, user: bool, parallel: bool, break_system_packages: bool, modules: Sequence[str]) -> None: +def module_install_cmd(user: bool, parallel: bool, break_system_packages: bool, modules: Sequence[str]) -> None: ''' Install dependencies for modules using pip @@ -806,18 +784,17 @@ def module_install_cmd(*, user: bool, parallel: bool, break_system_packages: boo help='ignore any errors returned as objects from the functions') @click.argument('FUNCTION_NAME', nargs=-1, required=True, shell_complete=_module_autocomplete) def query_cmd( - *, function_name: Sequence[str], output: str, stream: bool, - order_key: str | None, - order_type: str | None, - after: str | None, - before: str | None, - within: str | None, - recent: str | None, + order_key: Optional[str], + order_type: Optional[str], + after: Optional[str], + before: Optional[str], + within: Optional[str], + recent: Optional[str], reverse: bool, - limit: int | None, + limit: Optional[int], drop_unsorted: bool, wrap_unsorted: bool, warn_exceptions: bool, @@ -851,9 +828,9 @@ def query_cmd( hpi query --order-type datetime --after '2016-01-01' --before '2019-01-01' my.reddit.all.comments ''' - from datetime import date, datetime + from datetime import datetime, date - chosen_order_type: type | None + chosen_order_type: Optional[Type] if order_type == "datetime": chosen_order_type = datetime elif order_type == "date": @@ -889,8 +866,7 @@ def query_cmd( wrap_unsorted=wrap_unsorted, warn_exceptions=warn_exceptions, raise_exceptions=raise_exceptions, - drop_exceptions=drop_exceptions, - ) + drop_exceptions=drop_exceptions) except QueryException as qe: eprint(str(qe)) sys.exit(1) @@ -905,7 +881,6 @@ def query_cmd( def test_requires() -> None: from click.testing import CliRunner - result = CliRunner().invoke(main, ['module', 'requires', 'my.github.ghexport', 'my.browser.export']) assert result.exit_code == 0 assert "github.com/karlicoss/ghexport" in result.output diff --git a/my/core/_cpu_pool.py b/my/core/_cpu_pool.py index 6b107a7..5ac66de 100644 --- a/my/core/_cpu_pool.py +++ b/my/core/_cpu_pool.py @@ -10,18 +10,16 @@ how many cores we want to dedicate to the DAL. Enabled by the env variable, specifying how many cores to dedicate e.g. "HPI_CPU_POOL=4 hpi query ..." """ - -from __future__ import annotations - -import os from concurrent.futures import ProcessPoolExecutor -from typing import cast +import os +from typing import cast, Optional + _NOT_SET = cast(ProcessPoolExecutor, object()) -_INSTANCE: ProcessPoolExecutor | None = _NOT_SET +_INSTANCE: Optional[ProcessPoolExecutor] = _NOT_SET -def get_cpu_pool() -> ProcessPoolExecutor | None: +def get_cpu_pool() -> Optional[ProcessPoolExecutor]: global _INSTANCE if _INSTANCE is _NOT_SET: use_cpu_pool = os.environ.get('HPI_CPU_POOL') diff --git a/my/core/_deprecated/dataset.py b/my/core/_deprecated/dataset.py deleted file mode 100644 index 9cca2fd..0000000 --- a/my/core/_deprecated/dataset.py +++ /dev/null @@ -1,12 +0,0 @@ -from ..common import PathIsh -from ..sqlite import sqlite_connect_immutable - - -def connect_readonly(db: PathIsh): - import dataset # type: ignore - - # see https://github.com/pudo/dataset/issues/136#issuecomment-128693122 - # todo not sure if mode=ro has any benefit, but it doesn't work on read-only filesystems - # maybe it should autodetect readonly filesystems and apply this? not sure - creator = lambda: sqlite_connect_immutable(db) - return dataset.connect('sqlite:///', engine_kwargs={'creator': creator}) diff --git a/my/core/_deprecated/kompress.py b/my/core/_deprecated/kompress.py index c3f333f..cd1bd9d 100644 --- a/my/core/_deprecated/kompress.py +++ b/my/core/_deprecated/kompress.py @@ -1,17 +1,16 @@ """ Various helpers for compression """ - # fmt: off from __future__ import annotations -import io -import pathlib -from collections.abc import Iterator, Sequence from datetime import datetime from functools import total_ordering +import io +import pathlib from pathlib import Path -from typing import IO, Union +import sys +from typing import Union, IO, Sequence, Any, Iterator PathIsh = Union[Path, str] @@ -28,11 +27,11 @@ class Ext: def is_compressed(p: Path) -> bool: # todo kinda lame way for now.. use mime ideally? # should cooperate with kompress.kopen? - return any(p.name.endswith(ext) for ext in [Ext.xz, Ext.zip, Ext.lz4, Ext.zstd, Ext.zst, Ext.targz]) + return any(p.name.endswith(ext) for ext in {Ext.xz, Ext.zip, Ext.lz4, Ext.zstd, Ext.zst, Ext.targz}) def _zstd_open(path: Path, *args, **kwargs) -> IO: - import zstandard as zstd # type: ignore + import zstandard as zstd # type: ignore fh = path.open('rb') dctx = zstd.ZstdDecompressor() reader = dctx.stream_reader(fh) @@ -86,9 +85,9 @@ def kopen(path: PathIsh, *args, mode: str='rt', **kwargs) -> IO: # todo 'expected "BinaryIO"'?? return io.TextIOWrapper(ifile, encoding=encoding) elif name.endswith(Ext.lz4): - import lz4.frame # type: ignore + import lz4.frame # type: ignore return lz4.frame.open(str(pp), mode, *args, **kwargs) - elif name.endswith(Ext.zstd) or name.endswith(Ext.zst): # noqa: PIE810 + elif name.endswith(Ext.zstd) or name.endswith(Ext.zst): kwargs['mode'] = mode return _zstd_open(pp, *args, **kwargs) elif name.endswith(Ext.targz): @@ -102,8 +101,8 @@ def kopen(path: PathIsh, *args, mode: str='rt', **kwargs) -> IO: return pp.open(mode, *args, **kwargs) -import os import typing +import os if typing.TYPE_CHECKING: # otherwise mypy can't figure out that BasePath is a type alias.. @@ -121,7 +120,7 @@ class CPath(BasePath): Path only has _accessor and _closed slots, so can't directly set .open method _accessor.open has to return file descriptor, doesn't work for compressed stuff. """ - def open(self, *args, **kwargs): # noqa: ARG002 + def open(self, *args, **kwargs): kopen_kwargs = {} mode = kwargs.get('mode') if mode is not None: @@ -142,16 +141,20 @@ open = kopen # TODO deprecate def kexists(path: PathIsh, subpath: str) -> bool: try: kopen(path, subpath) + return True except Exception: return False - else: - return True import zipfile - -# meh... zipfile.Path is not available on 3.7 -zipfile_Path = zipfile.Path +if sys.version_info[:2] >= (3, 8): + # meh... zipfile.Path is not available on 3.7 + zipfile_Path = zipfile.Path +else: + if typing.TYPE_CHECKING: + zipfile_Path = Any + else: + zipfile_Path = object @total_ordering @@ -159,7 +162,7 @@ class ZipPath(zipfile_Path): # NOTE: is_dir/is_file might not behave as expected, the base class checks it only based on the slash in path # seems that root/at are not exposed in the docs, so might be an implementation detail - root: zipfile.ZipFile # type: ignore[assignment] + root: zipfile.ZipFile at: str @property @@ -188,14 +191,14 @@ class ZipPath(zipfile_Path): # note: seems that zip always uses forward slash, regardless OS? return zipfile_Path(self.root, self.at + '/') - def rglob(self, glob: str) -> Iterator[ZipPath]: + def rglob(self, glob: str) -> Sequence[ZipPath]: # note: not 100% sure about the correctness, but seem fine? # Path.match() matches from the right, so need to rpaths = [p for p in self.root.namelist() if p.startswith(self.at)] rpaths = [p for p in rpaths if Path(p).match(glob)] - return (ZipPath(self.root, p) for p in rpaths) + return [ZipPath(self.root, p) for p in rpaths] - def relative_to(self, other: ZipPath) -> Path: # type: ignore[override, unused-ignore] + def relative_to(self, other: ZipPath) -> Path: assert self.filepath == other.filepath, (self.filepath, other.filepath) return self.subpath.relative_to(other.subpath) @@ -211,7 +214,7 @@ class ZipPath(zipfile_Path): def iterdir(self) -> Iterator[ZipPath]: for s in self._as_dir().iterdir(): - yield ZipPath(s.root, s.at) + yield ZipPath(s.root, s.at) # type: ignore[attr-defined] @property def stem(self) -> str: @@ -240,7 +243,7 @@ class ZipPath(zipfile_Path): # see https://en.wikipedia.org/wiki/ZIP_(file_format)#Structure dt = datetime(*self.root.getinfo(self.at).date_time) ts = int(dt.timestamp()) - params = dict( # noqa: C408 + params = dict( st_mode=0, st_ino=0, st_dev=0, diff --git a/my/core/cachew.py b/my/core/cachew.py index 8ce2f2b..7dd62d2 100644 --- a/my/core/cachew.py +++ b/my/core/cachew.py @@ -1,27 +1,13 @@ -from __future__ import annotations +from .common import assert_subpackage; assert_subpackage(__name__) -from .internal import assert_subpackage - -assert_subpackage(__name__) - -import logging -import sys -from collections.abc import Iterator from contextlib import contextmanager +import logging from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - Callable, - TypeVar, - Union, - cast, - overload, -) +import sys +from typing import Optional, Iterator, cast, TYPE_CHECKING, TypeVar, Callable, overload, Union, Any, Type +import warnings -import appdirs # type: ignore[import-untyped] - -from . import warnings +import appdirs PathIsh = Union[str, Path] # avoid circular import from .common @@ -61,12 +47,12 @@ def _appdirs_cache_dir() -> Path: _CACHE_DIR_NONE_HACK = Path('/tmp/hpi/cachew_none_hack') -def cache_dir(suffix: PathIsh | None = None) -> Path: +def cache_dir(suffix: Optional[PathIsh] = None) -> Path: from . import core_config as CC cdir_ = CC.config.get_cache_dir() - sp: Path | None = None + sp: Optional[Path] = None if suffix is not None: sp = Path(suffix) # guess if you do need absolute, better path it directly instead of as suffix? @@ -119,7 +105,7 @@ def _mcachew_impl(cache_path=_cache_path_dflt, **kwargs): try: import cachew except ModuleNotFoundError: - warnings.high('cachew library not found. You might want to install it to speed things up. See https://github.com/karlicoss/cachew') + warnings.warn('cachew library not found. You might want to install it to speed things up. See https://github.com/karlicoss/cachew') return lambda orig_func: orig_func else: kwargs['cache_path'] = cache_path @@ -136,7 +122,7 @@ if TYPE_CHECKING: CC = Callable[P, R] # need to give it a name, if inlined into bound=, mypy runs in a bug PathProvider = Union[PathIsh, Callable[P, PathIsh]] # NOTE: in cachew, HashFunction type returns str - # however in practice, cachew always calls str for its result + # however in practice, cachew alwasy calls str for its result # so perhaps better to switch it to Any in cachew as well HashFunction = Callable[P, Any] @@ -145,19 +131,21 @@ if TYPE_CHECKING: # we need two versions due to @doublewrap # this is when we just annotate as @cachew without any args @overload # type: ignore[no-overload-impl] - def mcachew(fun: F) -> F: ... + def mcachew(fun: F) -> F: + ... @overload def mcachew( - cache_path: PathProvider | None = ..., + cache_path: Optional[PathProvider] = ..., *, force_file: bool = ..., - cls: type | None = ..., + cls: Optional[Type] = ..., depends_on: HashFunction = ..., - logger: logging.Logger | None = ..., + logger: Optional[logging.Logger] = ..., chunk_by: int = ..., - synthetic_key: str | None = ..., - ) -> Callable[[F], F]: ... + synthetic_key: Optional[str] = ..., + ) -> Callable[[F], F]: + ... else: mcachew = _mcachew_impl diff --git a/my/core/cfg.py b/my/core/cfg.py index 9851443..0b59537 100644 --- a/my/core/cfg.py +++ b/my/core/cfg.py @@ -1,42 +1,34 @@ from __future__ import annotations -import importlib -import re -import sys -from collections.abc import Iterator -from contextlib import ExitStack, contextmanager -from typing import Any, Callable, TypeVar +from typing import TypeVar, Type, Callable, Dict, Any -Attrs = dict[str, Any] +Attrs = Dict[str, Any] C = TypeVar('C') - # todo not sure about it, could be overthinking... # but short enough to change later # TODO document why it's necessary? -def make_config(cls: type[C], migration: Callable[[Attrs], Attrs] = lambda x: x) -> C: +def make_config(cls: Type[C], migration: Callable[[Attrs], Attrs]=lambda x: x) -> C: user_config = cls.__base__ old_props = { # NOTE: deliberately use gettatr to 'force' class properties here - k: getattr(user_config, k) - for k in vars(user_config) + k: getattr(user_config, k) for k in vars(user_config) } new_props = migration(old_props) from dataclasses import fields - params = { k: v for k, v in new_props.items() - if k in {f.name for f in fields(cls)} # type: ignore[arg-type] # see https://github.com/python/typing_extensions/issues/115 + if k in {f.name for f in fields(cls)} # type: ignore[arg-type] # see https://github.com/python/typing_extensions/issues/115 } # todo maybe return type here? return cls(**params) F = TypeVar('F') - - +from contextlib import contextmanager +from typing import Iterator @contextmanager def _override_config(config: F) -> Iterator[F]: ''' @@ -54,9 +46,10 @@ def _override_config(config: F) -> Iterator[F]: delattr(config, k) +import importlib +import sys +from typing import Optional ModuleRegex = str - - @contextmanager def _reload_modules(modules: ModuleRegex) -> Iterator[None]: # need to use list here, otherwise reordering with set might mess things up @@ -86,15 +79,16 @@ def _reload_modules(modules: ModuleRegex) -> Iterator[None]: sys.modules.pop(m, None) +from contextlib import ExitStack +import re @contextmanager -def tmp_config(*, modules: ModuleRegex | None = None, config=None): +def tmp_config(*, modules: Optional[ModuleRegex]=None, config=None): if modules is None: assert config is None if modules is not None: assert config is not None import my.config - with ExitStack() as module_reload_stack, _override_config(my.config) as new_config: if config is not None: overrides = {k: v for k, v in vars(config).items() if not k.startswith('__')} @@ -109,7 +103,6 @@ def tmp_config(*, modules: ModuleRegex | None = None, config=None): def test_tmp_config() -> None: class extra: data_path = '/path/to/data' - with tmp_config() as c: assert c.google != 'whatever' assert not hasattr(c, 'extra') diff --git a/my/core/common.py b/my/core/common.py index aa994ea..85b9386 100644 --- a/my/core/common.py +++ b/my/core/common.py @@ -1,43 +1,199 @@ -from __future__ import annotations - -import os -from collections.abc import Iterable, Sequence from glob import glob as do_glob from pathlib import Path +from datetime import datetime +import functools +from contextlib import contextmanager +import os +import sys +import types from typing import ( - TYPE_CHECKING, + Any, Callable, - Generic, + Dict, + Iterable, + Iterator, + List, + NoReturn, + Optional, + Sequence, + TYPE_CHECKING, + Tuple, TypeVar, Union, + cast, + get_args, + get_type_hints, + get_origin, ) - -from . import compat, warnings +import warnings +from . import warnings as core_warnings # some helper functions -# TODO start deprecating this? soon we'd be able to use Path | str syntax which is shorter and more explicit PathIsh = Union[Path, str] +# TODO only used in tests? not sure if useful at all. +def import_file(p: PathIsh, name: Optional[str] = None) -> types.ModuleType: + p = Path(p) + if name is None: + name = p.stem + import importlib.util + spec = importlib.util.spec_from_file_location(name, p) + assert spec is not None, f"Fatal error; Could not create module spec from {name} {p}" + foo = importlib.util.module_from_spec(spec) + loader = spec.loader; assert loader is not None + loader.exec_module(foo) + return foo + + +def import_from(path: PathIsh, name: str) -> types.ModuleType: + path = str(path) + try: + sys.path.append(path) + import importlib + return importlib.import_module(name) + finally: + sys.path.remove(path) + + +def import_dir(path: PathIsh, extra: str='') -> types.ModuleType: + p = Path(path) + if p.parts[0] == '~': + p = p.expanduser() # TODO eh. not sure about this.. + return import_from(p.parent, p.name + extra) + + +T = TypeVar('T') +K = TypeVar('K') +V = TypeVar('V') + +# TODO deprecate? more_itertools.one should be used +def the(l: Iterable[T]) -> T: + it = iter(l) + try: + first = next(it) + except StopIteration: + raise RuntimeError('Empty iterator?') + assert all(e == first for e in it) + return first + + +# TODO more_itertools.bucket? +def group_by_key(l: Iterable[T], key: Callable[[T], K]) -> Dict[K, List[T]]: + res: Dict[K, List[T]] = {} + for i in l: + kk = key(i) + lst = res.get(kk, []) + lst.append(i) + res[kk] = lst + return res + + +def _identity(v: T) -> V: # type: ignore[type-var] + return cast(V, v) + + +# ugh. nothing in more_itertools? +def ensure_unique( + it: Iterable[T], + *, + key: Callable[[T], K], + value: Callable[[T], V]=_identity, + key2value: Optional[Dict[K, V]]=None +) -> Iterable[T]: + if key2value is None: + key2value = {} + for i in it: + k = key(i) + v = value(i) + pv = key2value.get(k, None) + if pv is not None: + raise RuntimeError(f"Duplicate key: {k}. Previous value: {pv}, new value: {v}") + key2value[k] = v + yield i + + +def test_ensure_unique() -> None: + import pytest + assert list(ensure_unique([1, 2, 3], key=lambda i: i)) == [1, 2, 3] + + dups = [1, 2, 1, 4] + # this works because it's lazy + it = ensure_unique(dups, key=lambda i: i) + + # but forcing throws + with pytest.raises(RuntimeError, match='Duplicate key'): + list(it) + + # hacky way to force distinct objects? + list(ensure_unique(dups, key=lambda i: object())) + + +def make_dict( + it: Iterable[T], + *, + key: Callable[[T], K], + value: Callable[[T], V]=_identity +) -> Dict[K, V]: + res: Dict[K, V] = {} + uniques = ensure_unique(it, key=key, value=value, key2value=res) + for _ in uniques: + pass # force the iterator + return res + + +def test_make_dict() -> None: + it = range(5) + d = make_dict(it, key=lambda i: i, value=lambda i: i % 2) + assert d == {0: 0, 1: 1, 2: 0, 3: 1, 4: 0} + + # check type inference + d2: Dict[str, int ] = make_dict(it, key=lambda i: str(i)) + d3: Dict[str, bool] = make_dict(it, key=lambda i: str(i), value=lambda i: i % 2 == 0) + + +# https://stackoverflow.com/a/12377059/706389 +def listify(fn=None, wrapper=list): + """ + Wraps a function's return value in wrapper (e.g. list) + Useful when an algorithm can be expressed more cleanly as a generator + """ + def listify_return(fn): + @functools.wraps(fn) + def listify_helper(*args, **kw): + return wrapper(fn(*args, **kw)) + return listify_helper + if fn is None: + return listify_return + return listify_return(fn) + + +# todo use in bluemaestro +# def dictify(fn=None, key=None, value=None): +# def md(it): +# return make_dict(it, key=key, value=value) +# return listify(fn=fn, wrapper=md) + + +from .logging import setup_logger, LazyLogger + + Paths = Union[Sequence[PathIsh], PathIsh] DEFAULT_GLOB = '*' - - def get_files( - pp: Paths, - glob: str = DEFAULT_GLOB, - *, - sort: bool = True, - guess_compression: bool = True, -) -> tuple[Path, ...]: + pp: Paths, + glob: str=DEFAULT_GLOB, + sort: bool=True, + guess_compression: bool=True, +) -> Tuple[Path, ...]: """ Helper function to avoid boilerplate. Tuple as return type is a bit friendlier for hashing/caching, so hopefully makes sense """ # TODO FIXME mm, some wrapper to assert iterator isn't empty? - sources: list[Path] + sources: List[Path] if isinstance(pp, Path): sources = [pp] elif isinstance(pp, str): @@ -54,7 +210,7 @@ def get_files( # TODO ugh. very flaky... -3 because [, get_files(), ] return traceback.extract_stack()[-3].filename - paths: list[Path] = [] + paths: List[Path] = [] for src in sources: if src.parts[0] == '~': src = src.expanduser() @@ -62,9 +218,9 @@ def get_files( gs = str(src) if '*' in gs: if glob != DEFAULT_GLOB: - warnings.medium(f"{caller()}: treating {gs} as glob path. Explicit glob={glob} argument is ignored!") - paths.extend(map(Path, do_glob(gs))) # noqa: PTH207 - elif os.path.isdir(str(src)): # noqa: PTH112 + warnings.warn(f"{caller()}: treating {gs} as glob path. Explicit glob={glob} argument is ignored!") + paths.extend(map(Path, do_glob(gs))) + elif os.path.isdir(str(src)): # NOTE: we're using os.path here on purpose instead of src.is_dir # the reason is is_dir for archives might return True and then # this clause would try globbing insize the archives @@ -80,11 +236,11 @@ def get_files( paths.append(src) if sort: - paths = sorted(paths) + paths = list(sorted(paths)) if len(paths) == 0: # todo make it conditionally defensive based on some global settings - warnings.high(f''' + core_warnings.high(f''' {caller()}: no paths were matched against {pp}. This might result in missing data. Likely, the directory you passed is empty. '''.strip()) # traceback is useful to figure out what config caused it? @@ -93,7 +249,7 @@ def get_files( traceback.print_stack() if guess_compression: - from .kompress import CPath, ZipPath, is_compressed + from .kompress import CPath, is_compressed, ZipPath # NOTE: wrap is just for backwards compat with vendorized kompress # with kompress library, only is_compressed check and Cpath should be enough @@ -110,33 +266,44 @@ def get_files( return tuple(paths) +@functools.lru_cache(1) +def _magic(): + import magic # type: ignore + return magic.Magic(mime=True) + + +# TODO could reuse in pdf module? +import mimetypes # todo do I need init()? +# todo wtf? fastermime thinks it's mime is application/json even if the extension is xz?? +# whereas magic detects correctly: application/x-zstd and application/x-xz +def fastermime(path: PathIsh) -> str: + paths = str(path) + # mimetypes is faster + (mime, _) = mimetypes.guess_type(paths) + if mime is not None: + return mime + # magic is slower but returns more stuff + # TODO Result type?; it's kinda racey, but perhaps better to let the caller decide? + return _magic().from_file(paths) + + +Json = Dict[str, Any] + + +from typing import TypeVar, Callable, Generic + +_C = TypeVar('_C') _R = TypeVar('_R') - # https://stackoverflow.com/a/5192374/706389 -# NOTE: it was added to stdlib in 3.9 and then deprecated in 3.11 -# seems that the suggested solution is to use custom decorator? class classproperty(Generic[_R]): - def __init__(self, f: Callable[..., _R]) -> None: + def __init__(self, f: Callable[[_C], _R]) -> None: self.f = f - def __get__(self, obj, cls) -> _R: + def __get__(self, obj: None, cls: _C) -> _R: return self.f(cls) -def test_classproperty() -> None: - from .compat import assert_type - - class C: - @classproperty - def prop(cls) -> str: - return 'hello' - - res = C.prop - assert_type(res, str) - assert res == 'hello' - - # hmm, this doesn't really work with mypy well.. # https://github.com/python/mypy/issues/6244 # class staticproperty(Generic[_R]): @@ -146,117 +313,394 @@ def test_classproperty() -> None: # def __get__(self) -> _R: # return self.f() +# TODO deprecate in favor of datetime_aware +tzdatetime = datetime + + +# TODO doctests? +def isoparse(s: str) -> tzdatetime: + """ + Parses timestamps formatted like 2020-05-01T10:32:02.925961Z + """ + # TODO could use dateutil? but it's quite slow as far as I remember.. + # TODO support non-utc.. somehow? + assert s.endswith('Z'), s + s = s[:-1] + '+00:00' + return datetime.fromisoformat(s) + import re - - # https://stackoverflow.com/a/295466/706389 def get_valid_filename(s: str) -> str: s = str(s).strip().replace(' ', '_') return re.sub(r'(?u)[^-\w.]', '', s) -# TODO deprecate and suggest to use one from my.core directly? not sure -from .utils.itertools import unique_everseen # noqa: F401 +from typing import Generic, Sized, Callable -### legacy imports, keeping them here for backwards compatibility -## hiding behind TYPE_CHECKING so it works in runtime -## in principle, warnings.deprecated decorator should cooperate with mypy, but doesn't look like it works atm? -## perhaps it doesn't work when it's used from typing_extensions -if not TYPE_CHECKING: - from .compat import deprecated +# X = TypeVar('X') +def _warn_iterator(it, f: Any=None): + emitted = False + for i in it: + yield i + emitted = True + if not emitted: + warnings.warn(f"Function {f} didn't emit any data, make sure your config paths are correct") - @deprecated('use my.core.compat.assert_never instead') - def assert_never(*args, **kwargs): - return compat.assert_never(*args, **kwargs) - @deprecated('use my.core.compat.fromisoformat instead') - def isoparse(*args, **kwargs): - return compat.fromisoformat(*args, **kwargs) +# TODO ugh, so I want to express something like: +# X = TypeVar('X') +# C = TypeVar('C', bound=Iterable[X]) +# _warn_iterable(it: C) -> C +# but apparently I can't??? ugh. +# https://github.com/python/typing/issues/548 +# I guess for now overloads are fine... - @deprecated('use more_itertools.one instead') - def the(*args, **kwargs): - import more_itertools +from typing import overload +X = TypeVar('X') +@overload +def _warn_iterable(it: List[X] , f: Any=None) -> List[X] : ... +@overload +def _warn_iterable(it: Iterable[X], f: Any=None) -> Iterable[X]: ... +def _warn_iterable(it, f=None): + if isinstance(it, Sized): + sz = len(it) + if sz == 0: + warnings.warn(f"Function {f} returned empty container, make sure your config paths are correct") + return it + else: + return _warn_iterator(it, f=f) - return more_itertools.one(*args, **kwargs) - @deprecated('use functools.cached_property instead') - def cproperty(*args, **kwargs): - import functools +# ok, this seems to work... +# https://github.com/python/mypy/issues/1927#issue-167100413 +FL = TypeVar('FL', bound=Callable[..., List]) +FI = TypeVar('FI', bound=Callable[..., Iterable]) - return functools.cached_property(*args, **kwargs) +@overload +def warn_if_empty(f: FL) -> FL: ... +@overload +def warn_if_empty(f: FI) -> FI: ... - @deprecated('use more_itertools.bucket instead') - def group_by_key(l, key): - res = {} - for i in l: - kk = key(i) - lst = res.get(kk, []) - lst.append(i) - res[kk] = lst - return res - @deprecated('use my.core.utils.itertools.make_dict instead') - def make_dict(*args, **kwargs): - from .utils import itertools as UI +def warn_if_empty(f): + from functools import wraps - return UI.make_dict(*args, **kwargs) + @wraps(f) + def wrapped(*args, **kwargs): + res = f(*args, **kwargs) + return _warn_iterable(res, f=f) + return wrapped - @deprecated('use my.core.utils.itertools.listify instead') - def listify(*args, **kwargs): - from .utils import itertools as UI - return UI.listify(*args, **kwargs) +# global state that turns on/off quick stats +# can use the 'quick_stats' contextmanager +# to enable/disable this in cli so that module 'stats' +# functions don't have to implement custom 'quick' logic +QUICK_STATS = False - @deprecated('use my.core.warn_if_empty instead') - def warn_if_empty(*args, **kwargs): - from .utils import itertools as UI - return UI.listify(*args, **kwargs) +# in case user wants to use the stats functions/quick option +# elsewhere -- can use this decorator instead of editing +# the global state directly +@contextmanager +def quick_stats(): + global QUICK_STATS + prev = QUICK_STATS + try: + QUICK_STATS = True + yield + finally: + QUICK_STATS = prev - @deprecated('use my.core.stat instead') - def stat(*args, **kwargs): - from . import stats - return stats.stat(*args, **kwargs) +C = TypeVar('C') +Stats = Dict[str, Any] +StatsFun = Callable[[], Stats] +# todo not sure about return type... +def stat( + func: Union[Callable[[], Iterable[C]], Iterable[C]], + *, + quick: bool = False, + name: Optional[str] = None, +) -> Stats: + if callable(func): + fr = func() + if hasattr(fr, '__enter__') and hasattr(fr, '__exit__'): + # context managers has Iterable type, but they aren't data providers + # sadly doesn't look like there is a way to tell from typing annotations + return {} + fname = func.__name__ + else: + # meh. means it's just a list.. not sure how to generate a name then + fr = func + fname = f'unnamed_{id(fr)}' + type_name = type(fr).__name__ + if type_name == 'DataFrame': + # dynamic, because pandas is an optional dependency.. + df = cast(Any, fr) # todo ugh, not sure how to annotate properly + res = dict( + dtypes=df.dtypes.to_dict(), + rows=len(df), + ) + else: + res = _stat_iterable(fr, quick=quick) - @deprecated('use my.core.make_logger instead') - def LazyLogger(*args, **kwargs): - from . import logging + stat_name = name if name is not None else fname + return { + stat_name: res, + } - return logging.LazyLogger(*args, **kwargs) - @deprecated('use my.core.types.asdict instead') - def asdict(*args, **kwargs): - from . import types +def _stat_iterable(it: Iterable[C], quick: bool = False) -> Any: + from more_itertools import ilen, take, first - return types.asdict(*args, **kwargs) + # todo not sure if there is something in more_itertools to compute this? + total = 0 + errors = 0 + first_item = None + last_item = None - # todo wrap these in deprecated decorator as well? - # TODO hmm how to deprecate these in runtime? - # tricky cause they are actually classes/types - from typing import Literal # noqa: F401 + def funcit(): + nonlocal errors, first_item, last_item, total + for x in it: + total += 1 + if isinstance(x, Exception): + errors += 1 + else: + last_item = x + if first_item is None: + first_item = x + yield x - from .cachew import mcachew # noqa: F401 + eit = funcit() + count: Any + if quick or QUICK_STATS: + initial = take(100, eit) + count = len(initial) + if first(eit, None) is not None: # todo can actually be none... + # haven't exhausted + count = f'{count}+' + else: + count = ilen(eit) - # this is kinda internal, should just use my.core.logging.setup_logger if necessary - from .logging import setup_logger - from .stats import Stats - from .types import ( - Json, - datetime_aware, - datetime_naive, - ) + res = { + 'count': count, + } - tzdatetime = datetime_aware -else: - from .compat import Never + if total == 0: + # not sure but I guess a good balance? wouldn't want to throw early here? + res['warning'] = 'THE ITERABLE RETURNED NO DATA' - # make these invalid during type check while working in runtime - Stats = Never - tzdatetime = Never - Json = Never - datetime_naive = Never - datetime_aware = Never -### + if errors > 0: + res['errors'] = errors + + def stat_item(item): + if item is None: + return None + if isinstance(item, Path): + return str(item) + return guess_datetime(item) + + if (stat_first := stat_item(first_item)) is not None: + res['first'] = stat_first + + if (stat_last := stat_item(last_item)) is not None: + res['last'] = stat_last + + return res + + +def test_stat_iterable() -> None: + from datetime import datetime, timedelta + from typing import NamedTuple + + dd = datetime.utcfromtimestamp(123) + day = timedelta(days=3) + + X = NamedTuple('X', [('x', int), ('d', datetime)]) + + def it(): + yield RuntimeError('oops!') + for i in range(2): + yield X(x=i, d=dd + day * i) + yield RuntimeError('bad!') + for i in range(3): + yield X(x=i * 10, d=dd + day * (i * 10)) + yield X(x=123, d=dd + day * 50) + + res = _stat_iterable(it()) + assert res['count'] == 1 + 2 + 1 + 3 + 1 + assert res['errors'] == 1 + 1 + assert res['last'] == dd + day * 50 + + +# experimental, not sure about it.. +def guess_datetime(x: Any) -> Optional[datetime]: + # todo hmm implement withoutexception.. + try: + d = asdict(x) + except: # noqa: E722 bare except + return None + for k, v in d.items(): + if isinstance(v, datetime): + return v + return None + +def test_guess_datetime() -> None: + from datetime import datetime + from dataclasses import dataclass + from typing import NamedTuple + + dd = isoparse('2021-02-01T12:34:56Z') + + # ugh.. https://github.com/python/mypy/issues/7281 + A = NamedTuple('A', [('x', int)]) + B = NamedTuple('B', [('x', int), ('created', datetime)]) + + assert guess_datetime(A(x=4)) is None + assert guess_datetime(B(x=4, created=dd)) == dd + + @dataclass + class C: + a: datetime + x: int + assert guess_datetime(C(a=dd, x=435)) == dd + # TODO not sure what to return when multiple datetime fields? + # TODO test @property? + + +def is_namedtuple(thing: Any) -> bool: + # basic check to see if this is namedtuple-like + _asdict = getattr(thing, '_asdict', None) + return (_asdict is not None) and callable(_asdict) + + +def asdict(thing: Any) -> Json: + # todo primitive? + # todo exception? + if isinstance(thing, dict): + return thing + import dataclasses as D + if D.is_dataclass(thing): + return D.asdict(thing) + if is_namedtuple(thing): + return thing._asdict() + raise TypeError(f'Could not convert object {thing} to dict') + + +# for now just serves documentation purposes... but one day might make it statically verifiable where possible? +# TODO e.g. maybe use opaque mypy alias? +datetime_naive = datetime +datetime_aware = datetime + + +def assert_subpackage(name: str) -> None: + # can lead to some unexpected issues if you 'import cachew' which being in my/core directory.. so let's protect against it + # NOTE: if we use overlay, name can be smth like my.origg.my.core.cachew ... + assert name == '__main__' or 'my.core' in name, f'Expected module __name__ ({name}) to be __main__ or start with my.core' + + +from .compat import ParamSpec +_P = ParamSpec('_P') +_T = TypeVar('_T') + +# https://stackoverflow.com/a/10436851/706389 +from concurrent.futures import Future, Executor +class DummyExecutor(Executor): + def __init__(self, max_workers: Optional[int]=1) -> None: + self._shutdown = False + self._max_workers = max_workers + + if TYPE_CHECKING: + if sys.version_info[:2] <= (3, 8): + # 3.8 doesn't support ParamSpec as Callable arg :( + # and any attempt to type results in incompatible supertype.. so whatever + def submit(self, fn, *args, **kwargs): ... + else: + def submit(self, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs) -> Future[_T]: ... + else: + def submit(self, fn, *args, **kwargs): + if self._shutdown: + raise RuntimeError('cannot schedule new futures after shutdown') + + f: Future[Any] = Future() + try: + result = fn(*args, **kwargs) + except KeyboardInterrupt: + raise + except BaseException as e: + f.set_exception(e) + else: + f.set_result(result) + + return f + + def shutdown(self, wait: bool=True, **kwargs) -> None: + self._shutdown = True + + +# see https://hakibenita.com/python-mypy-exhaustive-checking#exhaustiveness-checking +def assert_never(value: NoReturn) -> NoReturn: + assert False, f'Unhandled value: {value} ({type(value).__name__})' + + +def _check_all_hashable(fun): + # TODO ok, take callable? + hints = get_type_hints(fun) + # TODO needs to be defensive like in cachew? + return_type = hints.get('return') + # TODO check if None + origin = get_origin(return_type) # Iterator etc? + (arg,) = get_args(return_type) + # options we wanna handle are simple type on the top level or union + arg_origin = get_origin(arg) + + if sys.version_info[:2] >= (3, 10): + is_uniontype = arg_origin is types.UnionType + else: + is_uniontype = False + + is_union = arg_origin is Union or is_uniontype + if is_union: + to_check = get_args(arg) + else: + to_check = (arg,) + + no_hash = [ + t + for t in to_check + # seems that objects that have not overridden hash have the attribute but it's set to None + if getattr(t, '__hash__', None) is None + ] + assert len(no_hash) == 0, f'Types {no_hash} are not hashable, this will result in significant performance downgrade for unique_everseen' + + +_UET = TypeVar('_UET') +_UEU = TypeVar('_UEU') + + +def unique_everseen( + fun: Callable[[], Iterable[_UET]], + key: Optional[Callable[[_UET], _UEU]] = None, +) -> Iterator[_UET]: + # TODO support normal iterable as well? + import more_itertools + + # NOTE: it has to take original callable, because otherwise we don't have access to generator type annotations + iterable = fun() + + if key is None: + # todo check key return type as well? but it's more likely to be hashable + if os.environ.get('HPI_CHECK_UNIQUE_EVERSEEN') is not None: + _check_all_hashable(fun) + + return more_itertools.unique_everseen(iterable=iterable, key=key) + + +## legacy imports, keeping them here for backwards compatibility +from functools import cached_property as cproperty +from typing import Literal +from .cachew import mcachew +## diff --git a/my/core/compat.py b/my/core/compat.py index 8f719a8..48e194b 100644 --- a/my/core/compat.py +++ b/my/core/compat.py @@ -2,60 +2,57 @@ Contains backwards compatibility helpers for different python versions. If something is relevant to HPI itself, please put it in .hpi_compat instead ''' - -from __future__ import annotations - +import os import sys from typing import TYPE_CHECKING -if sys.version_info[:2] >= (3, 13): - from warnings import deprecated -else: - from typing_extensions import deprecated + +windows = os.name == 'nt' # keeping just for backwards compatibility, used to have compat implementation for 3.6 -if not TYPE_CHECKING: - import sqlite3 +import sqlite3 +def sqlite_backup(*, source: sqlite3.Connection, dest: sqlite3.Connection, **kwargs) -> None: + source.backup(dest, **kwargs) - @deprecated('use .backup method on sqlite3.Connection directly instead') - def sqlite_backup(*, source: sqlite3.Connection, dest: sqlite3.Connection, **kwargs) -> None: - # TODO warn here? - source.backup(dest, **kwargs) - # keeping for runtime backwards compatibility (added in 3.9) - @deprecated('use .removeprefix method on string directly instead') - def removeprefix(text: str, prefix: str) -> str: - return text.removeprefix(prefix) +# can remove after python3.9 (although need to keep the method itself for bwd compat) +def removeprefix(text: str, prefix: str) -> str: + if text.startswith(prefix): + return text[len(prefix):] + return text - @deprecated('use .removesuffix method on string directly instead') - def removesuffix(text: str, suffix: str) -> str: - return text.removesuffix(suffix) - ## - - ## used to have compat function before 3.8 for these, keeping for runtime back compatibility - from functools import cached_property - from typing import Literal, Protocol, TypedDict +## used to have compat function before 3.8 for these +from functools import cached_property +from typing import Literal, Protocol, TypedDict ## if sys.version_info[:2] >= (3, 10): from typing import ParamSpec else: - from typing_extensions import ParamSpec + if TYPE_CHECKING: + from typing_extensions import ParamSpec + else: + from typing import NamedTuple, Any + # erm.. I guess as long as it's not crashing, whatever... + class _ParamSpec: + def __call__(self, args): + class _res: + args = None + kwargs = None + return _res + ParamSpec = _ParamSpec() # bisect_left doesn't have a 'key' parameter (which we use) # till python3.10 if sys.version_info[:2] <= (3, 9): - from typing import Any, Callable, List, Optional, TypeVar # noqa: UP035 - + from typing import List, TypeVar, Any, Optional, Callable X = TypeVar('X') - # copied from python src - # fmt: off - def bisect_left(a: list[Any], x: Any, lo: int=0, hi: int | None=None, *, key: Callable[..., Any] | None=None) -> int: + def bisect_left(a: List[Any], x: Any, lo: int=0, hi: Optional[int]=None, *, key: Optional[Callable[..., Any]]=None) -> int: if lo < 0: raise ValueError('lo must be non-negative') if hi is None: @@ -77,63 +74,5 @@ if sys.version_info[:2] <= (3, 9): else: hi = mid return lo - # fmt: on - else: from bisect import bisect_left - - -from datetime import datetime - -if sys.version_info[:2] >= (3, 11): - fromisoformat = datetime.fromisoformat -else: - # fromisoformat didn't support Z as "utc" before 3.11 - # https://docs.python.org/3/library/datetime.html#datetime.datetime.fromisoformat - - def fromisoformat(date_string: str) -> datetime: - if date_string.endswith('Z'): - date_string = date_string[:-1] + '+00:00' - return datetime.fromisoformat(date_string) - - -def test_fromisoformat() -> None: - from datetime import timezone - - # fmt: off - # feedbin has this format - assert fromisoformat('2020-05-01T10:32:02.925961Z') == datetime( - 2020, 5, 1, 10, 32, 2, 925961, timezone.utc, - ) - - # polar has this format - assert fromisoformat('2018-11-28T22:04:01.304Z') == datetime( - 2018, 11, 28, 22, 4, 1, 304000, timezone.utc, - ) - - # stackexchange, runnerup has this format - assert fromisoformat('2020-11-30T00:53:12Z') == datetime( - 2020, 11, 30, 0, 53, 12, 0, timezone.utc, - ) - # fmt: on - - # arbtt has this format (sometimes less/more than 6 digits in milliseconds) - # TODO doesn't work atm, not sure if really should be supported... - # maybe should have flags for weird formats? - # assert isoparse('2017-07-18T18:59:38.21731Z') == datetime( - # 2017, 7, 18, 18, 59, 38, 217310, timezone.utc, - # ) - - -if sys.version_info[:2] >= (3, 10): - from types import NoneType - from typing import TypeAlias -else: - NoneType = type(None) - from typing_extensions import TypeAlias - - -if sys.version_info[:2] >= (3, 11): - from typing import Never, assert_never, assert_type -else: - from typing_extensions import Never, assert_never, assert_type diff --git a/my/core/core_config.py b/my/core/core_config.py index 3f26c03..e70dc05 100644 --- a/my/core/core_config.py +++ b/my/core/core_config.py @@ -1,22 +1,16 @@ ''' Bindings for the 'core' HPI configuration ''' - -from __future__ import annotations - import re -from collections.abc import Sequence -from dataclasses import dataclass -from pathlib import Path +from typing import Sequence, Optional -from . import warnings +from . import warnings, PathIsh, Path try: from my.config import core as user_config # type: ignore[attr-defined] except Exception as e: try: - from my.config import common as user_config # type: ignore[attr-defined] - + from my.config import common as user_config # type: ignore[attr-defined] warnings.high("'common' config section is deprecated. Please rename it to 'core'.") except Exception as e2: # make it defensive, because it's pretty commonly used and would be annoying if it breaks hpi doctor etc. @@ -27,7 +21,7 @@ except Exception as e: _HPI_CACHE_DIR_DEFAULT = '' - +from dataclasses import dataclass @dataclass class Config(user_config): ''' @@ -38,7 +32,7 @@ class Config(user_config): cache_dir = '/your/custom/cache/path' ''' - cache_dir: Path | str | None = _HPI_CACHE_DIR_DEFAULT + cache_dir: Optional[PathIsh] = _HPI_CACHE_DIR_DEFAULT ''' Base directory for cachew. - if None , means cache is disabled @@ -48,7 +42,7 @@ class Config(user_config): NOTE: you shouldn't use this attribute in HPI modules directly, use Config.get_cache_dir()/cachew.cache_dir() instead ''' - tmp_dir: Path | str | None = None + tmp_dir: Optional[PathIsh] = None ''' Path to a temporary directory. This can be used temporarily while extracting zipfiles etc... @@ -56,36 +50,34 @@ class Config(user_config): - otherwise , use the specified directory as the base temporary directory ''' - enabled_modules: Sequence[str] | None = None + enabled_modules : Optional[Sequence[str]] = None ''' list of regexes/globs - None means 'rely on disabled_modules' ''' - disabled_modules: Sequence[str] | None = None + disabled_modules: Optional[Sequence[str]] = None ''' list of regexes/globs - None means 'rely on enabled_modules' ''' - def get_cache_dir(self) -> Path | None: + def get_cache_dir(self) -> Optional[Path]: cdir = self.cache_dir if cdir is None: return None if cdir == _HPI_CACHE_DIR_DEFAULT: from .cachew import _appdirs_cache_dir - return _appdirs_cache_dir() else: return Path(cdir).expanduser() def get_tmp_dir(self) -> Path: - tdir: Path | str | None = self.tmp_dir + tdir: Optional[PathIsh] = self.tmp_dir tpath: Path # use tempfile if unset if tdir is None: import tempfile - tpath = Path(tempfile.gettempdir()) / 'HPI' else: tpath = Path(tdir) @@ -93,10 +85,10 @@ class Config(user_config): tpath.mkdir(parents=True, exist_ok=True) return tpath - def _is_module_active(self, module: str) -> bool | None: + def _is_module_active(self, module: str) -> Optional[bool]: # None means the config doesn't specify anything # todo might be nice to return the 'reason' too? e.g. which option has matched - def matches(specs: Sequence[str]) -> str | None: + def matches(specs: Sequence[str]) -> Optional[str]: for spec in specs: # not sure because . (packages separate) matches anything, but I guess unlikely to clash if re.match(spec, module): @@ -112,25 +104,22 @@ class Config(user_config): return None else: return False - else: # not None + else: # not None if off is None: return True - else: # not None + else: # not None # fallback onto the 'enable everything', then the user will notice warnings.medium(f"[module]: conflicting regexes '{on}' and '{off}' are set in the config. Please only use one of them.") return True from .cfg import make_config - config = make_config(Config) ### tests start -from collections.abc import Iterator +from typing import Iterator from contextlib import contextmanager as ctx - - @ctx def _reset_config() -> Iterator[Config]: # todo maybe have this decorator for the whole of my.config? @@ -169,5 +158,4 @@ def test_active_modules() -> None: assert cc._is_module_active("my.body.exercise") is True assert len(record_warnings) == 1 - ### tests end diff --git a/my/core/dataset.py b/my/core/dataset.py index 40237a0..31de4f4 100644 --- a/my/core/dataset.py +++ b/my/core/dataset.py @@ -1,5 +1,31 @@ -from . import warnings +from __future__ import annotations +from .common import assert_subpackage; assert_subpackage(__name__) -warnings.high(f"{__name__} is deprecated, please use dataset directly if you need or switch to my.core.sqlite") +from .common import PathIsh +from .sqlite import sqlite_connect_immutable -from ._deprecated.dataset import * +## sadly dataset doesn't have any type definitions +from typing import Iterable, Iterator, Dict, Optional, Any, Protocol +from contextlib import AbstractContextManager + + +# NOTE: may not be true in general, but will be in the vast majority of cases +row_type_T = Dict[str, Any] + + +class TableT(Iterable, Protocol): + def find(self, *, order_by: Optional[str]=None) -> Iterator[row_type_T]: ... + + +class DatabaseT(AbstractContextManager['DatabaseT'], Protocol): + def __getitem__(self, table: str) -> TableT: ... +## + +# TODO wonder if also need to open without WAL.. test this on read-only directory/db file +def connect_readonly(db: PathIsh) -> DatabaseT: + import dataset # type: ignore + # see https://github.com/pudo/dataset/issues/136#issuecomment-128693122 + # todo not sure if mode=ro has any benefit, but it doesn't work on read-only filesystems + # maybe it should autodetect readonly filesystems and apply this? not sure + creator = lambda: sqlite_connect_immutable(db) + return dataset.connect('sqlite:///', engine_kwargs={'creator': creator}) diff --git a/my/core/denylist.py b/my/core/denylist.py index c92f9a0..8c18e06 100644 --- a/my/core/denylist.py +++ b/my/core/denylist.py @@ -5,25 +5,23 @@ A helper module for defining denylists for sources programmatically For docs, see doc/DENYLIST.md """ -from __future__ import annotations - -import functools -import json import sys +import json +import functools from collections import defaultdict -from collections.abc import Iterator, Mapping +from typing import TypeVar, Set, Any, Mapping, Iterator, Dict, List from pathlib import Path -from typing import Any, TypeVar import click from more_itertools import seekable +from my.core.serialize import dumps +from my.core.common import PathIsh +from my.core.warnings import medium -from .serialize import dumps -from .warnings import medium T = TypeVar("T") -DenyMap = Mapping[str, set[Any]] +DenyMap = Mapping[str, Set[Any]] def _default_key_func(obj: T) -> str: @@ -31,9 +29,9 @@ def _default_key_func(obj: T) -> str: class DenyList: - def __init__(self, denylist_file: Path | str) -> None: + def __init__(self, denylist_file: PathIsh): self.file = Path(denylist_file).expanduser().absolute() - self._deny_raw_list: list[dict[str, Any]] = [] + self._deny_raw_list: List[Dict[str, Any]] = [] self._deny_map: DenyMap = defaultdict(set) # deny cli, user can override these @@ -47,7 +45,7 @@ class DenyList: return deny_map: DenyMap = defaultdict(set) - data: list[dict[str, Any]] = json.loads(self.file.read_text()) + data: List[Dict[str, Any]]= json.loads(self.file.read_text()) self._deny_raw_list = data for ignore in data: @@ -98,7 +96,6 @@ class DenyList: def filter( self, itr: Iterator[T], - *, invert: bool = False, ) -> Iterator[T]: denyf = functools.partial(self._allow, deny_map=self.load()) @@ -106,7 +103,7 @@ class DenyList: return filter(lambda x: not denyf(x), itr) return filter(denyf, itr) - def deny(self, key: str, value: Any, *, write: bool = False) -> None: + def deny(self, key: str, value: Any, write: bool = False) -> None: ''' add a key/value pair to the denylist ''' @@ -114,7 +111,7 @@ class DenyList: self._load() self._deny_raw({key: self._stringify_value(value)}, write=write) - def _deny_raw(self, data: dict[str, Any], *, write: bool = False) -> None: + def _deny_raw(self, data: Dict[str, Any], write: bool = False) -> None: self._deny_raw_list.append(data) if write: self.write() @@ -133,7 +130,7 @@ class DenyList: def _deny_cli_remember( self, items: Iterator[T], - mem: dict[str, T], + mem: Dict[str, T], ) -> Iterator[str]: keyf = self._deny_cli_key_func or _default_key_func # i.e., convert each item to a string, and map str -> item @@ -159,8 +156,10 @@ class DenyList: # reset the iterator sit.seek(0) # so we can map the selected string from fzf back to the original objects - memory_map: dict[str, T] = {} - picker = FzfPrompt(executable_path=self.fzf_path, default_options="--no-multi") + memory_map: Dict[str, T] = {} + picker = FzfPrompt( + executable_path=self.fzf_path, default_options="--no-multi" + ) picked_l = picker.prompt( self._deny_cli_remember(itr, memory_map), "--read0", diff --git a/my/core/discovery_pure.py b/my/core/discovery_pure.py index 18a19c4..85b75ab 100644 --- a/my/core/discovery_pure.py +++ b/my/core/discovery_pure.py @@ -10,20 +10,17 @@ This potentially allows it to be: It should be free of external modules, importlib, exec, etc. etc. ''' -from __future__ import annotations - REQUIRES = 'REQUIRES' NOT_HPI_MODULE_VAR = '__NOT_HPI_MODULE__' ### import ast -import logging import os -import re -from collections.abc import Iterable, Sequence +from typing import Optional, Sequence, List, NamedTuple, Iterable, cast, Any from pathlib import Path -from typing import Any, NamedTuple, Optional, cast +import re +import logging ''' None means that requirements weren't defined (different from empty requirements) @@ -33,11 +30,11 @@ Requires = Optional[Sequence[str]] class HPIModule(NamedTuple): name: str - skip_reason: str | None - doc: str | None = None - file: Path | None = None + skip_reason: Optional[str] + doc: Optional[str] = None + file: Optional[Path] = None requires: Requires = None - legacy: str | None = None # contains reason/deprecation warning + legacy: Optional[str] = None # contains reason/deprecation warning def ignored(m: str) -> bool: @@ -58,13 +55,13 @@ def has_stats(src: Path) -> bool: def _has_stats(code: str) -> bool: a: ast.Module = ast.parse(code) for x in a.body: - try: # maybe assign + try: # maybe assign [tg] = cast(Any, x).targets if tg.id == 'stats': return True except: pass - try: # maybe def? + try: # maybe def? name = cast(Any, x).name if name == 'stats': return True @@ -147,7 +144,7 @@ def all_modules() -> Iterable[HPIModule]: def _iter_my_roots() -> Iterable[Path]: import my # doesn't import any code, because of namespace package - paths: list[str] = list(my.__path__) + paths: List[str] = list(my.__path__) if len(paths) == 0: # should probably never happen?, if this code is running, it was imported # because something was added to __path__ to match this name @@ -245,7 +242,7 @@ def test_pure() -> None: src = Path(__file__).read_text() # 'import my' is allowed, but # dont allow anything other HPI modules - assert re.findall('import ' + r'my\.\S+', src, re.MULTILINE) == [] + assert re.findall('import ' + r'my\.\S+', src, re.M) == [] assert 'from ' + 'my' not in src diff --git a/my/core/error.py b/my/core/error.py index b308869..e1737c1 100644 --- a/my/core/error.py +++ b/my/core/error.py @@ -3,25 +3,12 @@ Various error handling helpers See https://beepb00p.xyz/mypy-error-handling.html#kiss for more detail """ -from __future__ import annotations - -import traceback -from collections.abc import Iterable, Iterator -from datetime import datetime from itertools import tee -from typing import ( - Any, - Callable, - Literal, - TypeVar, - Union, - cast, -) +from typing import Union, TypeVar, Iterable, List, Tuple, Type, Optional, Callable, Any, cast, Iterator, Literal -from .types import Json T = TypeVar('T') -E = TypeVar('E', bound=Exception) # TODO make covariant? +E = TypeVar('E', bound=Exception) # TODO make covariant? ResT = Union[T, E] @@ -29,8 +16,7 @@ Res = ResT[T, Exception] ErrorPolicy = Literal["yield", "raise", "drop"] - -def notnone(x: T | None) -> T: +def notnone(x: Optional[T]) -> T: assert x is not None return x @@ -38,8 +24,8 @@ def notnone(x: T | None) -> T: def unwrap(res: Res[T]) -> T: if isinstance(res, Exception): raise res - return res - + else: + return res def drop_exceptions(itr: Iterator[Res[T]]) -> Iterator[T]: """Return non-errors from the iterable""" @@ -57,15 +43,13 @@ def raise_exceptions(itr: Iterable[Res[T]]) -> Iterator[T]: yield o -def warn_exceptions(itr: Iterable[Res[T]], warn_func: Callable[[Exception], None] | None = None) -> Iterator[T]: +def warn_exceptions(itr: Iterable[Res[T]], warn_func: Optional[Callable[[Exception], None]] = None) -> Iterator[T]: # if not provided, use the 'warnings' module if warn_func is None: from my.core.warnings import medium - def _warn_func(e: Exception) -> None: # TODO: print traceback? but user could always --raise-exceptions as well medium(str(e)) - warn_func = _warn_func for o in itr: @@ -80,7 +64,7 @@ def echain(ex: E, cause: Exception) -> E: return ex -def split_errors(l: Iterable[ResT[T, E]], ET: type[E]) -> tuple[Iterable[T], Iterable[E]]: +def split_errors(l: Iterable[ResT[T, E]], ET: Type[E]) -> Tuple[Iterable[T], Iterable[E]]: # TODO would be nice to have ET=Exception default? but it causes some mypy complaints? vit, eit = tee(l) # TODO ugh, not sure if I can reconcile type checking and runtime and convince mypy that ET and E are the same type? @@ -98,9 +82,7 @@ def split_errors(l: Iterable[ResT[T, E]], ET: type[E]) -> tuple[Iterable[T], Ite K = TypeVar('K') - - -def sort_res_by(items: Iterable[Res[T]], key: Callable[[Any], K]) -> list[Res[T]]: +def sort_res_by(items: Iterable[Res[T]], key: Callable[[Any], K]) -> List[Res[T]]: """ Sort a sequence potentially interleaved with errors/entries on which the key can't be computed. The general idea is: the error sticks to the non-error entry that follows it @@ -108,7 +90,7 @@ def sort_res_by(items: Iterable[Res[T]], key: Callable[[Any], K]) -> list[Res[T] group = [] groups = [] for i in items: - k: K | None + k: Optional[K] try: k = key(i) except Exception: # error white computing key? dunno, might be nice to handle... @@ -118,10 +100,10 @@ def sort_res_by(items: Iterable[Res[T]], key: Callable[[Any], K]) -> list[Res[T] groups.append((k, group)) group = [] - results: list[Res[T]] = [] - for _v, grp in sorted(groups, key=lambda p: p[0]): # type: ignore[return-value, arg-type] # TODO SupportsLessThan?? + results: List[Res[T]] = [] + for v, grp in sorted(groups, key=lambda p: p[0]): # type: ignore[return-value, arg-type] # TODO SupportsLessThan?? results.extend(grp) - results.extend(group) # handle last group (it will always be errors only) + results.extend(group) # handle last group (it will always be errors only) return results @@ -153,7 +135,7 @@ def test_sort_res_by() -> None: Exc('last'), ] - results2 = sort_res_by([*ress, 0], lambda x: int(x)) + results2 = sort_res_by(ress + [0], lambda x: int(x)) assert results2 == [Exc('last'), 0] + results[:-1] assert sort_res_by(['caba', 'a', 'aba', 'daba'], key=lambda x: len(x)) == ['a', 'aba', 'caba', 'daba'] @@ -162,23 +144,23 @@ def test_sort_res_by() -> None: # helpers to associate timestamps with the errors (so something meaningful could be displayed on the plots, for example) # todo document it under 'patterns' somewhere... + # todo proper typevar? -def set_error_datetime(e: Exception, dt: datetime | None) -> None: +from datetime import datetime +def set_error_datetime(e: Exception, dt: Optional[datetime]) -> None: if dt is None: return - e.args = (*e.args, dt) + e.args = e.args + (dt,) # todo not sure if should return new exception? - -def attach_dt(e: Exception, *, dt: datetime | None) -> Exception: +def attach_dt(e: Exception, *, dt: Optional[datetime]) -> Exception: set_error_datetime(e, dt) return e - # todo it might be problematic because might mess with timezones (when it's converted to string, it's converted to a shift) -def extract_error_datetime(e: Exception) -> datetime | None: +def extract_error_datetime(e: Exception) -> Optional[datetime]: import re - + from datetime import datetime for x in reversed(e.args): if isinstance(x, datetime): return x @@ -193,6 +175,8 @@ def extract_error_datetime(e: Exception) -> datetime | None: return None +import traceback +from .common import Json def error_to_json(e: Exception) -> Json: estr = ''.join(traceback.format_exception(Exception, e, e.__traceback__)) return {'error': estr} @@ -200,13 +184,7 @@ def error_to_json(e: Exception) -> Json: MODULE_SETUP_URL = 'https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#private-configuration-myconfig' - -def warn_my_config_import_error( - err: ImportError | AttributeError, - *, - help_url: str | None = None, - module_name: str | None = None, -) -> bool: +def warn_my_config_import_error(err: Union[ImportError, AttributeError], help_url: Optional[str] = None) -> bool: """ If the user tried to import something from my.config but it failed, possibly due to missing the config block in my.config? @@ -214,12 +192,10 @@ def warn_my_config_import_error( Returns True if it matched a possible config error """ import re - import click - if help_url is None: help_url = MODULE_SETUP_URL - if type(err) is ImportError: + if type(err) == ImportError: if err.name != 'my.config': return False # parse name that user attempted to import @@ -231,31 +207,17 @@ You may be missing the '{section_name}' section from your config. See {help_url}\ """, fg='yellow', err=True) return True - elif type(err) is AttributeError: + elif type(err) == AttributeError: # test if user had a nested config block missing # https://github.com/karlicoss/HPI/issues/223 if hasattr(err, 'obj') and hasattr(err, "name"): config_obj = cast(object, getattr(err, 'obj')) # the object that caused the attribute error # e.g. active_browser for my.browser nested_block_name = err.name - errmsg = f"""You're likely missing the nested config block for '{getattr(config_obj, '__name__', str(config_obj))}.{nested_block_name}'. -See {help_url} or check the corresponding module.py file for an example\ -""" if config_obj.__module__ == 'my.config': - click.secho(errmsg, fg='yellow', err=True) - return True - if module_name is not None and nested_block_name == module_name.split('.')[-1]: - # this tries to cover cases like these - # user config: - # class location: - # class via_ip: - # accuracy = 10_000 - # then when we import it, we do something like - # from my.config import location - # user_config = location.via_ip - # so if location is present, but via_ip is not, we get - # AttributeError: type object 'location' has no attribute 'via_ip' - click.secho(errmsg, fg='yellow', err=True) + click.secho(f"""You're likely missing the nested config block for '{getattr(config_obj, '__name__', str(config_obj))}.{nested_block_name}'. +See {help_url} or check the corresponding module.py file for an example\ +""", fg='yellow', err=True) return True else: click.echo(f"Unexpected error... {err}", err=True) @@ -263,10 +225,9 @@ See {help_url} or check the corresponding module.py file for an example\ def test_datetime_errors() -> None: - import pytz # noqa: I001 - + import pytz dt_notz = datetime.now() - dt_tz = datetime.now(tz=pytz.timezone('Europe/Amsterdam')) + dt_tz = datetime.now(tz=pytz.timezone('Europe/Amsterdam')) for dt in [dt_tz, dt_notz]: e1 = RuntimeError('whatever') assert extract_error_datetime(e1) is None diff --git a/my/core/experimental.py b/my/core/experimental.py index 0a1c3b4..c10ba71 100644 --- a/my/core/experimental.py +++ b/my/core/experimental.py @@ -1,8 +1,6 @@ -from __future__ import annotations - import sys +from typing import Any, Dict, Optional import types -from typing import Any # The idea behind this one is to support accessing "overlaid/shadowed" modules from namespace packages @@ -22,7 +20,7 @@ def import_original_module( file: str, *, star: bool = False, - globals: dict[str, Any] | None = None, + globals: Optional[Dict[str, Any]] = None, ) -> types.ModuleType: module_to_restore = sys.modules[module_name] diff --git a/my/core/freezer.py b/my/core/freezer.py index 4fb0e25..649a2b7 100644 --- a/my/core/freezer.py +++ b/my/core/freezer.py @@ -1,29 +1,27 @@ -from __future__ import annotations +from .common import assert_subpackage; assert_subpackage(__name__) -from .internal import assert_subpackage - -assert_subpackage(__name__) - -import dataclasses +import dataclasses as dcl import inspect -from typing import Any, Generic, TypeVar +from typing import TypeVar, Type, Any D = TypeVar('D') -def _freeze_dataclass(Orig: type[D]): - ofields = [(f.name, f.type, f) for f in dataclasses.fields(Orig)] # type: ignore[arg-type] # see https://github.com/python/typing_extensions/issues/115 +def _freeze_dataclass(Orig: Type[D]): + ofields = [(f.name, f.type, f) for f in dcl.fields(Orig)] # type: ignore[arg-type] # see https://github.com/python/typing_extensions/issues/115 # extract properties along with their types - props = list(inspect.getmembers(Orig, lambda o: isinstance(o, property))) + props = list(inspect.getmembers(Orig, lambda o: isinstance(o, property))) pfields = [(name, inspect.signature(getattr(prop, 'fget')).return_annotation) for name, prop in props] # FIXME not sure about name? # NOTE: sadly passing bases=[Orig] won't work, python won't let us override properties with fields - RRR = dataclasses.make_dataclass('RRR', fields=[*ofields, *pfields]) + RRR = dcl.make_dataclass('RRR', fields=[*ofields, *pfields]) # todo maybe even declare as slots? return props, RRR +# todo need some decorator thingie? +from typing import Generic class Freezer(Generic[D]): ''' Some magic which converts dataclass properties into fields. @@ -31,13 +29,13 @@ class Freezer(Generic[D]): For now only supports dataclasses. ''' - def __init__(self, Orig: type[D]) -> None: + def __init__(self, Orig: Type[D]) -> None: self.Orig = Orig self.props, self.Frozen = _freeze_dataclass(Orig) def freeze(self, value: D) -> D: pvalues = {name: getattr(value, name) for name, _ in self.props} - return self.Frozen(**dataclasses.asdict(value), **pvalues) # type: ignore[call-overload] # see https://github.com/python/typing_extensions/issues/115 + return self.Frozen(**dcl.asdict(value), **pvalues) # type: ignore[call-overload] # see https://github.com/python/typing_extensions/issues/115 ### tests @@ -45,7 +43,7 @@ class Freezer(Generic[D]): # this needs to be defined here to prevent a mypy bug # see https://github.com/python/mypy/issues/7281 -@dataclasses.dataclass +@dcl.dataclass class _A: x: Any @@ -60,10 +58,8 @@ class _A: def test_freezer() -> None: - val = _A(x={ - 'an_int': 123, - 'an_any': [1, 2, 3], - }) + + val = _A(x=dict(an_int=123, an_any=[1, 2, 3])) af = Freezer(_A) fval = af.freeze(val) @@ -71,7 +67,6 @@ def test_freezer() -> None: assert fd['typed'] == 123 assert fd['untyped'] == [1, 2, 3] - ### # TODO shit. what to do with exceptions? diff --git a/my/core/hpi_compat.py b/my/core/hpi_compat.py index 3687483..61121de 100644 --- a/my/core/hpi_compat.py +++ b/my/core/hpi_compat.py @@ -2,15 +2,11 @@ Contains various backwards compatibility/deprecation helpers relevant to HPI itself. (as opposed to .compat module which implements compatibility between python versions) """ - -from __future__ import annotations - -import inspect import os +import inspect import re -from collections.abc import Iterator, Sequence from types import ModuleType -from typing import TypeVar +from typing import Iterator, List, Optional, TypeVar from . import warnings @@ -18,7 +14,7 @@ from . import warnings def handle_legacy_import( parent_module_name: str, legacy_submodule_name: str, - parent_module_path: list[str], + parent_module_path: List[str], ) -> bool: ### # this is to trick mypy into treating this as a proper namespace package @@ -75,7 +71,7 @@ def pre_pip_dal_handler( name: str, e: ModuleNotFoundError, cfg, - requires: Sequence[str] = (), + requires=[], ) -> ModuleType: ''' https://github.com/karlicoss/HPI/issues/79 @@ -105,7 +101,7 @@ Please install {' '.join(requires)} as PIP packages (see the corresponding READM def _get_dal(cfg, module_name: str): mpath = getattr(cfg, module_name, None) if mpath is not None: - from .utils.imports import import_dir + from .common import import_dir return import_dir(mpath, '.dal') else: @@ -120,141 +116,32 @@ V = TypeVar('V') # named to be kinda consistent with more_itertools, e.g. more_itertools.always_iterable class always_supports_sequence(Iterator[V]): """ - Helper to make migration from Sequence/List to Iterable/Iterator type backwards compatible in runtime + Helper to make migration from Sequence/List to Iterable/Iterator type backwards compatible """ def __init__(self, it: Iterator[V]) -> None: - self._it = it - self._list: list[V] | None = None - self._lit: Iterator[V] | None = None + self.it = it + self._list: Optional[List] = None - def __iter__(self) -> Iterator[V]: # noqa: PYI034 - if self._list is not None: - self._lit = iter(self._list) - return self + def __iter__(self) -> Iterator[V]: + return self.it.__iter__() def __next__(self) -> V: - if self._list is not None: - assert self._lit is not None - delegate = self._lit - else: - delegate = self._it - return next(delegate) + return self.it.__next__() def __getattr__(self, name): - return getattr(self._it, name) + return getattr(self.it, name) @property - def _aslist(self) -> list[V]: + def aslist(self) -> List[V]: if self._list is None: - qualname = getattr(self._it, '__qualname__', '') # defensive just in case + qualname = getattr(self.it, '__qualname__', '') # defensive just in case warnings.medium(f'Using {qualname} as list is deprecated. Migrate to iterative processing or call list() explicitly.') - self._list = list(self._it) - - # this is necessary for list constructor to work correctly - # since it's __iter__ first, then tries to compute length and then starts iterating... - self._lit = iter(self._list) + self._list = list(self.it) return self._list def __len__(self) -> int: - return len(self._aslist) + return len(self.aslist) def __getitem__(self, i: int) -> V: - return self._aslist[i] - - -def test_always_supports_sequence_list_constructor() -> None: - exhausted = 0 - - def it() -> Iterator[str]: - nonlocal exhausted - yield from ['a', 'b', 'c'] - exhausted += 1 - - sit = always_supports_sequence(it()) - - # list constructor is a bit special... it's trying to compute length if it's available to optimize memory allocation - # so, what's happening in this case is - # - sit.__iter__ is called - # - sit.__len__ is called - # - sit.__next__ is called - res = list(sit) - assert res == ['a', 'b', 'c'] - assert exhausted == 1 - - res = list(sit) - assert res == ['a', 'b', 'c'] - assert exhausted == 1 # this will iterate over 'cached' list now, so original generator is only exhausted once - - -def test_always_supports_sequence_indexing() -> None: - exhausted = 0 - - def it() -> Iterator[str]: - nonlocal exhausted - yield from ['a', 'b', 'c'] - exhausted += 1 - - sit = always_supports_sequence(it()) - - assert len(sit) == 3 - assert exhausted == 1 - - assert sit[2] == 'c' - assert sit[1] == 'b' - assert sit[0] == 'a' - assert exhausted == 1 - - # a few tests to make sure list-like operations are working.. - assert list(sit) == ['a', 'b', 'c'] - assert [x for x in sit] == ['a', 'b', 'c'] # noqa: C416 - assert list(sit) == ['a', 'b', 'c'] - assert [x for x in sit] == ['a', 'b', 'c'] # noqa: C416 - assert exhausted == 1 - - -def test_always_supports_sequence_next() -> None: - exhausted = 0 - - def it() -> Iterator[str]: - nonlocal exhausted - yield from ['a', 'b', 'c'] - exhausted += 1 - - sit = always_supports_sequence(it()) - - x = next(sit) - assert x == 'a' - assert exhausted == 0 - - x = next(sit) - assert x == 'b' - assert exhausted == 0 - - -def test_always_supports_sequence_iter() -> None: - exhausted = 0 - - def it() -> Iterator[str]: - nonlocal exhausted - yield from ['a', 'b', 'c'] - exhausted += 1 - - sit = always_supports_sequence(it()) - - for x in sit: - assert x == 'a' - break - - x = next(sit) - assert x == 'b' - - assert exhausted == 0 - - x = next(sit) - assert x == 'c' - assert exhausted == 0 - - for _ in sit: - raise RuntimeError # shouldn't trigger, just exhaust the iterator - assert exhausted == 1 + return self.aslist[i] diff --git a/my/core/influxdb.py b/my/core/influxdb.py index 78a439a..8407264 100644 --- a/my/core/influxdb.py +++ b/my/core/influxdb.py @@ -1,22 +1,14 @@ ''' TODO doesn't really belong to 'core' morally, but can think of moving out later ''' +from .common import assert_subpackage; assert_subpackage(__name__) -from __future__ import annotations +from typing import Iterable, Any, Optional, Dict -from .internal import assert_subpackage +from .common import LazyLogger, asdict, Json -assert_subpackage(__name__) -from collections.abc import Iterable -from typing import Any - -import click - -from .logging import make_logger -from .types import Json, asdict - -logger = make_logger(__name__) +logger = LazyLogger(__name__) class config: @@ -26,7 +18,7 @@ class config: RESET_DEFAULT = False -def fill(it: Iterable[Any], *, measurement: str, reset: bool = RESET_DEFAULT, dt_col: str = 'dt') -> None: +def fill(it: Iterable[Any], *, measurement: str, reset: bool=RESET_DEFAULT, dt_col: str='dt') -> None: # todo infer dt column automatically, reuse in stat? # it doesn't like dots, ends up some syntax error? measurement = measurement.replace('.', '_') @@ -34,8 +26,7 @@ def fill(it: Iterable[Any], *, measurement: str, reset: bool = RESET_DEFAULT, dt db = config.db - from influxdb import InfluxDBClient # type: ignore - + from influxdb import InfluxDBClient # type: ignore client = InfluxDBClient() # todo maybe create if not exists? # client.create_database(db) @@ -46,7 +37,7 @@ def fill(it: Iterable[Any], *, measurement: str, reset: bool = RESET_DEFAULT, dt client.delete_series(database=db, measurement=measurement) # TODO need to take schema here... - cache: dict[str, bool] = {} + cache: Dict[str, bool] = {} def good(f, v) -> bool: c = cache.get(f) @@ -65,9 +56,9 @@ def fill(it: Iterable[Any], *, measurement: str, reset: bool = RESET_DEFAULT, dt def dit() -> Iterable[Json]: for i in it: d = asdict(i) - tags: Json | None = None - tags_ = d.get('tags') # meh... handle in a more robust manner - if tags_ is not None and isinstance(tags_, dict): # FIXME meh. + tags: Optional[Json] = None + tags_ = d.get('tags') # meh... handle in a more robust manner + if tags_ is not None and isinstance(tags_, dict): # FIXME meh. del d['tags'] tags = tags_ @@ -78,19 +69,18 @@ def fill(it: Iterable[Any], *, measurement: str, reset: bool = RESET_DEFAULT, dt fields = filter_dict(d) - yield { - 'measurement': measurement, + yield dict( + measurement=measurement, # TODO maybe good idea to tag with database file/name? to inspect inconsistencies etc.. # hmm, so tags are autoindexed and might be faster? # not sure what's the big difference though # "fields are data and tags are metadata" - 'tags': tags, - 'time': dt, - 'fields': fields, - } + tags=tags, + time=dt, + fields=fields, + ) from more_itertools import chunked - # "The optimal batch size is 5000 lines of line protocol." # some chunking is def necessary, otherwise it fails inserted = 0 @@ -104,9 +94,9 @@ def fill(it: Iterable[Any], *, measurement: str, reset: bool = RESET_DEFAULT, dt # todo "Specify timestamp precision when writing to InfluxDB."? -def magic_fill(it, *, name: str | None = None, reset: bool = RESET_DEFAULT) -> None: +def magic_fill(it, *, name: Optional[str]=None, reset: bool=RESET_DEFAULT) -> None: if name is None: - assert callable(it) # generators have no name/module + assert callable(it) # generators have no name/module name = f'{it.__module__}:{it.__name__}' assert name is not None @@ -114,9 +104,7 @@ def magic_fill(it, *, name: str | None = None, reset: bool = RESET_DEFAULT) -> N it = it() from itertools import tee - from more_itertools import first, one - it, x = tee(it) f = first(x, default=None) if f is None: @@ -126,17 +114,17 @@ def magic_fill(it, *, name: str | None = None, reset: bool = RESET_DEFAULT) -> N # TODO can we reuse pandas code or something? # from .pandas import _as_columns - schema = _as_columns(type(f)) from datetime import datetime - dtex = RuntimeError(f'expected single datetime field. schema: {schema}') dtf = one((f for f, t in schema.items() if t == datetime), too_short=dtex, too_long=dtex) fill(it, measurement=name, reset=reset, dt_col=dtf) +import click + @click.group() def main() -> None: pass @@ -145,9 +133,8 @@ def main() -> None: @main.command(name='populate', short_help='populate influxdb') @click.option('--reset', is_flag=True, help='Reset Influx measurements before inserting', show_default=True) @click.argument('FUNCTION_NAME', type=str, required=True) -def populate(*, function_name: str, reset: bool) -> None: +def populate(function_name: str, reset: bool) -> None: from .__main__ import _locate_functions_or_prompt - [provider] = list(_locate_functions_or_prompt([function_name])) # todo could have a non-interactive version which populates from all data sources for the provider? magic_fill(provider, reset=reset) diff --git a/my/core/init.py b/my/core/init.py index 644c7b4..bec3a9a 100644 --- a/my/core/init.py +++ b/my/core/init.py @@ -14,19 +14,18 @@ Please let me know if you are aware of a better way of dealing with this! # separate function to present namespace pollution def setup_config() -> None: + from pathlib import Path import sys import warnings - from pathlib import Path from .preinit import get_mycfg_dir - mycfg_dir = get_mycfg_dir() if not mycfg_dir.exists(): warnings.warn(f""" 'my.config' package isn't found! (expected at '{mycfg_dir}'). This is likely to result in issues. See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-modules for more info. -""".strip(), stacklevel=1) +""".strip()) return mpath = str(mycfg_dir) @@ -44,12 +43,11 @@ See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-mo except ImportError as ex: # just in case... who knows what crazy setup users have import logging - logging.exception(ex) warnings.warn(f""" Importing 'my.config' failed! (error: {ex}). This is likely to result in issues. See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-modules for more info. -""", stacklevel=1) +""") else: # defensive just in case -- __file__ may not be present if there is some dynamic magic involved used_config_file = getattr(my.config, '__file__', None) @@ -65,7 +63,7 @@ See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-mo Expected my.config to be located at {mycfg_dir}, but instead its path is {used_config_path}. This will likely cause issues down the line -- double check {mycfg_dir} structure. See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-the-modules for more info. -""", stacklevel=1 +""", ) diff --git a/my/core/internal.py b/my/core/internal.py deleted file mode 100644 index 8b9882b..0000000 --- a/my/core/internal.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Utils specific to hpi core, shouldn't really be used by HPI modules -""" - - -def assert_subpackage(name: str) -> None: - # can lead to some unexpected issues if you 'import cachew' which being in my/core directory.. so let's protect against it - # NOTE: if we use overlay, name can be smth like my.origg.my.core.cachew ... - assert name == '__main__' or 'my.core' in name, f'Expected module __name__ ({name}) to be __main__ or start with my.core' diff --git a/my/core/kompress.py b/my/core/kompress.py index 8accb2d..25dba8c 100644 --- a/my/core/kompress.py +++ b/my/core/kompress.py @@ -1,7 +1,4 @@ -from .internal import assert_subpackage - -assert_subpackage(__name__) - +from .common import assert_subpackage; assert_subpackage(__name__) from . import warnings # do this later -- for now need to transition modules to avoid using kompress directly (e.g. ZipPath) @@ -11,7 +8,10 @@ try: from kompress import * except ModuleNotFoundError as e: if e.name == 'kompress': - warnings.high('Please install kompress (pip3 install kompress). Falling onto vendorized kompress for now.') + warnings.high('Please install kompress (pip3 install kompress), it will be required in the future. Falling onto vendorized kompress for now.') from ._deprecated.kompress import * # type: ignore[assignment] else: raise e + +# this is deprecated in compress, keep here for backwards compatibility +open = kopen # noqa: F405 diff --git a/my/core/konsume.py b/my/core/konsume.py index 41b5a4e..588bfe1 100644 --- a/my/core/konsume.py +++ b/my/core/konsume.py @@ -5,21 +5,17 @@ This can potentially allow both for safer defensive parsing, and let you know if TODO perhaps need to get some inspiration from linear logic to decide on a nice API... ''' -from __future__ import annotations - from collections import OrderedDict -from typing import Any +from typing import Any, List def ignore(w, *keys): for k in keys: w[k].ignore() - def zoom(w, *keys): return [w[k].zoom() for k in keys] - # TODO need to support lists class Zoomable: def __init__(self, parent, *args, **kwargs) -> None: @@ -44,7 +40,7 @@ class Zoomable: assert self.parent is not None self.parent._remove(self) - def zoom(self) -> Zoomable: + def zoom(self) -> 'Zoomable': self.consume() return self @@ -67,7 +63,6 @@ class Wdict(Zoomable, OrderedDict): def this_consumed(self): return len(self) == 0 - # TODO specify mypy type for the index special method? @@ -82,7 +77,6 @@ class Wlist(Zoomable, list): def this_consumed(self): return len(self) == 0 - class Wvalue(Zoomable): def __init__(self, parent, value: Any) -> None: super().__init__(parent) @@ -93,20 +87,21 @@ class Wvalue(Zoomable): return [] def this_consumed(self): - return True # TODO not sure.. + return True # TODO not sure.. def __repr__(self): return 'WValue{' + repr(self.value) + '}' -def _wrap(j, parent=None) -> tuple[Zoomable, list[Zoomable]]: +from typing import Tuple +def _wrap(j, parent=None) -> Tuple[Zoomable, List[Zoomable]]: res: Zoomable - cc: list[Zoomable] + cc: List[Zoomable] if isinstance(j, dict): res = Wdict(parent) cc = [res] for k, v in j.items(): - vv, c = _wrap(v, parent=res) + vv, c = _wrap(v, parent=res) res[k] = vv cc.extend(c) return res, cc @@ -125,23 +120,21 @@ def _wrap(j, parent=None) -> tuple[Zoomable, list[Zoomable]]: raise RuntimeError(f'Unexpected type: {type(j)} {j}') -from collections.abc import Iterator from contextlib import contextmanager - +from typing import Iterator class UnconsumedError(Exception): pass - # TODO think about error policy later... @contextmanager -def wrap(j, *, throw=True) -> Iterator[Zoomable]: +def wrap(j, throw=True) -> Iterator[Zoomable]: w, children = _wrap(j) yield w for c in children: - if not c.this_consumed(): # TODO hmm. how does it figure out if it's consumed??? + if not c.this_consumed(): # TODO hmm. how does it figure out if it's consumed??? if throw: # TODO need to keep a full path or something... raise UnconsumedError(f''' @@ -153,11 +146,8 @@ Expected {c} to be fully consumed by the parser. from typing import cast - - def test_unconsumed() -> None: import pytest - with pytest.raises(UnconsumedError): with wrap({'a': 1234}) as w: w = cast(Wdict, w) @@ -168,7 +158,6 @@ def test_unconsumed() -> None: w = cast(Wdict, w) d = w['c']['d'].zoom() - def test_consumed() -> None: with wrap({'a': 1234}) as w: w = cast(Wdict, w) @@ -179,7 +168,6 @@ def test_consumed() -> None: c = w['c'].zoom() d = c['d'].zoom() - def test_types() -> None: # (string, number, object, array, boolean or nul with wrap({'string': 'string', 'number': 3.14, 'boolean': True, 'null': None, 'list': [1, 2, 3]}) as w: @@ -188,10 +176,9 @@ def test_types() -> None: w['number'].consume() w['boolean'].zoom() w['null'].zoom() - for x in list(w['list'].zoom()): # TODO eh. how to avoid the extra list thing? + for x in list(w['list'].zoom()): # TODO eh. how to avoid the extra list thing? x.consume() - def test_consume_all() -> None: with wrap({'aaa': {'bbb': {'hi': 123}}}) as w: w = cast(Wdict, w) @@ -201,9 +188,11 @@ def test_consume_all() -> None: def test_consume_few() -> None: import pytest - pytest.skip('Will think about it later..') - with wrap({'important': 123, 'unimportant': 'whatever'}) as w: + with wrap({ + 'important': 123, + 'unimportant': 'whatever' + }) as w: w = cast(Wdict, w) w['important'].zoom() w.consume_all() @@ -212,7 +201,6 @@ def test_consume_few() -> None: def test_zoom() -> None: import pytest - with wrap({'aaa': 'whatever'}) as w: w = cast(Wdict, w) with pytest.raises(KeyError): @@ -221,34 +209,3 @@ def test_zoom() -> None: # TODO type check this... - -# TODO feels like the whole thing kind of unnecessarily complex -# - cons: -# - in most cases this is not even needed? who cares if we miss a few attributes? -# - pro: on the other hand it could be interesting to know about new attributes in data, -# and without this kind of processing we wouldn't even know -# alternatives -# - manually process data -# e.g. use asserts, dict.pop and dict.values() methods to unpack things -# - pros: -# - very simple, since uses built in syntax -# - very performant, as fast as it gets -# - very flexible, easy to adjust behaviour -# - cons: -# - can forget to assert about extra entities etc, so error prone -# - if we do something like =assert j.pop('status') == 200, j=, by the time assert happens we already popped item -- makes error handling harder -# - a bit verbose.. so probably requires some helper functions though (could be much leaner than current konsume though) -# - if we assert, then terminates parsing too early, if we're defensive then inflates the code a lot with if statements -# - TODO perhaps combine warnings somehow or at least only emit once per module? -# - hmm actually tbh if we carefully go through everything and don't make copies, then only requires one assert at the very end? -# - TODO this is kinda useful? https://discuss.python.org/t/syntax-for-dictionnary-unpacking-to-variables/18718 -# operator.itemgetter? -# - TODO can use match operator in python for this? quite nice actually! and allows for dynamic behaviour -# only from 3.10 tho, and gonna be tricky to do dynamic defensive behaviour with this -# - TODO in a sense, blenser already would hint if some meaningful fields aren't being processed? only if they are changing though -# - define a "schema" for data, then just recursively match data against the schema? -# possibly pydantic already does something like that? not sure about performance though -# pros: -# - much simpler to extend and understand what's going on -# cons: -# - more rigid, so it becomes tricky to do dynamic stuff (e.g. if schema actually changes) diff --git a/my/core/logging.py b/my/core/logging.py index 167a167..5d2af99 100644 --- a/my/core/logging.py +++ b/my/core/logging.py @@ -1,11 +1,11 @@ from __future__ import annotations +from functools import lru_cache import logging import os import sys +from typing import Union import warnings -from functools import lru_cache -from typing import TYPE_CHECKING, Union def test() -> None: @@ -15,7 +15,7 @@ def test() -> None: ## prepare exception for later try: - None.whatever # type: ignore[attr-defined] # noqa: B018 + None.whatever # type: ignore[attr-defined] except Exception as e: ex = e ## @@ -146,7 +146,7 @@ def _setup_handlers_and_formatters(name: str) -> None: # try colorlog first, so user gets nice colored logs import colorlog except ModuleNotFoundError: - warnings.warn("You might want to 'pip install colorlog' for nice colored logs", stacklevel=1) + warnings.warn("You might want to 'pip install colorlog' for nice colored logs") formatter = logging.Formatter(FORMAT_NOCOLOR) else: # log_color/reset are specific to colorlog @@ -222,9 +222,7 @@ def make_logger(name: str, *, level: LevelIsh = None) -> logging.Logger: # OK, when stdout is not a tty, enlighten doesn't log anything, good def get_enlighten(): # TODO could add env variable to disable enlighten for a module? - from unittest.mock import ( - Mock, # Mock to return stub so cients don't have to think about it - ) + from unittest.mock import Mock # Mock to return stub so cients don't have to think about it # for now hidden behind the flag since it's a little experimental if os.environ.get('ENLIGHTEN_ENABLE', None) is None: @@ -233,7 +231,7 @@ def get_enlighten(): try: import enlighten # type: ignore[import-untyped] except ModuleNotFoundError: - warnings.warn("You might want to 'pip install enlighten' for a nice progress bar", stacklevel=1) + warnings.warn("You might want to 'pip install enlighten' for a nice progress bar") return Mock() @@ -250,17 +248,7 @@ if __name__ == '__main__': test() -## legacy/deprecated methods for backwards compatibility -if not TYPE_CHECKING: - from .compat import deprecated - - @deprecated('use make_logger instead') - def LazyLogger(*args, **kwargs): - return make_logger(*args, **kwargs) - - @deprecated('use make_logger instead') - def logger(*args, **kwargs): - return make_logger(*args, **kwargs) - - +## legacy/deprecated methods for backwards compatilibity +LazyLogger = make_logger +logger = make_logger ## diff --git a/my/core/mime.py b/my/core/mime.py deleted file mode 100644 index 8235960..0000000 --- a/my/core/mime.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Utils for mime/filetype handling -""" - -from __future__ import annotations - -from .internal import assert_subpackage - -assert_subpackage(__name__) - -import functools -from pathlib import Path - - -@functools.lru_cache(1) -def _magic(): - import magic # type: ignore - - # TODO also has uncompess=True? could be useful - return magic.Magic(mime=True) - - -# TODO could reuse in pdf module? -import mimetypes # todo do I need init()? - - -# todo wtf? fastermime thinks it's mime is application/json even if the extension is xz?? -# whereas magic detects correctly: application/x-zstd and application/x-xz -def fastermime(path: Path | str) -> str: - paths = str(path) - # mimetypes is faster, so try it first - (mime, _) = mimetypes.guess_type(paths) - if mime is not None: - return mime - # magic is slower but handles more types - # TODO Result type?; it's kinda racey, but perhaps better to let the caller decide? - return _magic().from_file(paths) diff --git a/my/core/orgmode.py b/my/core/orgmode.py index 96c09a4..5894b23 100644 --- a/my/core/orgmode.py +++ b/my/core/orgmode.py @@ -1,46 +1,37 @@ """ Various helpers for reading org-mode data """ - from datetime import datetime - - def parse_org_datetime(s: str) -> datetime: s = s.strip('[]') - for fmt, _cls in [ - ("%Y-%m-%d %a %H:%M", datetime), - ("%Y-%m-%d %H:%M" , datetime), - # todo not sure about these... fallback on 00:00? - # ("%Y-%m-%d %a" , date), - # ("%Y-%m-%d" , date), + for fmt, cl in [ + ("%Y-%m-%d %a %H:%M", datetime), + ("%Y-%m-%d %H:%M" , datetime), + # todo not sure about these... fallback on 00:00? + # ("%Y-%m-%d %a" , date), + # ("%Y-%m-%d" , date), ]: try: return datetime.strptime(s, fmt) except ValueError: continue - raise RuntimeError(f"Bad datetime string {s}") + else: + raise RuntimeError(f"Bad datetime string {s}") # TODO I guess want to borrow inspiration from bs4? element type <-> tag; and similar logic for find_one, find_all -from collections.abc import Iterable -from typing import Callable, TypeVar - from orgparse import OrgNode - +from typing import Iterable, TypeVar, Callable V = TypeVar('V') - def collect(n: OrgNode, cfun: Callable[[OrgNode], Iterable[V]]) -> Iterable[V]: yield from cfun(n) for c in n.children: yield from collect(c, cfun) - from more_itertools import one from orgparse.extra import Table - - def one_table(o: OrgNode) -> Table: return one(collect(o, lambda n: (x for x in n.body_rich if isinstance(x, Table)))) @@ -50,7 +41,7 @@ class TypedTable(Table): tt = super().__new__(TypedTable) tt.__dict__ = orig.__dict__ blocks = list(orig.blocks) - header = blocks[0] # fist block is schema + header = blocks[0] # fist block is schema if len(header) == 2: # TODO later interpret first line as types header = header[1:] diff --git a/my/core/pandas.py b/my/core/pandas.py index d444965..1b7a644 100644 --- a/my/core/pandas.py +++ b/my/core/pandas.py @@ -1,31 +1,23 @@ ''' Various pandas helpers and convenience functions ''' - from __future__ import annotations # todo not sure if belongs to 'core'. It's certainly 'more' core than actual modules, but still not essential # NOTE: this file is meant to be importable without Pandas installed import dataclasses -from collections.abc import Iterable, Iterator from datetime import datetime, timezone from pprint import pformat -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Literal, - TypeVar, -) +from typing import TYPE_CHECKING, Any, Iterable, Type, Dict, Literal, Callable, TypeVar from decorator import decorator -from . import warnings -from .error import Res, error_to_json, extract_error_datetime -from .logging import make_logger -from .types import Json, asdict +from . import warnings, Res +from .common import LazyLogger, Json, asdict +from .error import error_to_json, extract_error_datetime -logger = make_logger(__name__) + +logger = LazyLogger(__name__) if TYPE_CHECKING: @@ -46,7 +38,7 @@ else: S1 = Any -def _check_dateish(s: SeriesT[S1]) -> Iterable[str]: +def check_dateish(s: SeriesT[S1]) -> Iterable[str]: import pandas as pd # noqa: F811 not actually a redefinition ctype = s.dtype @@ -58,7 +50,7 @@ def _check_dateish(s: SeriesT[S1]) -> Iterable[str]: all_timestamps = s.apply(lambda x: isinstance(x, (pd.Timestamp, datetime))).all() if not all_timestamps: return # not sure why it would happen, but ok - tzs = s.map(lambda x: x.tzinfo).drop_duplicates() # type: ignore[union-attr, var-annotated, arg-type, return-value, unused-ignore] + tzs = s.map(lambda x: x.tzinfo).drop_duplicates() examples = s[tzs.index] # todo not so sure this warning is that useful... except for stuff without tz yield f''' @@ -70,37 +62,9 @@ def _check_dateish(s: SeriesT[S1]) -> Iterable[str]: def test_check_dateish() -> None: import pandas as pd - from .compat import fromisoformat - - # empty series shouldn't warn - assert list(_check_dateish(pd.Series([]))) == [] - - # if no dateimes, shouldn't return any warnings - assert list(_check_dateish(pd.Series([1, 2, 3]))) == [] - - # all values are datetimes, shouldn't warn - # fmt: off - assert list(_check_dateish(pd.Series([ - fromisoformat('2024-08-19T01:02:03'), - fromisoformat('2024-08-19T03:04:05'), - ]))) == [] - # fmt: on - - # mixture of timezones -- should warn - # fmt: off - assert len(list(_check_dateish(pd.Series([ - fromisoformat('2024-08-19T01:02:03'), - fromisoformat('2024-08-19T03:04:05Z'), - ])))) == 1 - # fmt: on - - # TODO hmm. maybe this should actually warn? - # fmt: off - assert len(list(_check_dateish(pd.Series([ - 'whatever', - fromisoformat('2024-08-19T01:02:03'), - ])))) == 0 - # fmt: on + # todo just a dummy test to check it doesn't crash, need something meaningful + s1 = pd.Series([1, 2, 3]) + list(check_dateish(s1)) # fmt: off @@ -138,7 +102,7 @@ def check_dataframe(f: FuncT, error_col_policy: ErrorColPolicy = 'add_if_missing # makes sense to keep super defensive try: for col, data in df.reset_index().items(): - for w in _check_dateish(data): + for w in check_dateish(data): warnings.low(f"{tag}, column '{col}': {w}") except Exception as e: logger.exception(e) @@ -162,7 +126,8 @@ def error_to_row(e: Exception, *, dt_col: str = 'dt', tz: timezone | None = None return err_dict -def _to_jsons(it: Iterable[Res[Any]]) -> Iterable[Json]: +# todo not sure about naming +def to_jsons(it: Iterable[Res[Any]]) -> Iterable[Json]: for r in it: if isinstance(r, Exception): yield error_to_row(r) @@ -175,10 +140,10 @@ def _to_jsons(it: Iterable[Res[Any]]) -> Iterable[Json]: Schema = Any -def _as_columns(s: Schema) -> dict[str, type]: +def _as_columns(s: Schema) -> Dict[str, Type]: # todo would be nice to extract properties; add tests for this as well if dataclasses.is_dataclass(s): - return {f.name: f.type for f in dataclasses.fields(s)} # type: ignore[misc] # ugh, why mypy thinks f.type can return str?? + return {f.name: f.type for f in dataclasses.fields(s)} # else must be NamedTuple?? # todo assert my.core.common.is_namedtuple? return getattr(s, '_field_types') @@ -197,7 +162,7 @@ def as_dataframe(it: Iterable[Res[Any]], schema: Schema | None = None) -> DataFr import pandas as pd # noqa: F811 not actually a redefinition columns = None if schema is None else list(_as_columns(schema).keys()) - return pd.DataFrame(_to_jsons(it), columns=columns) + return pd.DataFrame(to_jsons(it), columns=columns) # ugh. in principle this could be inside the test @@ -207,76 +172,20 @@ def as_dataframe(it: Iterable[Res[Any]], schema: Schema | None = None) -> DataFr # see https://github.com/pytest-dev/pytest/issues/7856 @dataclasses.dataclass class _X: - # FIXME try moving inside? x: int def test_as_dataframe() -> None: - import numpy as np - import pandas as pd import pytest - from pandas.testing import assert_frame_equal - from .compat import fromisoformat - - it = ({'i': i, 's': f'str{i}'} for i in range(5)) + it = (dict(i=i, s=f'str{i}') for i in range(10)) with pytest.warns(UserWarning, match=r"No 'error' column") as record_warnings: # noqa: F841 df: DataFrameT = as_dataframe(it) # todo test other error col policies + assert list(df.columns) == ['i', 's', 'error'] - # fmt: off - assert_frame_equal( - df, - pd.DataFrame({ - 'i' : [0 , 1 , 2 , 3 , 4 ], - 's' : ['str0', 'str1', 'str2', 'str3', 'str4'], - # NOTE: error column is always added - 'error': [None , None , None , None , None ], - }), - ) - # fmt: on - assert_frame_equal(as_dataframe([]), pd.DataFrame(columns=['error'])) + assert len(as_dataframe([])) == 0 + # makes sense to specify the schema so the downstream program doesn't fail in case of empty iterable df2: DataFrameT = as_dataframe([], schema=_X) - assert_frame_equal( - df2, - # FIXME hmm. x column type should be an int?? and error should be string (or object??) - pd.DataFrame(columns=['x', 'error']), - ) - - @dataclasses.dataclass - class S: - value: str - - def it2() -> Iterator[Res[S]]: - yield S(value='test') - yield RuntimeError('i failed') - - df = as_dataframe(it2()) - # fmt: off - assert_frame_equal( - df, - pd.DataFrame(data={ - 'value': ['test', np.nan ], - 'error': [np.nan, 'RuntimeError: i failed\n'], - 'dt' : [np.nan, np.nan ], - }).astype(dtype={'dt': 'float'}), # FIXME should be datetime64 as below - ) - # fmt: on - - def it3() -> Iterator[Res[S]]: - yield S(value='aba') - yield RuntimeError('whoops') - yield S(value='cde') - yield RuntimeError('exception with datetime', fromisoformat('2024-08-19T22:47:01Z')) - - df = as_dataframe(it3()) - - # fmt: off - assert_frame_equal(df, pd.DataFrame(data={ - 'value': ['aba' , np.nan , 'cde' , np.nan ], - 'error': [np.nan, 'RuntimeError: whoops\n', np.nan, "RuntimeError: ('exception with datetime', datetime.datetime(2024, 8, 19, 22, 47, 1, tzinfo=datetime.timezone.utc))\n"], - # note: dt column is added even if errors don't have an associated datetime - 'dt' : [np.nan, np.nan , np.nan, '2024-08-19 22:47:01+00:00'], - }).astype(dtype={'dt': 'datetime64[ns, UTC]'})) - # fmt: on + assert list(df2.columns) == ['x', 'error'] diff --git a/my/core/preinit.py b/my/core/preinit.py index eb3a34f..88bcb27 100644 --- a/my/core/preinit.py +++ b/my/core/preinit.py @@ -1,14 +1,11 @@ from pathlib import Path - # todo preinit isn't really a good name? it's only in a separate file because # - it's imported from my.core.init (so we wan't to keep this file as small/reliable as possible, hence not common or something) # - we still need this function in __main__, so has to be separate from my/core/init.py def get_mycfg_dir() -> Path: + import appdirs import os - - import appdirs # type: ignore[import-untyped] - # not sure if that's necessary, i.e. could rely on PYTHONPATH instead # on the other hand, by using MY_CONFIG we are guaranteed to load it from the desired path? mvar = os.environ.get('MY_CONFIG') diff --git a/my/core/pytest.py b/my/core/pytest.py deleted file mode 100644 index ad9e7d7..0000000 --- a/my/core/pytest.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Helpers to prevent depending on pytest in runtime -""" - -from .internal import assert_subpackage - -assert_subpackage(__name__) - -import sys -import typing - -under_pytest = 'pytest' in sys.modules - -if typing.TYPE_CHECKING or under_pytest: - import pytest - - parametrize = pytest.mark.parametrize -else: - - def parametrize(*_args, **_kwargs): - def wrapper(f): - return f - - return wrapper diff --git a/my/core/query.py b/my/core/query.py index 50724a7..4e00569 100644 --- a/my/core/query.py +++ b/my/core/query.py @@ -5,29 +5,21 @@ The main entrypoint to this library is the 'select' function below; try: python3 -c "from my.core.query import select; help(select)" """ -from __future__ import annotations - import dataclasses import importlib import inspect import itertools -from collections.abc import Iterable, Iterator from datetime import datetime -from typing import ( - Any, - Callable, - NamedTuple, - Optional, - TypeVar, -) +from typing import TypeVar, Tuple, Optional, Union, Callable, Iterable, Iterator, Dict, Any, NamedTuple, List import more_itertools -from . import error as err +import my.core.error as err +from .common import is_namedtuple from .error import Res, unwrap -from .types import is_namedtuple from .warnings import low + T = TypeVar("T") ET = Res[T] @@ -48,7 +40,6 @@ class Unsortable(NamedTuple): class QueryException(ValueError): """Used to differentiate query-related errors, so the CLI interface is more expressive""" - pass @@ -61,7 +52,7 @@ def locate_function(module_name: str, function_name: str) -> Callable[[], Iterab """ try: mod = importlib.import_module(module_name) - for fname, f in inspect.getmembers(mod, inspect.isfunction): + for (fname, f) in inspect.getmembers(mod, inspect.isfunction): if fname == function_name: return f # in case the function is defined dynamically, @@ -70,7 +61,7 @@ def locate_function(module_name: str, function_name: str) -> Callable[[], Iterab if func is not None and callable(func): return func except Exception as e: - raise QueryException(str(e)) # noqa: B904 + raise QueryException(str(e)) raise QueryException(f"Could not find function '{function_name}' in '{module_name}'") @@ -81,10 +72,10 @@ def locate_qualified_function(qualified_name: str) -> Callable[[], Iterable[ET]] if "." not in qualified_name: raise QueryException("Could not find a '.' in the function name, e.g. my.reddit.rexport.comments") rdot_index = qualified_name.rindex(".") - return locate_function(qualified_name[:rdot_index], qualified_name[rdot_index + 1 :]) + return locate_function(qualified_name[:rdot_index], qualified_name[rdot_index + 1:]) -def attribute_func(obj: T, where: Where, default: U | None = None) -> OrderFunc | None: +def attribute_func(obj: T, where: Where, default: Optional[U] = None) -> Optional[OrderFunc]: """ Attempts to find an attribute which matches the 'where_function' on the object, using some getattr/dict checks. Returns a function which when called with @@ -112,7 +103,7 @@ def attribute_func(obj: T, where: Where, default: U | None = None) -> OrderFunc if where(v): return lambda o: o.get(k, default) # type: ignore[union-attr] elif dataclasses.is_dataclass(obj): - for field_name in obj.__annotations__.keys(): + for (field_name, _annotation) in obj.__annotations__.items(): if where(getattr(obj, field_name)): return lambda o: getattr(o, field_name, default) elif is_namedtuple(obj): @@ -129,13 +120,12 @@ def attribute_func(obj: T, where: Where, default: U | None = None) -> OrderFunc def _generate_order_by_func( - obj_res: Res[T], - *, - key: str | None = None, - where_function: Where | None = None, - default: U | None = None, - force_unsortable: bool = False, -) -> OrderFunc | None: + obj_res: Res[T], + key: Optional[str] = None, + where_function: Optional[Where] = None, + default: Optional[U] = None, + force_unsortable: bool = False, +) -> Optional[OrderFunc]: """ Accepts an object Res[T] (Instance of some class or Exception) @@ -188,7 +178,7 @@ pass 'drop_exceptions' to ignore exceptions""") return lambda o: o.get(key, default) # type: ignore[union-attr] else: if hasattr(obj, key): - return lambda o: getattr(o, key, default) + return lambda o: getattr(o, key, default) # type: ignore[arg-type] # Note: if the attribute you're ordering by is an Optional type, # and on some objects it'll return None, the getattr(o, field_name, default) won't @@ -200,7 +190,7 @@ pass 'drop_exceptions' to ignore exceptions""") # user must provide either a key or a where predicate if where_function is not None: - func: OrderFunc | None = attribute_func(obj, where_function, default) + func: Optional[OrderFunc] = attribute_func(obj, where_function, default) if func is not None: return func @@ -216,13 +206,15 @@ pass 'drop_exceptions' to ignore exceptions""") return None # couldn't compute a OrderFunc for this class/instance + + # currently using the 'key set' as a proxy for 'this is the same type of thing' def _determine_order_by_value_key(obj_res: ET) -> Any: """ Returns either the class, or a tuple of the dictionary keys """ key = obj_res.__class__ - if key is dict: + if key == dict: # assuming same keys signify same way to determine ordering return tuple(obj_res.keys()) # type: ignore[union-attr] return key @@ -240,7 +232,7 @@ def _drop_unsorted(itr: Iterator[ET], orderfunc: OrderFunc) -> Iterator[ET]: # try getting the first value from the iterator # similar to my.core.common.warn_if_empty? this doesn't go through the whole iterator though -def _peek_iter(itr: Iterator[ET]) -> tuple[ET | None, Iterator[ET]]: +def _peek_iter(itr: Iterator[ET]) -> Tuple[Optional[ET], Iterator[ET]]: itr = more_itertools.peekable(itr) try: first_item = itr.peek() @@ -251,9 +243,9 @@ def _peek_iter(itr: Iterator[ET]) -> tuple[ET | None, Iterator[ET]]: # similar to 'my.core.error.sort_res_by'? -def _wrap_unsorted(itr: Iterator[ET], orderfunc: OrderFunc) -> tuple[Iterator[Unsortable], Iterator[ET]]: - unsortable: list[Unsortable] = [] - sortable: list[ET] = [] +def _wrap_unsorted(itr: Iterator[ET], orderfunc: OrderFunc) -> Tuple[Iterator[Unsortable], Iterator[ET]]: + unsortable: List[Unsortable] = [] + sortable: List[ET] = [] for o in itr: # if input to select was another select if isinstance(o, Unsortable): @@ -271,11 +263,10 @@ def _wrap_unsorted(itr: Iterator[ET], orderfunc: OrderFunc) -> tuple[Iterator[Un # the second being items for which orderfunc returned a non-none value def _handle_unsorted( itr: Iterator[ET], - *, orderfunc: OrderFunc, drop_unsorted: bool, wrap_unsorted: bool -) -> tuple[Iterator[Unsortable], Iterator[ET]]: +) -> Tuple[Iterator[Unsortable], Iterator[ET]]: # prefer drop_unsorted to wrap_unsorted, if both were present if drop_unsorted: return iter([]), _drop_unsorted(itr, orderfunc) @@ -290,16 +281,16 @@ def _handle_unsorted( # different types. ***This consumes the iterator***, so # you should definitely itertoolts.tee it beforehand # as to not exhaust the values -def _generate_order_value_func(itr: Iterator[ET], order_value: Where, default: U | None = None) -> OrderFunc: +def _generate_order_value_func(itr: Iterator[ET], order_value: Where, default: Optional[U] = None) -> OrderFunc: # TODO: add a kwarg to force lookup for every item? would sort of be like core.common.guess_datetime then - order_by_lookup: dict[Any, OrderFunc] = {} + order_by_lookup: Dict[Any, OrderFunc] = {} # need to go through a copy of the whole iterator here to # pre-generate functions to support sorting mixed types for obj_res in itr: key: Any = _determine_order_by_value_key(obj_res) if key not in order_by_lookup: - keyfunc: OrderFunc | None = _generate_order_by_func( + keyfunc: Optional[OrderFunc] = _generate_order_by_func( obj_res, where_function=order_value, default=default, @@ -320,12 +311,12 @@ def _generate_order_value_func(itr: Iterator[ET], order_value: Where, default: U def _handle_generate_order_by( itr, *, - order_by: OrderFunc | None = None, - order_key: str | None = None, - order_value: Where | None = None, - default: U | None = None, -) -> tuple[OrderFunc | None, Iterator[ET]]: - order_by_chosen: OrderFunc | None = order_by # if the user just supplied a function themselves + order_by: Optional[OrderFunc] = None, + order_key: Optional[str] = None, + order_value: Optional[Where] = None, + default: Optional[U] = None, +) -> Tuple[Optional[OrderFunc], Iterator[ET]]: + order_by_chosen: Optional[OrderFunc] = order_by # if the user just supplied a function themselves if order_by is not None: return order_by, itr if order_key is not None: @@ -350,19 +341,19 @@ def _handle_generate_order_by( def select( - src: Iterable[ET] | Callable[[], Iterable[ET]], + src: Union[Iterable[ET], Callable[[], Iterable[ET]]], *, - where: Where | None = None, - order_by: OrderFunc | None = None, - order_key: str | None = None, - order_value: Where | None = None, - default: U | None = None, + where: Optional[Where] = None, + order_by: Optional[OrderFunc] = None, + order_key: Optional[str] = None, + order_value: Optional[Where] = None, + default: Optional[U] = None, reverse: bool = False, - limit: int | None = None, + limit: Optional[int] = None, drop_unsorted: bool = False, wrap_unsorted: bool = True, warn_exceptions: bool = False, - warn_func: Callable[[Exception], None] | None = None, + warn_func: Optional[Callable[[Exception], None]] = None, drop_exceptions: bool = False, raise_exceptions: bool = False, ) -> Iterator[ET]: @@ -464,7 +455,7 @@ Will attempt to call iter() on the value""") try: itr: Iterator[ET] = iter(it) except TypeError as t: - raise QueryException("Could not convert input src to an Iterator: " + str(t)) # noqa: B904 + raise QueryException("Could not convert input src to an Iterator: " + str(t)) # if both drop_exceptions and drop_exceptions are provided for some reason, # should raise exceptions before dropping them @@ -501,12 +492,7 @@ Will attempt to call iter() on the value""") # note: can't just attach sort unsortable values in the same iterable as the # other items because they don't have any lookups for order_key or functions # to handle items in the order_by_lookup dictionary - unsortable, itr = _handle_unsorted( - itr, - orderfunc=order_by_chosen, - drop_unsorted=drop_unsorted, - wrap_unsorted=wrap_unsorted, - ) + unsortable, itr = _handle_unsorted(itr, order_by_chosen, drop_unsorted, wrap_unsorted) # run the sort, with the computed order by function itr = iter(sorted(itr, key=order_by_chosen, reverse=reverse)) # type: ignore[arg-type] @@ -597,7 +583,7 @@ def test_couldnt_determine_order() -> None: res = list(select(iter([object()]), order_value=lambda o: isinstance(o, datetime))) assert len(res) == 1 assert isinstance(res[0], Unsortable) - assert type(res[0].obj) is object + assert type(res[0].obj) == object # same value type, different keys, with clashing keys @@ -613,7 +599,7 @@ class _B(NamedTuple): # move these to tests/? They are re-used so much in the tests below, # not sure where the best place for these is -def _mixed_iter() -> Iterator[_A | _B]: +def _mixed_iter() -> Iterator[Union[_A, _B]]: yield _A(x=datetime(year=2009, month=5, day=10, hour=4, minute=10, second=1), y=5, z=10) yield _B(y=datetime(year=2015, month=5, day=10, hour=4, minute=10, second=1)) yield _A(x=datetime(year=2005, month=5, day=10, hour=4, minute=10, second=1), y=10, z=2) @@ -622,7 +608,7 @@ def _mixed_iter() -> Iterator[_A | _B]: yield _A(x=datetime(year=2005, month=4, day=10, hour=4, minute=10, second=1), y=2, z=-5) -def _mixed_iter_errors() -> Iterator[Res[_A | _B]]: +def _mixed_iter_errors() -> Iterator[Res[Union[_A, _B]]]: m = _mixed_iter() yield from itertools.islice(m, 0, 3) yield RuntimeError("Unhandled error!") @@ -658,7 +644,7 @@ def test_wrap_unsortable() -> None: # by default, wrap unsortable res = list(select(_mixed_iter(), order_key="z")) - assert Counter(type(t).__name__ for t in res) == Counter({"_A": 4, "Unsortable": 2}) + assert Counter(map(lambda t: type(t).__name__, res)) == Counter({"_A": 4, "Unsortable": 2}) def test_disabled_wrap_unsorted() -> None: @@ -677,7 +663,7 @@ def test_drop_unsorted() -> None: # test drop unsortable, should remove them before the 'sorted' call res = list(select(_mixed_iter(), order_key="z", wrap_unsorted=False, drop_unsorted=True)) assert len(res) == 4 - assert Counter(type(t).__name__ for t in res) == Counter({"_A": 4}) + assert Counter(map(lambda t: type(t).__name__, res)) == Counter({"_A": 4}) def test_drop_exceptions() -> None: @@ -701,16 +687,15 @@ def test_raise_exceptions() -> None: def test_wrap_unsortable_with_error_and_warning() -> None: - from collections import Counter - import pytest + from collections import Counter # by default should wrap unsortable (error) with pytest.warns(UserWarning, match=r"encountered exception"): res = list(select(_mixed_iter_errors(), order_value=lambda o: isinstance(o, datetime))) - assert Counter(type(t).__name__ for t in res) == Counter({"_A": 4, "_B": 2, "Unsortable": 1}) + assert Counter(map(lambda t: type(t).__name__, res)) == Counter({"_A": 4, "_B": 2, "Unsortable": 1}) # compare the returned error wrapped in the Unsortable - returned_error = next(o for o in res if isinstance(o, Unsortable)).obj + returned_error = next((o for o in res if isinstance(o, Unsortable))).obj assert "Unhandled error!" == str(returned_error) @@ -720,7 +705,7 @@ def test_order_key_unsortable() -> None: # both unsortable and items which dont match the order_by (order_key) in this case should be classified unsorted res = list(select(_mixed_iter_errors(), order_key="z")) - assert Counter(type(t).__name__ for t in res) == Counter({"_A": 4, "Unsortable": 3}) + assert Counter(map(lambda t: type(t).__name__, res)) == Counter({"_A": 4, "Unsortable": 3}) def test_order_default_param() -> None: @@ -740,7 +725,7 @@ def test_no_recursive_unsortables() -> None: # select to select as input, wrapping unsortables the first time, second should drop them # reverse=True to send errors to the end, so the below order_key works res = list(select(_mixed_iter_errors(), order_key="z", reverse=True)) - assert Counter(type(t).__name__ for t in res) == Counter({"_A": 4, "Unsortable": 3}) + assert Counter(map(lambda t: type(t).__name__, res)) == Counter({"_A": 4, "Unsortable": 3}) # drop_unsorted dropped = list(select(res, order_key="z", drop_unsorted=True)) diff --git a/my/core/query_range.py b/my/core/query_range.py index 83728bf..a1cfaed 100644 --- a/my/core/query_range.py +++ b/my/core/query_range.py @@ -7,30 +7,27 @@ filtered iterator See the select_range function below """ -from __future__ import annotations - import re import time -from collections.abc import Iterator -from datetime import date, datetime, timedelta -from functools import cache -from typing import Any, Callable, NamedTuple +from functools import lru_cache +from datetime import datetime, timedelta, date +from typing import Callable, Iterator, NamedTuple, Optional, Any, Type import more_itertools -from .compat import fromisoformat from .query import ( - ET, - OrderFunc, QueryException, + select, + OrderFunc, Where, _handle_generate_order_by, - select, + ET, ) -timedelta_regex = re.compile( - r"^((?P[\.\d]+?)w)?((?P[\.\d]+?)d)?((?P[\.\d]+?)h)?((?P[\.\d]+?)m)?((?P[\.\d]+?)s)?$" -) +from .common import isoparse + + +timedelta_regex = re.compile(r"^((?P[\.\d]+?)w)?((?P[\.\d]+?)d)?((?P[\.\d]+?)h)?((?P[\.\d]+?)m)?((?P[\.\d]+?)s)?$") # https://stackoverflow.com/a/51916936 @@ -81,7 +78,7 @@ def parse_datetime_float(date_str: str) -> float: except ValueError: pass try: - return fromisoformat(ds).timestamp() + return isoparse(ds).timestamp() except (AssertionError, ValueError): pass @@ -93,7 +90,7 @@ def parse_datetime_float(date_str: str) -> float: # dateparser is a bit more lenient than the above, lets you type # all sorts of dates as inputs # https://github.com/scrapinghub/dateparser#how-to-use - res: datetime | None = dateparser.parse(ds, settings={"DATE_ORDER": "YMD"}) + res: Optional[datetime] = dateparser.parse(ds, settings={"DATE_ORDER": "YMD"}) if res is not None: return res.timestamp() @@ -103,7 +100,7 @@ def parse_datetime_float(date_str: str) -> float: # probably DateLike input? but a user could specify an order_key # which is an epoch timestamp or a float value which they # expect to be converted to a datetime to compare -@cache +@lru_cache(maxsize=None) def _datelike_to_float(dl: Any) -> float: if isinstance(dl, datetime): return dl.timestamp() @@ -114,7 +111,7 @@ def _datelike_to_float(dl: Any) -> float: try: return parse_datetime_float(dl) except QueryException as q: - raise QueryException(f"While attempting to extract datetime from {dl}, to order by datetime:\n\n" + str(q)) # noqa: B904 + raise QueryException(f"While attempting to extract datetime from {dl}, to order by datetime:\n\n" + str(q)) class RangeTuple(NamedTuple): @@ -135,12 +132,11 @@ class RangeTuple(NamedTuple): of the timeframe -- 'before' - before and after - anything after 'after' and before 'before', acts as a time range """ - # technically doesn't need to be Optional[Any], # just to make it more clear these can be None - after: Any | None - before: Any | None - within: Any | None + after: Optional[Any] + before: Optional[Any] + within: Optional[Any] Converter = Callable[[Any], Any] @@ -151,9 +147,9 @@ def _parse_range( unparsed_range: RangeTuple, end_parser: Converter, within_parser: Converter, - parsed_range: RangeTuple | None = None, - error_message: str | None = None, -) -> RangeTuple | None: + parsed_range: Optional[RangeTuple] = None, + error_message: Optional[str] = None +) -> Optional[RangeTuple]: if parsed_range is not None: return parsed_range @@ -182,11 +178,11 @@ def _create_range_filter( end_parser: Converter, within_parser: Converter, attr_func: Where, - parsed_range: RangeTuple | None = None, - default_before: Any | None = None, - value_coercion_func: Converter | None = None, - error_message: str | None = None, -) -> Where | None: + parsed_range: Optional[RangeTuple] = None, + default_before: Optional[Any] = None, + value_coercion_func: Optional[Converter] = None, + error_message: Optional[str] = None, +) -> Optional[Where]: """ Handles: - parsing the user input into values that are comparable to items the iterable returns @@ -278,17 +274,17 @@ def _create_range_filter( def select_range( itr: Iterator[ET], *, - where: Where | None = None, - order_key: str | None = None, - order_value: Where | None = None, - order_by_value_type: type | None = None, - unparsed_range: RangeTuple | None = None, + where: Optional[Where] = None, + order_key: Optional[str] = None, + order_value: Optional[Where] = None, + order_by_value_type: Optional[Type] = None, + unparsed_range: Optional[RangeTuple] = None, reverse: bool = False, - limit: int | None = None, + limit: Optional[int] = None, drop_unsorted: bool = False, wrap_unsorted: bool = False, warn_exceptions: bool = False, - warn_func: Callable[[Exception], None] | None = None, + warn_func: Optional[Callable[[Exception], None]] = None, drop_exceptions: bool = False, raise_exceptions: bool = False, ) -> Iterator[ET]: @@ -323,22 +319,19 @@ def select_range( drop_exceptions=drop_exceptions, raise_exceptions=raise_exceptions, warn_exceptions=warn_exceptions, - warn_func=warn_func, - ) + warn_func=warn_func) - order_by_chosen: OrderFunc | None = None + order_by_chosen: Optional[OrderFunc] = None # if the user didn't specify an attribute to order value, but specified a type # we should search for on each value in the iterator if order_value is None and order_by_value_type is not None: # search for that type on the iterator object - order_value = lambda o: isinstance(o, order_by_value_type) + order_value = lambda o: isinstance(o, order_by_value_type) # type: ignore # if the user supplied a order_key, and/or we've generated an order_value, create # the function that accesses that type on each value in the iterator if order_key is not None or order_value is not None: - # _generate_order_value_func internally here creates a copy of the iterator, which has to - # be consumed in-case we're sorting by mixed types order_by_chosen, itr = _handle_generate_order_by(itr, order_key=order_key, order_value=order_value) # signifies that itr is empty -- can early return here if order_by_chosen is None: @@ -350,39 +343,37 @@ def select_range( if order_by_chosen is None: raise QueryException("""Can't order by range if we have no way to order_by! Specify a type or a key to order the value by""") - - # force drop_unsorted=True so we can use _create_range_filter - # sort the iterable by the generated order_by_chosen function - itr = select(itr, order_by=order_by_chosen, drop_unsorted=True) - filter_func: Where | None - if order_by_value_type in [datetime, date]: - filter_func = _create_range_filter( - unparsed_range=unparsed_range, - end_parser=parse_datetime_float, - within_parser=parse_timedelta_float, - attr_func=order_by_chosen, # type: ignore[arg-type] - default_before=time.time(), - value_coercion_func=_datelike_to_float, - ) - elif order_by_value_type in [int, float]: - # allow primitives to be converted using the default int(), float() callables - filter_func = _create_range_filter( - unparsed_range=unparsed_range, - end_parser=order_by_value_type, - within_parser=order_by_value_type, - attr_func=order_by_chosen, # type: ignore[arg-type] - default_before=None, - value_coercion_func=order_by_value_type, - ) else: - # TODO: add additional kwargs to let the user sort by other values, by specifying the parsers? - # would need to allow passing the end_parser, within parser, default before and value_coercion_func... - # (seems like a lot?) - raise QueryException("Sorting by custom types is currently unsupported") + # force drop_unsorted=True so we can use _create_range_filter + # sort the iterable by the generated order_by_chosen function + itr = select(itr, order_by=order_by_chosen, drop_unsorted=True) + filter_func: Optional[Where] + if order_by_value_type in [datetime, date]: + filter_func = _create_range_filter( + unparsed_range=unparsed_range, + end_parser=parse_datetime_float, + within_parser=parse_timedelta_float, + attr_func=order_by_chosen, # type: ignore[arg-type] + default_before=time.time(), + value_coercion_func=_datelike_to_float) + elif order_by_value_type in [int, float]: + # allow primitives to be converted using the default int(), float() callables + filter_func = _create_range_filter( + unparsed_range=unparsed_range, + end_parser=order_by_value_type, + within_parser=order_by_value_type, + attr_func=order_by_chosen, # type: ignore[arg-type] + default_before=None, + value_coercion_func=order_by_value_type) + else: + # TODO: add additional kwargs to let the user sort by other values, by specifying the parsers? + # would need to allow passing the end_parser, within parser, default before and value_coercion_func... + # (seems like a lot?) + raise QueryException("Sorting by custom types is currently unsupported") - # use the created filter function - # we've already applied drop_exceptions and kwargs related to unsortable values above - itr = select(itr, where=filter_func, limit=limit, reverse=reverse) + # use the created filter function + # we've already applied drop_exceptions and kwargs related to unsortable values above + itr = select(itr, where=filter_func, limit=limit, reverse=reverse) else: # wrap_unsorted may be used here if the user specified an order_key, # or manually passed a order_value function @@ -400,7 +391,7 @@ Specify a type or a key to order the value by""") return itr -# reuse items from query for testing +# re-use items from query for testing from .query import _A, _B, _Float, _mixed_iter_errors @@ -480,8 +471,8 @@ def test_range_predicate() -> None: ) # filter from 0 to 5 - rn: RangeTuple = RangeTuple("0", "5", None) - zero_to_five_filter: Where | None = int_filter_func(unparsed_range=rn) + rn: Optional[RangeTuple] = RangeTuple("0", "5", None) + zero_to_five_filter: Optional[Where] = int_filter_func(unparsed_range=rn) assert zero_to_five_filter is not None # this is just a Where function, given some input it return True/False if the value is allowed assert zero_to_five_filter(3) is True @@ -494,7 +485,6 @@ def test_range_predicate() -> None: rn = RangeTuple(None, 3, "3.5") assert list(filter(int_filter_func(unparsed_range=rn, attr_func=identity), src())) == ["0", "1", "2"] - def test_parse_range() -> None: from functools import partial @@ -538,8 +528,9 @@ def test_parse_timedelta_string() -> None: def test_parse_datetime_float() -> None: + pnow = parse_datetime_float("now") - sec_diff = abs(pnow - datetime.now().timestamp()) + sec_diff = abs((pnow - datetime.now().timestamp())) # should probably never fail? could mock time.time # but there seems to be issues with doing that use C-libraries (as time.time) does # https://docs.python.org/3/library/unittest.mock-examples.html#partial-mocking diff --git a/my/core/serialize.py b/my/core/serialize.py index e36da8f..c5f4cba 100644 --- a/my/core/serialize.py +++ b/my/core/serialize.py @@ -1,15 +1,12 @@ -from __future__ import annotations - import datetime -from dataclasses import asdict, is_dataclass -from decimal import Decimal -from functools import cache +import dataclasses from pathlib import Path -from typing import Any, Callable, NamedTuple +from decimal import Decimal +from typing import Any, Optional, Callable, NamedTuple +from functools import lru_cache +from .common import is_namedtuple from .error import error_to_json -from .pytest import parametrize -from .types import is_namedtuple # note: it would be nice to combine the 'asdict' and _default_encode to some function # that takes a complex python object and returns JSON-compatible fields, while still @@ -19,8 +16,6 @@ from .types import is_namedtuple DefaultEncoder = Callable[[Any], Any] -Dumps = Callable[[Any], str] - def _default_encode(obj: Any) -> Any: """ @@ -38,9 +33,8 @@ def _default_encode(obj: Any) -> Any: # convert paths to their string representation if isinstance(obj, Path): return str(obj) - if is_dataclass(obj): - assert not isinstance(obj, type) # to help mypy - return asdict(obj) + if dataclasses.is_dataclass(obj): + return dataclasses.asdict(obj) if isinstance(obj, Exception): return error_to_json(obj) # if something was stored as 'decimal', you likely @@ -59,12 +53,12 @@ def _default_encode(obj: Any) -> Any: # could possibly run multiple times/raise warning if you provide different 'default' # functions or change the kwargs? The alternative is to maintain all of this at the module # level, which is just as annoying -@cache +@lru_cache(maxsize=None) def _dumps_factory(**kwargs) -> Callable[[Any], str]: use_default: DefaultEncoder = _default_encode # if the user passed an additional 'default' parameter, # try using that to serialize before before _default_encode - _additional_default: DefaultEncoder | None = kwargs.get("default") + _additional_default: Optional[DefaultEncoder] = kwargs.get("default") if _additional_default is not None and callable(_additional_default): def wrapped_default(obj: Any) -> Any: @@ -80,29 +74,22 @@ def _dumps_factory(**kwargs) -> Callable[[Any], str]: kwargs["default"] = use_default - prefer_factory: str | None = kwargs.pop('_prefer_factory', None) - - def orjson_factory() -> Dumps | None: - try: - import orjson - except ModuleNotFoundError: - return None + try: + import orjson # todo: add orjson.OPT_NON_STR_KEYS? would require some bitwise ops # most keys are typically attributes from a NT/Dataclass, # so most seem to work: https://github.com/ijl/orjson#opt_non_str_keys - def _orjson_dumps(obj: Any) -> str: # TODO rename? + def _orjson_dumps(obj: Any) -> str: # orjson returns json as bytes, encode to string return orjson.dumps(obj, **kwargs).decode('utf-8') return _orjson_dumps + except ModuleNotFoundError: + pass - def simplejson_factory() -> Dumps | None: - try: - from simplejson import dumps as simplejson_dumps - except ModuleNotFoundError: - return None - + try: + from simplejson import dumps as simplejson_dumps # if orjson couldn't be imported, try simplejson # This is included for compatibility reasons because orjson # is rust-based and compiling on rarer architectures may not work @@ -117,42 +104,23 @@ def _dumps_factory(**kwargs) -> Callable[[Any], str]: return _simplejson_dumps - def stdlib_factory() -> Dumps | None: - import json + except ModuleNotFoundError: + pass - from .warnings import high + import json + from .warnings import high - high( - "You might want to install 'orjson' to support serialization for lots more types! If that does not work for you, you can install 'simplejson' instead" - ) + high("You might want to install 'orjson' to support serialization for lots more types! If that does not work for you, you can install 'simplejson' instead") - def _stdlib_dumps(obj: Any) -> str: - return json.dumps(obj, **kwargs) + def _stdlib_dumps(obj: Any) -> str: + return json.dumps(obj, **kwargs) - return _stdlib_dumps - - factories = { - 'orjson': orjson_factory, - 'simplejson': simplejson_factory, - 'stdlib': stdlib_factory, - } - - if prefer_factory is not None: - factory = factories[prefer_factory] - res = factory() - assert res is not None, prefer_factory - return res - - for factory in factories.values(): - res = factory() - if res is not None: - return res - raise RuntimeError("Should not happen!") + return _stdlib_dumps def dumps( obj: Any, - default: DefaultEncoder | None = None, + default: Optional[DefaultEncoder] = None, **kwargs, ) -> str: """ @@ -185,17 +153,8 @@ def dumps( return _dumps_factory(default=default, **kwargs)(obj) -@parametrize('factory', ['orjson', 'simplejson', 'stdlib']) -def test_dumps(factory: str) -> None: - import pytest - - orig_dumps = globals()['dumps'] # hack to prevent error from using local variable before declaring - - def dumps(*args, **kwargs) -> str: - kwargs['_prefer_factory'] = factory - return orig_dumps(*args, **kwargs) - - import json as json_builtin # dont cause possible conflicts with module code +def test_serialize_fallback() -> None: + import json as jsn # dont cause possible conflicts with module code # can't use a namedtuple here, since the default json.dump serializer # serializes namedtuples as tuples, which become arrays @@ -206,12 +165,36 @@ def test_dumps(factory: str) -> None: # the lru_cache'd warning may have already been sent, # so checking may be nondeterministic? import warnings - with warnings.catch_warnings(): warnings.simplefilter("ignore") - res = json_builtin.loads(dumps(X)) + res = jsn.loads(dumps(X)) assert res == [5, 5.0] + +# this needs to be defined here to prevent a mypy bug +# see https://github.com/python/mypy/issues/7281 +class _A(NamedTuple): + x: int + y: float + + +def test_nt_serialize() -> None: + import json as jsn # dont cause possible conflicts with module code + import orjson # import to make sure this is installed + + res: str = dumps(_A(x=1, y=2.0)) + assert res == '{"x":1,"y":2.0}' + + # test orjson option kwarg + data = {datetime.date(year=1970, month=1, day=1): 5} + res2 = jsn.loads(dumps(data, option=orjson.OPT_NON_STR_KEYS)) + assert res2 == {'1970-01-01': 5} + + +def test_default_serializer() -> None: + import pytest + import json as jsn # dont cause possible conflicts with module code + class Unserializable: def __init__(self, x: int): self.x = x @@ -225,7 +208,7 @@ def test_dumps(factory: str) -> None: def _serialize(self) -> Any: return {"x": self.x, "y": self.y} - res = json_builtin.loads(dumps(WithUnderscoreSerialize(6))) + res = jsn.loads(dumps(WithUnderscoreSerialize(6))) assert res == {"x": 6, "y": 6.0} # test passing additional 'default' func @@ -237,25 +220,5 @@ def test_dumps(factory: str) -> None: # this serializes both Unserializable, which is a custom type otherwise # not handled, and timedelta, which is handled by the '_default_encode' # in the 'wrapped_default' function - res2 = json_builtin.loads(dumps(Unserializable(10), default=_serialize_with_default)) + res2 = jsn.loads(dumps(Unserializable(10), default=_serialize_with_default)) assert res2 == {"x": 10, "y": 10.0} - - if factory == 'orjson': - import orjson - - # test orjson option kwarg - data = {datetime.date(year=1970, month=1, day=1): 5} - res2 = json_builtin.loads(dumps(data, option=orjson.OPT_NON_STR_KEYS)) - assert res2 == {'1970-01-01': 5} - - -@parametrize('factory', ['orjson', 'simplejson']) -def test_dumps_namedtuple(factory: str) -> None: - import json as json_builtin # dont cause possible conflicts with module code - - class _A(NamedTuple): - x: int - y: float - - res: str = dumps(_A(x=1, y=2.0), _prefer_factory=factory) - assert json_builtin.loads(res) == {'x': 1, 'y': 2.0} diff --git a/my/core/source.py b/my/core/source.py index a309d13..6d0f0fd 100644 --- a/my/core/source.py +++ b/my/core/source.py @@ -3,12 +3,9 @@ Decorator to gracefully handle importing a data source, or warning and yielding nothing (or a default) when its not available """ -from __future__ import annotations - -import warnings -from collections.abc import Iterable, Iterator from functools import wraps -from typing import Any, Callable, TypeVar +from typing import Any, Iterator, TypeVar, Callable, Optional, Iterable +import warnings from .warnings import medium @@ -29,8 +26,8 @@ _DEFAULT_ITR = () def import_source( *, default: Iterable[T] = _DEFAULT_ITR, - module_name: str | None = None, - help_url: str | None = None, + module_name: Optional[str] = None, + help_url: Optional[str] = None, ) -> Callable[..., Callable[..., Iterator[T]]]: """ doesn't really play well with types, but is used to catch @@ -53,7 +50,6 @@ def import_source( except (ImportError, AttributeError) as err: from . import core_config as CC from .error import warn_my_config_import_error - suppressed_in_conf = False if module_name is not None and CC.config._is_module_active(module_name) is False: suppressed_in_conf = True @@ -65,18 +61,16 @@ def import_source( warnings.warn(f"""If you don't want to use this module, to hide this message, add '{module_name}' to your core config disabled_modules in your config, like: class core: - disabled_modules = [{module_name!r}] -""", stacklevel=1) + disabled_modules = [{repr(module_name)}] +""") # try to check if this is a config error or based on dependencies not being installed if isinstance(err, (ImportError, AttributeError)): - matched_config_err = warn_my_config_import_error(err, module_name=module_name, help_url=help_url) + matched_config_err = warn_my_config_import_error(err, help_url=help_url) # if we determined this wasn't a config error, and it was an attribute error # it could be *any* attribute error -- we should raise this since its otherwise a fatal error # from some code in the module failing if not matched_config_err and isinstance(err, AttributeError): raise err yield from default - return wrapper - return decorator diff --git a/my/core/sqlite.py b/my/core/sqlite.py index 6167d2e..e04f6fc 100644 --- a/my/core/sqlite.py +++ b/my/core/sqlite.py @@ -1,19 +1,15 @@ -from __future__ import annotations +from .common import assert_subpackage; assert_subpackage(__name__) -from .internal import assert_subpackage # noqa: I001 -assert_subpackage(__name__) - -import shutil -import sqlite3 -from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path +import shutil +import sqlite3 from tempfile import TemporaryDirectory -from typing import Any, Callable, Literal, Union, overload +from typing import Tuple, Any, Iterator, Callable, Optional, Union, Literal -from .common import PathIsh -from .compat import assert_never + +from .common import PathIsh, assert_never def sqlite_connect_immutable(db: PathIsh) -> sqlite3.Connection: @@ -26,7 +22,6 @@ def test_sqlite_connect_immutable(tmp_path: Path) -> None: conn.execute('CREATE TABLE testtable (col)') import pytest - with pytest.raises(sqlite3.OperationalError, match='readonly database'): with sqlite_connect_immutable(db) as conn: conn.execute('DROP TABLE testtable') @@ -38,17 +33,15 @@ def test_sqlite_connect_immutable(tmp_path: Path) -> None: SqliteRowFactory = Callable[[sqlite3.Cursor, sqlite3.Row], Any] - def dict_factory(cursor, row): fields = [column[0] for column in cursor.description] - return dict(zip(fields, row)) + return {key: value for key, value in zip(fields, row)} Factory = Union[SqliteRowFactory, Literal['row', 'dict']] - @contextmanager -def sqlite_connection(db: PathIsh, *, immutable: bool = False, row_factory: Factory | None = None) -> Iterator[sqlite3.Connection]: +def sqlite_connection(db: PathIsh, *, immutable: bool=False, row_factory: Optional[Factory]=None) -> Iterator[sqlite3.Connection]: dbp = f'file:{db}' # https://www.sqlite.org/draft/uri.html#uriimmutable if immutable: @@ -104,76 +97,32 @@ def sqlite_copy_and_open(db: PathIsh) -> sqlite3.Connection: # and then the return type ends up as Iterator[Tuple[str, ...]], which isn't desirable :( # a bit annoying to have this copy-pasting, but hopefully not a big issue -# fmt: off +from typing import overload @overload -def select(cols: tuple[str ], rest: str, *, db: sqlite3.Connection) -> \ - Iterator[tuple[Any ]]: ... +def select(cols: Tuple[str ], rest: str, *, db: sqlite3.Connection) -> \ + Iterator[Tuple[Any ]]: ... @overload -def select(cols: tuple[str, str ], rest: str, *, db: sqlite3.Connection) -> \ - Iterator[tuple[Any, Any ]]: ... +def select(cols: Tuple[str, str ], rest: str, *, db: sqlite3.Connection) -> \ + Iterator[Tuple[Any, Any ]]: ... @overload -def select(cols: tuple[str, str, str ], rest: str, *, db: sqlite3.Connection) -> \ - Iterator[tuple[Any, Any, Any ]]: ... +def select(cols: Tuple[str, str, str ], rest: str, *, db: sqlite3.Connection) -> \ + Iterator[Tuple[Any, Any, Any ]]: ... @overload -def select(cols: tuple[str, str, str, str ], rest: str, *, db: sqlite3.Connection) -> \ - Iterator[tuple[Any, Any, Any, Any ]]: ... +def select(cols: Tuple[str, str, str, str ], rest: str, *, db: sqlite3.Connection) -> \ + Iterator[Tuple[Any, Any, Any, Any ]]: ... @overload -def select(cols: tuple[str, str, str, str, str ], rest: str, *, db: sqlite3.Connection) -> \ - Iterator[tuple[Any, Any, Any, Any, Any ]]: ... +def select(cols: Tuple[str, str, str, str, str ], rest: str, *, db: sqlite3.Connection) -> \ + Iterator[Tuple[Any, Any, Any, Any, Any ]]: ... @overload -def select(cols: tuple[str, str, str, str, str, str ], rest: str, *, db: sqlite3.Connection) -> \ - Iterator[tuple[Any, Any, Any, Any, Any, Any ]]: ... +def select(cols: Tuple[str, str, str, str, str, str ], rest: str, *, db: sqlite3.Connection) -> \ + Iterator[Tuple[Any, Any, Any, Any, Any, Any ]]: ... @overload -def select(cols: tuple[str, str, str, str, str, str, str ], rest: str, *, db: sqlite3.Connection) -> \ - Iterator[tuple[Any, Any, Any, Any, Any, Any, Any ]]: ... +def select(cols: Tuple[str, str, str, str, str, str, str ], rest: str, *, db: sqlite3.Connection) -> \ + Iterator[Tuple[Any, Any, Any, Any, Any, Any, Any ]]: ... @overload -def select(cols: tuple[str, str, str, str, str, str, str, str], rest: str, *, db: sqlite3.Connection) -> \ - Iterator[tuple[Any, Any, Any, Any, Any, Any, Any, Any]]: ... -# fmt: on +def select(cols: Tuple[str, str, str, str, str, str, str, str], rest: str, *, db: sqlite3.Connection) -> \ + Iterator[Tuple[Any, Any, Any, Any, Any, Any, Any, Any]]: ... def select(cols, rest, *, db): # db arg is last cause that results in nicer code formatting.. return db.execute('SELECT ' + ','.join(cols) + ' ' + rest) - - -class SqliteTool: - def __init__(self, connection: sqlite3.Connection) -> None: - self.connection = connection - - def _get_sqlite_master(self) -> dict[str, str]: - res = {} - for c in self.connection.execute('SELECT name, type FROM sqlite_master'): - [name, type_] = c - assert type_ in {'table', 'index', 'view', 'trigger'}, (name, type_) # just in case - res[name] = type_ - return res - - def get_table_names(self) -> list[str]: - master = self._get_sqlite_master() - res = [] - for name, type_ in master.items(): - if type_ != 'table': - continue - res.append(name) - return res - - def get_table_schema(self, name: str) -> dict[str, str]: - """ - Returns map from column name to column type - - NOTE: Sometimes this doesn't work if the db has some extensions (e.g. happens for facebook apps) - In this case you might still be able to use get_table_names - """ - schema: dict[str, str] = {} - for row in self.connection.execute(f'PRAGMA table_info(`{name}`)'): - col = row[1] - type_ = row[2] - # hmm, somewhere between 3.34.1 and 3.37.2, sqlite started normalising type names to uppercase - # let's do this just in case since python < 3.10 are using the old version - # e.g. it could have returned 'blob' and that would confuse blob check (see _check_allowed_blobs) - type_ = type_.upper() - schema[col] = type_ - return schema - - def get_table_schemas(self) -> dict[str, dict[str, str]]: - return {name: self.get_table_schema(name) for name in self.get_table_names()} diff --git a/my/core/stats.py b/my/core/stats.py index a553db3..44735b8 100644 --- a/my/core/stats.py +++ b/my/core/stats.py @@ -1,178 +1,23 @@ ''' Helpers for hpi doctor/stats functionality. ''' - -from __future__ import annotations - -import collections.abc +import collections import importlib import inspect import typing -from collections.abc import Iterable, Iterator, Sequence -from contextlib import contextmanager -from datetime import datetime -from pathlib import Path -from types import ModuleType -from typing import ( - Any, - Callable, - Protocol, - cast, -) +from typing import Optional, Callable, Any, Iterator, Sequence, Dict, List -from .types import asdict - -Stats = dict[str, Any] - - -class StatsFun(Protocol): - def __call__(self, *, quick: bool = False) -> Stats: ... - - -# global state that turns on/off quick stats -# can use the 'quick_stats' contextmanager -# to enable/disable this in cli so that module 'stats' -# functions don't have to implement custom 'quick' logic -QUICK_STATS = False - - -# in case user wants to use the stats functions/quick option -# elsewhere -- can use this decorator instead of editing -# the global state directly -@contextmanager -def quick_stats(): - global QUICK_STATS - prev = QUICK_STATS - try: - QUICK_STATS = True - yield - finally: - QUICK_STATS = prev - - -def stat( - func: Callable[[], Iterable[Any]] | Iterable[Any], - *, - quick: bool = False, - name: str | None = None, -) -> Stats: - """ - Extracts various statistics from a passed iterable/callable, e.g.: - - number of items - - first/last item - - timestamps associated with first/last item - - If quick is set, then only first 100 items of the iterable will be processed - """ - if callable(func): - fr = func() - if hasattr(fr, '__enter__') and hasattr(fr, '__exit__'): - # context managers has Iterable type, but they aren't data providers - # sadly doesn't look like there is a way to tell from typing annotations - # Ideally we'd detect this in is_data_provider... - # but there is no way of knowing without actually calling it first :( - return {} - fname = func.__name__ - else: - # meh. means it's just a list.. not sure how to generate a name then - fr = func - fname = f'unnamed_{id(fr)}' - type_name = type(fr).__name__ - extras = {} - if type_name == 'DataFrame': - # dynamic, because pandas is an optional dependency.. - df = cast(Any, fr) # todo ugh, not sure how to annotate properly - df = df.reset_index() - - fr = df.to_dict(orient='records') - - dtypes = df.dtypes.to_dict() - extras['dtypes'] = dtypes - - res = _stat_iterable(fr, quick=quick) - res.update(extras) - - stat_name = name if name is not None else fname - return { - stat_name: res, - } - - -def test_stat() -> None: - # the bulk of testing is in test_stat_iterable - - # works with 'anonymous' lists - res = stat([1, 2, 3]) - [(name, v)] = res.items() - # note: name will be a little funny since anonymous list doesn't have one - assert v == {'count': 3} - # - - # works with functions: - def fun(): - return [4, 5, 6] - - assert stat(fun) == {'fun': {'count': 3}} - # - - # context managers are technically iterable - # , but usually we wouldn't want to compute stats for them - # this is mainly intended for guess_stats, - # since it can't tell whether the function is a ctx manager without calling it - @contextmanager - def cm(): - yield 1 - yield 3 - - assert stat(cm) == {} # type: ignore[arg-type] - # - - # works with pandas dataframes - import numpy as np - import pandas as pd - - def df() -> pd.DataFrame: - dates = pd.date_range(start='2024-02-10 08:00', end='2024-02-11 16:00', freq='5h') - return pd.DataFrame([f'value{i}' for i, _ in enumerate(dates)], index=dates, columns=['value']) - - assert stat(df) == { - 'df': { - 'count': 7, - 'dtypes': { - 'index': np.dtype(' StatsFun | None: - stats: StatsFun | None = None - try: - module = importlib.import_module(module_name) - except Exception: - return None - stats = getattr(module, 'stats', None) - if stats is None: - stats = guess_stats(module) - return stats +from .common import StatsFun, Stats, stat # TODO maybe could be enough to annotate OUTPUTS or something like that? # then stats could just use them as hints? -def guess_stats(module: ModuleType) -> StatsFun | None: - """ - If the module doesn't have explicitly defined 'stat' function, - this is used to try to guess what could be included in stats automatically - """ - providers = _guess_data_providers(module) +def guess_stats(module_name: str, quick: bool = False) -> Optional[StatsFun]: + providers = guess_data_providers(module_name) if len(providers) == 0: return None - def auto_stats(*, quick: bool = False) -> Stats: + def auto_stats() -> Stats: res = {} for k, v in providers.items(): res.update(stat(v, quick=quick, name=k)) @@ -182,11 +27,12 @@ def guess_stats(module: ModuleType) -> StatsFun | None: def test_guess_stats() -> None: + from datetime import datetime import my.core.tests.auto_stats as M - auto_stats = guess_stats(M) + auto_stats = guess_stats(M.__name__) assert auto_stats is not None - res = auto_stats(quick=False) + res = auto_stats() assert res == { 'inputs': { @@ -202,15 +48,15 @@ def test_guess_stats() -> None: } -def _guess_data_providers(module: ModuleType) -> dict[str, Callable]: +def guess_data_providers(module_name: str) -> Dict[str, Callable]: + module = importlib.import_module(module_name) mfunctions = inspect.getmembers(module, inspect.isfunction) return {k: v for k, v in mfunctions if is_data_provider(v)} -# todo how to exclude deprecated data providers? +# todo how to exclude deprecated stuff? def is_data_provider(fun: Any) -> bool: """ - Criteria for being a "data provider": 1. returns iterable or something like that 2. takes no arguments? (otherwise not callable by stats anyway?) 3. doesn't start with an underscore (those are probably helper functions?) @@ -226,7 +72,7 @@ def is_data_provider(fun: Any) -> bool: return False # has at least one argument without default values - if len(list(_sig_required_params(sig))) > 0: + if len(list(sig_required_params(sig))) > 0: return False if hasattr(fun, '__name__'): @@ -242,7 +88,7 @@ def is_data_provider(fun: Any) -> bool: if return_type is None: return False - return _type_is_iterable(return_type) + return type_is_iterable(return_type) def test_is_data_provider() -> None: @@ -253,42 +99,34 @@ def test_is_data_provider() -> None: def no_return_type(): return [1, 2, 3] - assert not idp(no_return_type) lam = lambda: [1, 2] assert not idp(lam) - def has_extra_args(count) -> list[int]: + def has_extra_args(count) -> List[int]: return list(range(count)) - assert not idp(has_extra_args) def has_return_type() -> Sequence[str]: return ['a', 'b', 'c'] - assert idp(has_return_type) def _helper_func() -> Iterator[Any]: yield 1 - assert not idp(_helper_func) def inputs() -> Iterator[Any]: yield 1 - assert idp(inputs) def producer_inputs() -> Iterator[Any]: yield 1 - assert idp(producer_inputs) -def _sig_required_params(sig: inspect.Signature) -> Iterator[inspect.Parameter]: - """ - Returns parameters the user is required to provide - e.g. ones that don't have default values - """ +# return any parameters the user is required to provide - those which don't have default values +def sig_required_params(sig: inspect.Signature) -> Iterator[inspect.Parameter]: for param in sig.parameters.values(): if param.default == inspect.Parameter.empty: yield param @@ -298,24 +136,21 @@ def test_sig_required_params() -> None: def x() -> int: return 5 - - assert len(list(_sig_required_params(inspect.signature(x)))) == 0 + assert len(list(sig_required_params(inspect.signature(x)))) == 0 def y(arg: int) -> int: return arg - - assert len(list(_sig_required_params(inspect.signature(y)))) == 1 + assert len(list(sig_required_params(inspect.signature(y)))) == 1 # from stats perspective, this should be treated as a data provider as well # could be that the default value to the data provider is the 'default' # path to use for inputs/a function to provide input data def z(arg: int = 5) -> int: return arg - - assert len(list(_sig_required_params(inspect.signature(z)))) == 0 + assert len(list(sig_required_params(inspect.signature(z)))) == 0 -def _type_is_iterable(type_spec) -> bool: +def type_is_iterable(type_spec) -> bool: origin = typing.get_origin(type_spec) if origin is None: return False @@ -332,139 +167,14 @@ def _type_is_iterable(type_spec) -> bool: # todo docstring test? def test_type_is_iterable() -> None: - fun = _type_is_iterable + from typing import List, Sequence, Iterable, Dict, Any + + fun = type_is_iterable assert not fun(None) assert not fun(int) assert not fun(Any) - assert not fun(dict[int, int]) + assert not fun(Dict[int, int]) - assert fun(list[int]) - assert fun(Sequence[dict[str, str]]) + assert fun(List[int]) + assert fun(Sequence[Dict[str, str]]) assert fun(Iterable[Any]) - - -def _stat_item(item): - if item is None: - return None - if isinstance(item, Path): - return str(item) - return _guess_datetime(item) - - -def _stat_iterable(it: Iterable[Any], *, quick: bool = False) -> Stats: - from more_itertools import first, ilen, take - - # todo not sure if there is something in more_itertools to compute this? - total = 0 - errors = 0 - first_item = None - last_item = None - - def funcit(): - nonlocal errors, first_item, last_item, total - for x in it: - total += 1 - if isinstance(x, Exception): - errors += 1 - else: - last_item = x - if first_item is None: - first_item = x - yield x - - eit = funcit() - count: Any - if quick or QUICK_STATS: - initial = take(100, eit) - count = len(initial) - if first(eit, None) is not None: # todo can actually be none... - # haven't exhausted - count = f'{count}+' - else: - count = ilen(eit) - - res = { - 'count': count, - } - - if total == 0: - # not sure but I guess a good balance? wouldn't want to throw early here? - res['warning'] = 'THE ITERABLE RETURNED NO DATA' - - if errors > 0: - res['errors'] = errors - - if (stat_first := _stat_item(first_item)) is not None: - res['first'] = stat_first - - if (stat_last := _stat_item(last_item)) is not None: - res['last'] = stat_last - - return res - - -def test_stat_iterable() -> None: - from datetime import datetime, timedelta, timezone - from typing import NamedTuple - - dd = datetime.fromtimestamp(123, tz=timezone.utc) - day = timedelta(days=3) - - class X(NamedTuple): - x: int - d: datetime - - def it(): - yield RuntimeError('oops!') - for i in range(2): - yield X(x=i, d=dd + day * i) - yield RuntimeError('bad!') - for i in range(3): - yield X(x=i * 10, d=dd + day * (i * 10)) - yield X(x=123, d=dd + day * 50) - - res = _stat_iterable(it()) - assert res['count'] == 1 + 2 + 1 + 3 + 1 - assert res['errors'] == 1 + 1 - assert res['last'] == dd + day * 50 - - -# experimental, not sure about it.. -def _guess_datetime(x: Any) -> datetime | None: - # todo hmm implement without exception.. - try: - d = asdict(x) - except: # noqa: E722 bare except - return None - for v in d.values(): - if isinstance(v, datetime): - return v - return None - - -def test_guess_datetime() -> None: - from dataclasses import dataclass - from typing import NamedTuple - - from .compat import fromisoformat - - dd = fromisoformat('2021-02-01T12:34:56Z') - - class A(NamedTuple): - x: int - - class B(NamedTuple): - x: int - created: datetime - - assert _guess_datetime(A(x=4)) is None - assert _guess_datetime(B(x=4, created=dd)) == dd - - @dataclass - class C: - a: datetime - x: int - - assert _guess_datetime(C(a=dd, x=435)) == dd - # TODO not sure what to return when multiple datetime fields? - # TODO test @property? diff --git a/my/core/structure.py b/my/core/structure.py index bb049e4..7a0c2a2 100644 --- a/my/core/structure.py +++ b/my/core/structure.py @@ -1,22 +1,20 @@ -from __future__ import annotations - -import atexit import os import shutil -import sys -import tarfile import tempfile import zipfile -from collections.abc import Generator, Sequence +import atexit + +from typing import Sequence, Generator, List, Union, Tuple from contextlib import contextmanager from pathlib import Path -from .logging import make_logger - -logger = make_logger(__name__, level="info") +from .common import LazyLogger -def _structure_exists(base_dir: Path, paths: Sequence[str], *, partial: bool = False) -> bool: +logger = LazyLogger(__name__, level="info") + + +def _structure_exists(base_dir: Path, paths: Sequence[str], partial: bool = False) -> bool: """ Helper function for match_structure to check if all subpaths exist at some base directory @@ -38,18 +36,17 @@ def _structure_exists(base_dir: Path, paths: Sequence[str], *, partial: bool = F ZIP_EXT = {".zip"} -TARGZ_EXT = {".tar.gz"} @contextmanager def match_structure( base: Path, - expected: str | Sequence[str], + expected: Union[str, Sequence[str]], *, partial: bool = False, -) -> Generator[tuple[Path, ...], None, None]: +) -> Generator[Tuple[Path, ...], None, None]: """ - Given a 'base' directory or archive (zip/tar.gz), recursively search for one or more paths that match the + Given a 'base' directory or zipfile, recursively search for one or more paths that match the pattern described in 'expected'. That can be a single string, or a list of relative paths (as strings) you expect at the same directory. @@ -57,12 +54,12 @@ def match_structure( expected be present, not all of them. This reduces the chances of the user misconfiguring gdpr exports, e.g. - if they archived the folders instead of the parent directory or vice-versa + if they zipped the folders instead of the parent directory or vice-versa When this finds a matching directory structure, it stops searching in that subdirectory and continues onto other possible subdirectories which could match - If base is an archive, this extracts it into a temporary directory + If base is a zipfile, this extracts the zipfile into a temporary directory (configured by core_config.config.get_tmp_dir), and then searches the extracted folder for matching structures @@ -72,21 +69,21 @@ def match_structure( export_dir ├── exp_2020 - │ ├── channel_data - │ │ ├── data1 - │ │ └── data2 - │ ├── index.json - │ ├── messages - │ │ └── messages.csv - │ └── profile - │ └── settings.json + │   ├── channel_data + │   │   ├── data1 + │   │   └── data2 + │   ├── index.json + │   ├── messages + │   │   └── messages.csv + │   └── profile + │   └── settings.json └── exp_2021 ├── channel_data - │ ├── data1 - │ └── data2 + │   ├── data1 + │   └── data2 ├── index.json ├── messages - │ └── messages.csv + │   └── messages.csv └── profile └── settings.json @@ -98,12 +95,12 @@ def match_structure( This doesn't require an exhaustive list of expected values, but its a good idea to supply a complete picture of the expected structure to avoid false-positives - This does not recursively decompress archives in the subdirectories, - it only unpacks into a temporary directory if 'base' is an archive + This does not recursively unzip zipfiles in the subdirectories, + it only unzips into a temporary directory if 'base' is a zipfile A common pattern for using this might be to use get_files to get a list - of archives or top-level gdpr export directories, and use match_structure - to search the resulting paths for an export structure you're expecting + of zipfiles or top-level gdpr export directories, and use match_structure + to search the resulting paths for a export structure you're expecting """ from . import core_config as CC @@ -113,37 +110,29 @@ def match_structure( expected = (expected,) is_zip: bool = base.suffix in ZIP_EXT - is_targz: bool = any(base.name.endswith(suffix) for suffix in TARGZ_EXT) searchdir: Path = base.absolute() try: - # if the file given by the user is an archive, create a temporary - # directory and extract it to that temporary directory + # if the file given by the user is a zipfile, create a temporary + # directory and extract the zipfile to that temporary directory # # this temporary directory is removed in the finally block - if is_zip or is_targz: + if is_zip: # sanity check before we start creating directories/rm-tree'ing things - assert base.exists(), f"archive at {base} doesn't exist" + assert base.exists(), f"zipfile at {base} doesn't exist" searchdir = Path(tempfile.mkdtemp(dir=tdir)) - if is_zip: - # base might already be a ZipPath, and str(base) would end with / - zf = zipfile.ZipFile(str(base).rstrip('/')) - zf.extractall(path=str(searchdir)) - elif is_targz: - with tarfile.open(str(base)) as tar: - # filter is a security feature, will be required param in later python version - mfilter = {'filter': 'data'} if sys.version_info[:2] >= (3, 12) else {} - tar.extractall(path=str(searchdir), **mfilter) # type: ignore[arg-type] - else: - raise RuntimeError("can't happen") + # base might already be a ZipPath, and str(base) would end with / + zf = zipfile.ZipFile(str(base).rstrip('/')) + zf.extractall(path=str(searchdir)) + else: if not searchdir.is_dir(): - raise NotADirectoryError(f"Expected either a zip/tar.gz archive or a directory, received {searchdir}") + raise NotADirectoryError(f"Expected either a zipfile or a directory, received {searchdir}") - matches: list[Path] = [] - possible_targets: list[Path] = [searchdir] + matches: List[Path] = [] + possible_targets: List[Path] = [searchdir] while len(possible_targets) > 0: p = possible_targets.pop(0) @@ -163,9 +152,9 @@ def match_structure( finally: - if is_zip or is_targz: + if is_zip: # make sure we're not mistakenly deleting data - assert str(searchdir).startswith(str(tdir)), f"Expected the temporary directory for extracting archive to start with the temporary directory prefix ({tdir}), found {searchdir}" + assert str(searchdir).startswith(str(tdir)), f"Expected the temporary directory for extracting zip to start with the temporary directory prefix ({tdir}), found {searchdir}" shutil.rmtree(str(searchdir)) @@ -174,7 +163,7 @@ def warn_leftover_files() -> None: from . import core_config as CC base_tmp: Path = CC.config.get_tmp_dir() - leftover: list[Path] = list(base_tmp.iterdir()) + leftover: List[Path] = list(base_tmp.iterdir()) if leftover: logger.debug(f"at exit warning: Found leftover files in temporary directory '{leftover}'. this may be because you have multiple hpi processes running -- if so this can be ignored") diff --git a/my/core/tests/auto_stats.py b/my/core/tests/auto_stats.py index fc49e03..2c09b5b 100644 --- a/my/core/tests/auto_stats.py +++ b/my/core/tests/auto_stats.py @@ -1,12 +1,11 @@ """ Helper 'module' for test_guess_stats """ - -from collections.abc import Iterable, Iterator, Sequence from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path +from typing import Iterable, Sequence, Iterator @dataclass diff --git a/my/core/tests/common.py b/my/core/tests/common.py deleted file mode 100644 index 073ea5f..0000000 --- a/my/core/tests/common.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -import os -from collections.abc import Iterator -from contextlib import contextmanager - -import pytest - -V = 'HPI_TESTS_USES_OPTIONAL_DEPS' - -# TODO use it for serialize tests that are using simplejson/orjson? -skip_if_uses_optional_deps = pytest.mark.skipif( - V not in os.environ, - reason=f'test only works when optional dependencies are installed. Set env variable {V}=true to override.', -) - - -# TODO maybe move to hpi core? -@contextmanager -def tmp_environ_set(key: str, value: str | None) -> Iterator[None]: - prev_value = os.environ.get(key) - if value is None: - os.environ.pop(key, None) - else: - os.environ[key] = value - try: - yield - finally: - if prev_value is None: - os.environ.pop(key, None) - else: - os.environ[key] = prev_value diff --git a/my/core/tests/denylist.py b/my/core/tests/denylist.py index 73c3165..cca757d 100644 --- a/my/core/tests/denylist.py +++ b/my/core/tests/denylist.py @@ -1,9 +1,8 @@ -import json -import warnings -from collections.abc import Iterator from datetime import datetime +import json from pathlib import Path -from typing import NamedTuple +from typing import NamedTuple, Iterator +import warnings from ..denylist import DenyList @@ -92,7 +91,8 @@ def test_denylist(tmp_path: Path) -> None: assert "59.40.113.87" not in [i.addr for i in filtered] - data_json = json.loads(tf.read_text()) + with open(tf, "r") as f: + data_json = json.loads(f.read()) assert data_json == [ { diff --git a/my/core/tests/sqlite.py b/my/core/tests/sqlite.py index 1ad0748..b3ecffe 100644 --- a/my/core/tests/sqlite.py +++ b/my/core/tests/sqlite.py @@ -1,7 +1,7 @@ -import shutil -import sqlite3 from concurrent.futures import ProcessPoolExecutor from pathlib import Path +import shutil +import sqlite3 from tempfile import TemporaryDirectory from ..sqlite import sqlite_connect_immutable, sqlite_copy_and_open diff --git a/my/core/tests/structure.py b/my/core/tests/structure.py index 741e0ea..beb8e7f 100644 --- a/my/core/tests/structure.py +++ b/my/core/tests/structure.py @@ -1,8 +1,9 @@ from pathlib import Path +from ..structure import match_structure + import pytest -from ..structure import match_structure structure_data: Path = Path(__file__).parent / "structure_data" @@ -14,9 +15,8 @@ def test_gdpr_structure_exists() -> None: assert results == (structure_data / "gdpr_subdirs" / "gdpr_export",) -@pytest.mark.parametrize("archive", ["gdpr_export.zip", "gdpr_export.tar.gz"]) -def test_gdpr_unpack(archive: str) -> None: - with match_structure(structure_data / archive, expected=gdpr_expected) as results: +def test_gdpr_unzip() -> None: + with match_structure(structure_data / "gdpr_export.zip", expected=gdpr_expected) as results: assert len(results) == 1 extracted = results[0] index_file = extracted / "messages" / "index.csv" @@ -33,6 +33,6 @@ def test_match_partial() -> None: def test_not_directory() -> None: - with pytest.raises(NotADirectoryError, match=r"Expected either a zip/tar.gz archive or a directory"): + with pytest.raises(NotADirectoryError, match=r"Expected either a zipfile or a directory"): with match_structure(structure_data / "messages/index.csv", expected=gdpr_expected): pass diff --git a/my/core/tests/structure_data/gdpr_export.tar.gz b/my/core/tests/structure_data/gdpr_export.tar.gz deleted file mode 100644 index 4f0597c..0000000 Binary files a/my/core/tests/structure_data/gdpr_export.tar.gz and /dev/null differ diff --git a/my/core/tests/test_cachew.py b/my/core/tests/test_cachew.py deleted file mode 100644 index a0d2267..0000000 --- a/my/core/tests/test_cachew.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -from .common import skip_if_uses_optional_deps as pytestmark - -# TODO ugh, this is very messy.. need to sort out config overriding here - - -def test_cachew() -> None: - from cachew import settings - - settings.ENABLE = True # by default it's off in tests (see conftest.py) - - from my.core.cachew import mcachew - - called = 0 - - # TODO ugh. need doublewrap or something to avoid having to pass parens - @mcachew() - def cf() -> list[int]: - nonlocal called - called += 1 - return [1, 2, 3] - - list(cf()) - cc = called - # todo ugh. how to clean cache? - # assert called == 1 # precondition, to avoid turdes from previous tests - - assert list(cf()) == [1, 2, 3] - assert called == cc - - -def test_cachew_dir_none() -> None: - from cachew import settings - - settings.ENABLE = True # by default it's off in tests (see conftest.py) - - from my.core.cachew import cache_dir, mcachew - from my.core.core_config import _reset_config as reset - - with reset() as cc: - cc.cache_dir = None - called = 0 - - @mcachew(cache_path=cache_dir() / 'ctest') - def cf() -> list[int]: - nonlocal called - called += 1 - return [called, called, called] - - assert list(cf()) == [1, 1, 1] - assert list(cf()) == [2, 2, 2] diff --git a/my/core/tests/test_cli.py b/my/core/tests/test_cli.py index 1838e84..4d847ae 100644 --- a/my/core/tests/test_cli.py +++ b/my/core/tests/test_cli.py @@ -1,6 +1,6 @@ import os -import sys from subprocess import check_call +import sys def test_lists_modules() -> None: diff --git a/my/core/tests/test_config.py b/my/core/tests/test_config.py deleted file mode 100644 index f6d12ba..0000000 --- a/my/core/tests/test_config.py +++ /dev/null @@ -1,178 +0,0 @@ -""" -Various tests that are checking behaviour of user config wrt to various things -""" - -import os -import sys -from pathlib import Path - -import pytest -import pytz - -import my.config -from my.core import notnone -from my.demo import items, make_config - -from .common import tmp_environ_set - -# TODO would be nice to randomize test order here to catch various config issues - - -# run the same test multiple times to make sure there are not issues with import order etc -@pytest.mark.parametrize('run_id', ['1', '2']) -def test_override_config(tmp_path: Path, run_id: str) -> None: - class user_config: - username = f'user_{run_id}' - data_path = f'{tmp_path}/*.json' - - my.config.demo = user_config # type: ignore[misc, assignment] - - [item1, item2] = items() - assert item1.username == f'user_{run_id}' - assert item2.username == f'user_{run_id}' - - -@pytest.mark.skip(reason="won't work at the moment because of inheritance") -def test_dynamic_config_simplenamespace(tmp_path: Path) -> None: - from types import SimpleNamespace - - user_config = SimpleNamespace( - username='user3', - data_path=f'{tmp_path}/*.json', - ) - my.config.demo = user_config # type: ignore[misc, assignment] - - cfg = make_config() - - assert cfg.username == 'user3' - - -def test_mixin_attribute_handling(tmp_path: Path) -> None: - """ - Tests that arbitrary mixin attributes work with our config handling pattern - """ - - nytz = pytz.timezone('America/New_York') - - class user_config: - # check that override is taken into the account - timezone = nytz - - irrelevant = 'hello' - - username = 'UUU' - data_path = f'{tmp_path}/*.json' - - my.config.demo = user_config # type: ignore[misc, assignment] - - cfg = make_config() - - assert cfg.username == 'UUU' - - # mypy doesn't know about it, but the attribute is there - assert getattr(cfg, 'irrelevant') == 'hello' - - # check that overridden default attribute is actually getting overridden - assert cfg.timezone == nytz - - [item1, item2] = items() - assert item1.username == 'UUU' - assert notnone(item1.dt.tzinfo).zone == nytz.zone # type: ignore[attr-defined] - assert item2.username == 'UUU' - assert notnone(item2.dt.tzinfo).zone == nytz.zone # type: ignore[attr-defined] - - -# use multiple identical tests to make sure there are no issues with cached imports etc -@pytest.mark.parametrize('run_id', ['1', '2']) -def test_dynamic_module_import(tmp_path: Path, run_id: str) -> None: - """ - Test for dynamic hackery in config properties - e.g. importing some external modules - """ - - ext = tmp_path / 'external' - ext.mkdir() - (ext / '__init__.py').write_text( - ''' -def transform(x): - from .submodule import do_transform - return do_transform(x) - -''' - ) - (ext / 'submodule.py').write_text( - f''' -def do_transform(x): - return {{"total_{run_id}": sum(x.values())}} -''' - ) - - class user_config: - username = 'someuser' - data_path = f'{tmp_path}/*.json' - external = f'{ext}' - - my.config.demo = user_config # type: ignore[misc, assignment] - - [item1, item2] = items() - assert item1.raw == {f'total_{run_id}': 1 + 123}, item1 - assert item2.raw == {f'total_{run_id}': 2 + 456}, item2 - - # need to reset these modules, otherwise they get cached - # kind of relevant to my.core.cfg.tmp_config - sys.modules.pop('external', None) - sys.modules.pop('external.submodule', None) - - -@pytest.mark.parametrize('run_id', ['1', '2']) -def test_my_config_env_variable(tmp_path: Path, run_id: str) -> None: - """ - Tests handling of MY_CONFIG variable - """ - - # ugh. so by this point, my.config is already loaded (default stub), so we need to unload it - sys.modules.pop('my.config', None) - # but my.config itself relies on my.core.init hook, so unless it's reloaded too it wouldn't help - sys.modules.pop('my.core', None) - sys.modules.pop('my.core.init', None) - # it's a bit of a mouthful of course, but in most cases MY_CONFIG would be set once - # , and before hpi runs, so hopefully it's not a huge deal - cfg_dir = tmp_path / 'my' - cfg_file = cfg_dir / 'config.py' - cfg_dir.mkdir() - - cfg_file.write_text( - f''' -# print("IMPORTING CONFIG {run_id}") -class demo: - username = 'xxx_{run_id}' - data_path = r'{tmp_path}{os.sep}*.json' # need raw string for windows... -''' - ) - - with tmp_environ_set('MY_CONFIG', str(tmp_path)): - [item1, item2] = items() - assert item1.username == f'xxx_{run_id}' - assert item2.username == f'xxx_{run_id}' - - # sigh.. so this is cached in sys.path - # so it takes precedence later during next import, not giving the MY_CONFIG hook - # (imported from builtin my.config) to kick in - sys.path.remove(str(tmp_path)) - - # FIXME ideally this shouldn't be necessary? - # remove this after we fixup my.tests.reddit and my.tests.commits - # (they were failing ci when running all tests) - sys.modules.pop('my.config', None) - - -@pytest.fixture(autouse=True) -def prepare_data(tmp_path: Path): - (tmp_path / 'data.json').write_text( - ''' -[ - {"key": 1, "value": 123}, - {"key": 2, "value": 456} -] -''' - ) diff --git a/my/core/tests/test_get_files.py b/my/core/tests/test_get_files.py index 68be4d9..2bdc903 100644 --- a/my/core/tests/test_get_files.py +++ b/my/core/tests/test_get_files.py @@ -1,15 +1,16 @@ import os +from pathlib import Path import shutil import tempfile -import zipfile -from pathlib import Path from typing import TYPE_CHECKING - -import pytest +import zipfile from ..common import get_files +from ..compat import windows from ..kompress import CPath, ZipPath +import pytest + # hack to replace all /tmp with 'real' tmp dir # not ideal, but makes tests more concise @@ -55,9 +56,8 @@ def test_single_file() -> None: ''' assert get_files('/tmp/hpi_test/file.ext') == (Path('/tmp/hpi_test/file.ext'),) - is_windows = os.name == 'nt' "if the path starts with ~, we expand it" - if not is_windows: # windows doesn't have bashrc.. ugh + if not windows: # windows doesn't have bashrc.. ugh assert get_files('~/.bashrc') == (Path('~').expanduser() / '.bashrc',) @@ -175,17 +175,12 @@ TMP = tempfile.gettempdir() test_path = Path(TMP) / 'hpi_test' -@pytest.fixture(autouse=True) -def prepare(): +def setup(): teardown() test_path.mkdir() - try: - yield - finally: - teardown() -def teardown() -> None: +def teardown(): if test_path.is_dir(): shutil.rmtree(test_path) diff --git a/my/core/tests/test_tmp_config.py b/my/core/tests/test_tmp_config.py index d99621d..e5a24cc 100644 --- a/my/core/tests/test_tmp_config.py +++ b/my/core/tests/test_tmp_config.py @@ -12,7 +12,7 @@ def _init_default_config() -> None: def test_tmp_config() -> None: ## ugh. ideally this would be on the top level (would be a better test) - ## but pytest imports everything first, executes hooks, and some reset_modules() fictures mess stuff up + ## but pytest imports eveything first, executes hooks, and some reset_modules() fictures mess stuff up ## later would be nice to be a bit more careful about them _init_default_config() from my.simple import items diff --git a/my/core/time.py b/my/core/time.py index a9b180d..7698332 100644 --- a/my/core/time.py +++ b/my/core/time.py @@ -1,11 +1,9 @@ -from __future__ import annotations - -from collections.abc import Sequence -from functools import cache, lru_cache +from functools import lru_cache +from typing import Sequence, Dict import pytz -from .types import datetime_aware, datetime_naive +from .common import datetime_aware, datetime_naive def user_forced() -> Sequence[str]: @@ -13,24 +11,22 @@ def user_forced() -> Sequence[str]: # https://stackoverflow.com/questions/36067621/python-all-possible-timezone-abbreviations-for-given-timezone-name-and-vise-ve try: from my.config import time as user_config - - return user_config.tz.force_abbreviations # type: ignore[attr-defined] # noqa: TRY300 - # note: noqa since we're catching case where config doesn't have attribute here as well + return user_config.tz.force_abbreviations # type: ignore[attr-defined] except: # todo log/apply policy return [] @lru_cache(1) -def _abbr_to_timezone_map() -> dict[str, pytz.BaseTzInfo]: +def _abbr_to_timezone_map() -> Dict[str, pytz.BaseTzInfo]: # also force UTC to always correspond to utc # this makes more sense than Zulu it ends up by default - timezones = [*pytz.all_timezones, 'UTC', *user_forced()] + timezones = pytz.all_timezones + ['UTC'] + list(user_forced()) - res: dict[str, pytz.BaseTzInfo] = {} + res: Dict[str, pytz.BaseTzInfo] = {} for tzname in timezones: tz = pytz.timezone(tzname) - infos = getattr(tz, '_tzinfos', []) # not sure if can rely on attr always present? + infos = getattr(tz, '_tzinfos', []) # not sure if can rely on attr always present? for info in infos: abbr = info[-1] # todo could support this with a better error handling strategy? @@ -46,7 +42,7 @@ def _abbr_to_timezone_map() -> dict[str, pytz.BaseTzInfo]: return res -@cache +@lru_cache(maxsize=None) def abbr_to_timezone(abbr: str) -> pytz.BaseTzInfo: return _abbr_to_timezone_map()[abbr] diff --git a/my/core/types.py b/my/core/types.py deleted file mode 100644 index dc19c19..0000000 --- a/my/core/types.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -from .internal import assert_subpackage - -assert_subpackage(__name__) - -from dataclasses import asdict as dataclasses_asdict -from dataclasses import is_dataclass -from datetime import datetime -from typing import Any - -Json = dict[str, Any] - - -# for now just serves documentation purposes... but one day might make it statically verifiable where possible? -# TODO e.g. maybe use opaque mypy alias? -datetime_naive = datetime -datetime_aware = datetime - - -def is_namedtuple(thing: Any) -> bool: - # basic check to see if this is namedtuple-like - _asdict = getattr(thing, '_asdict', None) - return (_asdict is not None) and callable(_asdict) - - -def asdict(thing: Any) -> Json: - # todo primitive? - # todo exception? - if isinstance(thing, dict): - return thing - if is_dataclass(thing): - assert not isinstance(thing, type) # to help mypy - return dataclasses_asdict(thing) - if is_namedtuple(thing): - return thing._asdict() - raise TypeError(f'Could not convert object {thing} to dict') diff --git a/my/core/util.py b/my/core/util.py index 74e71e1..1ca2de1 100644 --- a/my/core/util.py +++ b/my/core/util.py @@ -1,35 +1,41 @@ -from __future__ import annotations - +from pathlib import Path +from itertools import chain +from importlib import import_module import os import pkgutil import sys -from collections.abc import Iterable -from itertools import chain -from pathlib import Path -from types import ModuleType +from typing import List, Iterable, Optional -from .discovery_pure import HPIModule, _is_not_module_src, has_stats, ignored +from .discovery_pure import HPIModule, ignored, _is_not_module_src, has_stats def modules() -> Iterable[HPIModule]: import my + for m in _iter_all_importables(my): + yield m - yield from _iter_all_importables(my) + +from .common import StatsFun +def get_stats(module: str) -> Optional[StatsFun]: + # todo detect via ast? + try: + mod = import_module(module) + except Exception: + return None + + return getattr(mod, 'stats', None) __NOT_HPI_MODULE__ = 'Import this to mark a python file as a helper, not an actual HPI module' from .discovery_pure import NOT_HPI_MODULE_VAR - assert NOT_HPI_MODULE_VAR in globals() # check name consistency - -def is_not_hpi_module(module: str) -> str | None: +def is_not_hpi_module(module: str) -> Optional[str]: ''' None if a module, otherwise returns reason ''' - import importlib.util - - path: str | None = None + import importlib + path: Optional[str] = None try: # TODO annoying, this can cause import of the parent module? spec = importlib.util.find_spec(module) @@ -38,7 +44,7 @@ def is_not_hpi_module(module: str) -> str | None: except Exception as e: # todo a bit misleading.. it actually shouldn't import in most cases, it's just the weird parent module import thing return "import error (possibly missing config entry)" # todo add exc message? - assert path is not None # not sure if can happen? + assert path is not None # not sure if can happen? if _is_not_module_src(Path(path)): return f"marked explicitly (via {NOT_HPI_MODULE_VAR})" @@ -47,6 +53,7 @@ def is_not_hpi_module(module: str) -> str | None: return None +from types import ModuleType # todo reuse in readme/blog post # borrowed from https://github.com/sanitizers/octomachinery/blob/24288774d6dcf977c5033ae11311dbff89394c89/tests/circular_imports_test.py#L22-L55 def _iter_all_importables(pkg: ModuleType) -> Iterable[HPIModule]: @@ -60,10 +67,9 @@ def _iter_all_importables(pkg: ModuleType) -> Iterable[HPIModule]: def _discover_path_importables(pkg_pth: Path, pkg_name: str) -> Iterable[HPIModule]: + from .core_config import config + """Yield all importables under a given path and package.""" - - from .core_config import config # noqa: F401 - for dir_path, dirs, file_names in os.walk(pkg_pth): file_names.sort() # NOTE: sorting dirs in place is intended, it's the way you're supposed to do it with os.walk @@ -78,7 +84,7 @@ def _discover_path_importables(pkg_pth: Path, pkg_name: str) -> Iterable[HPIModu continue rel_pt = pkg_dir_path.relative_to(pkg_pth) - pkg_pref = '.'.join((pkg_name, *rel_pt.parts)) + pkg_pref = '.'.join((pkg_name, ) + rel_pt.parts) yield from _walk_packages( (str(pkg_dir_path), ), prefix=f'{pkg_pref}.', @@ -86,7 +92,6 @@ def _discover_path_importables(pkg_pth: Path, pkg_name: str) -> Iterable[HPIModu # TODO might need to make it defensive and yield Exception (otherwise hpi doctor might fail for no good reason) # use onerror=? - # ignored explicitly -> not HPI # if enabled in config -> HPI # if disabled in config -> HPI @@ -95,17 +100,17 @@ def _discover_path_importables(pkg_pth: Path, pkg_name: str) -> Iterable[HPIModu # TODO when do we need to recurse? -def _walk_packages(path: Iterable[str], prefix: str = '', onerror=None) -> Iterable[HPIModule]: +def _walk_packages(path: Iterable[str], prefix: str='', onerror=None) -> Iterable[HPIModule]: """ Modified version of https://github.com/python/cpython/blob/d50a0700265536a20bcce3fb108c954746d97625/Lib/pkgutil.py#L53, - to avoid importing modules that are skipped + to alvoid importing modules that are skipped """ from .core_config import config - def seen(p, m={}): # noqa: B006 + def seen(p, m={}): if p in m: return True - m[p] = True # noqa: RET503 + m[p] = True for info in pkgutil.iter_modules(path, prefix): mname = info.name @@ -158,9 +163,8 @@ def _walk_packages(path: Iterable[str], prefix: str = '', onerror=None) -> Itera path = [p for p in path if not seen(p)] yield from _walk_packages(path, mname + '.', onerror) - # deprecate? -def get_modules() -> list[HPIModule]: +def get_modules() -> List[HPIModule]: return list(modules()) @@ -175,14 +179,14 @@ def test_module_detection() -> None: with reset() as cc: cc.disabled_modules = ['my.location.*', 'my.body.*', 'my.workouts.*', 'my.private.*'] mods = {m.name: m for m in modules()} - assert mods['my.demo'].skip_reason == "has no 'stats()' function" + assert mods['my.demo'] .skip_reason == "has no 'stats()' function" with reset() as cc: cc.disabled_modules = ['my.location.*', 'my.body.*', 'my.workouts.*', 'my.private.*', 'my.lastfm'] - cc.enabled_modules = ['my.demo'] + cc.enabled_modules = ['my.demo'] mods = {m.name: m for m in modules()} - assert mods['my.demo'].skip_reason is None # not skipped + assert mods['my.demo'] .skip_reason is None # not skipped assert mods['my.lastfm'].skip_reason == "suppressed in the user config" @@ -200,7 +204,6 @@ from my.core import __NOT_HPI_MODULE__ ''') import sys - orig_path = list(sys.path) try: sys.path.insert(0, str(badp)) @@ -235,7 +238,6 @@ def stats(): ''') import sys - orig_path = list(sys.path) try: sys.path.insert(0, str(badp)) diff --git a/my/core/utils/concurrent.py b/my/core/utils/concurrent.py deleted file mode 100644 index 515c3f1..0000000 --- a/my/core/utils/concurrent.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import annotations - -from concurrent.futures import Executor, Future -from typing import Any, Callable, TypeVar - -from ..compat import ParamSpec - -_P = ParamSpec('_P') -_T = TypeVar('_T') - - -# https://stackoverflow.com/a/10436851/706389 -class DummyExecutor(Executor): - """ - This is useful if you're already using Executor for parallelising, - but also want to provide an option to run the code serially (e.g. for debugging) - """ - - def __init__(self, max_workers: int | None = 1) -> None: - self._shutdown = False - self._max_workers = max_workers - - def submit(self, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs) -> Future[_T]: - if self._shutdown: - raise RuntimeError('cannot schedule new futures after shutdown') - - f: Future[Any] = Future() - try: - result = fn(*args, **kwargs) - except KeyboardInterrupt: - raise - except BaseException as e: - f.set_exception(e) - else: - f.set_result(result) - - return f - - def shutdown(self, wait: bool = True, **kwargs) -> None: # noqa: FBT001,FBT002,ARG002 - self._shutdown = True diff --git a/my/core/utils/imports.py b/my/core/utils/imports.py deleted file mode 100644 index e0fb01d..0000000 --- a/my/core/utils/imports.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -import importlib -import importlib.util -import sys -from pathlib import Path -from types import ModuleType - - -# TODO only used in tests? not sure if useful at all. -def import_file(p: Path | str, name: str | None = None) -> ModuleType: - p = Path(p) - if name is None: - name = p.stem - spec = importlib.util.spec_from_file_location(name, p) - assert spec is not None, f"Fatal error; Could not create module spec from {name} {p}" - foo = importlib.util.module_from_spec(spec) - loader = spec.loader - assert loader is not None - loader.exec_module(foo) - return foo - - -def import_from(path: Path | str, name: str) -> ModuleType: - path = str(path) - sys.path.append(path) - try: - return importlib.import_module(name) - finally: - sys.path.remove(path) - - -def import_dir(path: Path | str, extra: str = '') -> ModuleType: - p = Path(path) - if p.parts[0] == '~': - p = p.expanduser() # TODO eh. not sure about this.. - return import_from(p.parent, p.name + extra) diff --git a/my/core/utils/itertools.py b/my/core/utils/itertools.py deleted file mode 100644 index 42b2b77..0000000 --- a/my/core/utils/itertools.py +++ /dev/null @@ -1,369 +0,0 @@ -""" -Various helpers/transforms of iterators - -Ideally this should be as small as possible and we should rely on stdlib itertools or more_itertools -""" - -from __future__ import annotations - -import warnings -from collections.abc import Hashable, Iterable, Iterator, Sized -from typing import ( - TYPE_CHECKING, - Callable, - TypeVar, - Union, - cast, -) - -import more_itertools -from decorator import decorator - -from .. import warnings as core_warnings -from ..compat import ParamSpec - -T = TypeVar('T') -K = TypeVar('K') -V = TypeVar('V') - - -def _identity(v: T) -> V: # type: ignore[type-var] - return cast(V, v) - - -# ugh. nothing in more_itertools? -# perhaps duplicates_everseen? but it doesn't yield non-unique elements? -def ensure_unique(it: Iterable[T], *, key: Callable[[T], K]) -> Iterable[T]: - key2item: dict[K, T] = {} - for i in it: - k = key(i) - pi = key2item.get(k, None) - if pi is not None: - raise RuntimeError(f"Duplicate key: {k}. Previous value: {pi}, new value: {i}") - key2item[k] = i - yield i - - -def test_ensure_unique() -> None: - import pytest - - assert list(ensure_unique([1, 2, 3], key=lambda i: i)) == [1, 2, 3] - - dups = [1, 2, 1, 4] - # this works because it's lazy - it = ensure_unique(dups, key=lambda i: i) - - # but forcing throws - with pytest.raises(RuntimeError, match='Duplicate key'): - list(it) - - # hacky way to force distinct objects? - list(ensure_unique(dups, key=lambda _: object())) - - -def make_dict( - it: Iterable[T], - *, - key: Callable[[T], K], - # TODO make value optional instead? but then will need a typing override for it? - value: Callable[[T], V] = _identity, -) -> dict[K, V]: - with_keys = ((key(i), i) for i in it) - uniques = ensure_unique(with_keys, key=lambda p: p[0]) - res: dict[K, V] = {} - for k, i in uniques: - res[k] = i if value is None else value(i) - return res - - -def test_make_dict() -> None: - import pytest - - it = range(5) - d = make_dict(it, key=lambda i: i, value=lambda i: i % 2) - assert d == {0: 0, 1: 1, 2: 0, 3: 1, 4: 0} - - it = range(5) - with pytest.raises(RuntimeError, match='Duplicate key'): - d = make_dict(it, key=lambda i: i % 2, value=lambda i: i) - - # check type inference - d2: dict[str, int] = make_dict(it, key=lambda i: str(i)) - d3: dict[str, bool] = make_dict(it, key=lambda i: str(i), value=lambda i: i % 2 == 0) - - -LFP = ParamSpec('LFP') -LV = TypeVar('LV') - - -@decorator -def _listify(func: Callable[LFP, Iterable[LV]], *args: LFP.args, **kwargs: LFP.kwargs) -> list[LV]: - """ - Wraps a function's return value in wrapper (e.g. list) - Useful when an algorithm can be expressed more cleanly as a generator - """ - return list(func(*args, **kwargs)) - - -# ugh. decorator library has stub types, but they are way too generic? -# tried implementing my own stub, but failed -- not sure if it's possible at all? -# so seems easiest to just use specialize instantiations of decorator instead -if TYPE_CHECKING: - - def listify(func: Callable[LFP, Iterable[LV]]) -> Callable[LFP, list[LV]]: ... # noqa: ARG001 - -else: - listify = _listify - - -def test_listify() -> None: - from ..compat import assert_type - - @listify - def it() -> Iterator[int]: - yield 1 - yield 2 - - res = it() - assert_type(res, list[int]) - assert res == [1, 2] - - -@decorator -def _warn_if_empty(func, *args, **kwargs): - # so there is a more_itertools.peekable which could work nicely for these purposes - # the downside is that it would start advancing the generator right after it's created - # , which can be somewhat confusing - iterable = func(*args, **kwargs) - - if isinstance(iterable, Sized): - sz = len(iterable) - if sz == 0: - core_warnings.medium(f"Function {func} returned empty container, make sure your config paths are correct") - return iterable - else: # must be an iterator - - def wit(): - empty = True - for i in iterable: - yield i - empty = False - if empty: - core_warnings.medium(f"Function {func} didn't emit any data, make sure your config paths are correct") - - return wit() - - -if TYPE_CHECKING: - FF = TypeVar('FF', bound=Callable[..., Iterable]) - - def warn_if_empty(func: FF) -> FF: ... # noqa: ARG001 - -else: - warn_if_empty = _warn_if_empty - - -def test_warn_if_empty_iterator() -> None: - from ..compat import assert_type - - @warn_if_empty - def nonempty() -> Iterator[str]: - yield 'a' - yield 'aba' - - with warnings.catch_warnings(record=True) as w: - res1 = nonempty() - assert len(w) == 0 # warning isn't emitted until iterator is consumed - assert_type(res1, Iterator[str]) - assert list(res1) == ['a', 'aba'] - assert len(w) == 0 - - @warn_if_empty - def empty() -> Iterator[int]: - yield from [] - - with warnings.catch_warnings(record=True) as w: - res2 = empty() - assert len(w) == 0 # warning isn't emitted until iterator is consumed - assert_type(res2, Iterator[int]) - assert list(res2) == [] - assert len(w) == 1 - - -def test_warn_if_empty_list() -> None: - from ..compat import assert_type - - ll = [1, 2, 3] - - @warn_if_empty - def nonempty() -> list[int]: - return ll - - with warnings.catch_warnings(record=True) as w: - res1 = nonempty() - assert len(w) == 0 - assert_type(res1, list[int]) - assert isinstance(res1, list) - assert res1 is ll # object should be unchanged! - - @warn_if_empty - def empty() -> list[str]: - return [] - - with warnings.catch_warnings(record=True) as w: - res2 = empty() - assert len(w) == 1 - assert_type(res2, list[str]) - assert isinstance(res2, list) - assert res2 == [] - - -def test_warn_if_empty_unsupported() -> None: - # these should be rejected by mypy! (will show "unused type: ignore" if we break it) - @warn_if_empty # type: ignore[type-var] - def bad_return_type() -> float: - return 0.00 - - -_HT = TypeVar('_HT', bound=Hashable) - - -# NOTE: ideally we'do It = TypeVar('It', bound=Iterable[_HT]), and function would be It -> It -# Sadly this doesn't work in mypy, doesn't look like we can have double bound TypeVar -# Not a huge deal, since this function is for unique_eversee and -# we need to pass iterator to unique_everseen anyway -# TODO maybe contribute to more_itertools? https://github.com/more-itertools/more-itertools/issues/898 -def check_if_hashable(iterable: Iterable[_HT]) -> Iterable[_HT]: - """ - NOTE: Despite Hashable bound, typing annotation doesn't guarantee runtime safety - Consider hashable type X, and Y that inherits from X, but not hashable - Then l: list[X] = [Y(...)] is a valid expression, and type checks against Hashable, - but isn't runtime hashable - """ - # Sadly this doesn't work 100% correctly with dataclasses atm... - # they all are considered hashable: https://github.com/python/mypy/issues/11463 - - if isinstance(iterable, Iterator): - - def res() -> Iterator[_HT]: - for i in iterable: - assert isinstance(i, Hashable), i - # ugh. need a cast due to https://github.com/python/mypy/issues/10817 - yield cast(_HT, i) - - return res() - else: - # hopefully, iterable that can be iterated over multiple times? - # not sure if should have 'allowlist' of types that don't have to be transformed instead? - for i in iterable: - assert isinstance(i, Hashable), i - return iterable - - -# TODO different policies -- error/warn/ignore? -def test_check_if_hashable() -> None: - from dataclasses import dataclass - - import pytest - - from ..compat import assert_type - - x1: list[int] = [1, 2] - r1 = check_if_hashable(x1) - assert_type(r1, Iterable[int]) - assert r1 is x1 - - x2: Iterator[int | str] = iter((123, 'aba')) - r2 = check_if_hashable(x2) - assert_type(r2, Iterable[Union[int, str]]) - assert list(r2) == [123, 'aba'] - - x3: tuple[object, ...] = (789, 'aba') - r3 = check_if_hashable(x3) - assert_type(r3, Iterable[object]) - assert r3 is x3 # object should be unchanged - - x4: list[set[int]] = [{1, 2, 3}, {4, 5, 6}] - with pytest.raises(Exception): - # should be rejected by mypy sice set isn't Hashable, but also throw at runtime - r4 = check_if_hashable(x4) # type: ignore[type-var] - - x5: Iterator[object] = iter([{1, 2}, {3, 4}]) - # here, we hide behind object, which is hashable - # so mypy can't really help us anything - r5 = check_if_hashable(x5) - with pytest.raises(Exception): - # note: this only throws when iterator is advanced - list(r5) - - # dataclass is unhashable by default! unless frozen=True and eq=True, or unsafe_hash=True - @dataclass(unsafe_hash=True) - class X: - a: int - - x6: list[X] = [X(a=123)] - r6 = check_if_hashable(x6) - assert x6 is r6 - - # inherited dataclass will not be hashable! - @dataclass - class Y(X): - b: str - - x7: list[Y] = [Y(a=123, b='aba')] - with pytest.raises(Exception): - # ideally that would also be rejected by mypy, but currently there is a bug - # which treats all dataclasses as hashable: https://github.com/python/mypy/issues/11463 - check_if_hashable(x7) - - -_UET = TypeVar('_UET') -_UEU = TypeVar('_UEU') - - -# NOTE: for historic reasons, this function had to accept Callable that returns iterator -# instead of just iterator -# TODO maybe deprecated Callable support? not sure -def unique_everseen( - fun: Callable[[], Iterable[_UET]] | Iterable[_UET], - key: Callable[[_UET], _UEU] | None = None, -) -> Iterator[_UET]: - import os - - if callable(fun): - iterable = fun() - else: - iterable = fun - - if key is None: - # todo check key return type as well? but it's more likely to be hashable - if os.environ.get('HPI_CHECK_UNIQUE_EVERSEEN') is not None: - iterable = check_if_hashable(iterable) - - return more_itertools.unique_everseen(iterable=iterable, key=key) - - -def test_unique_everseen() -> None: - import pytest - - from ..tests.common import tmp_environ_set - - def fun_good() -> Iterator[int]: - yield 123 - - def fun_bad(): - return [{1, 2}, {1, 2}, {1, 3}] - - with tmp_environ_set('HPI_CHECK_UNIQUE_EVERSEEN', 'yes'): - assert list(unique_everseen(fun_good)) == [123] - - with pytest.raises(Exception): - # since function returns a list rather than iterator, check happens immediately - # , even without advancing the iterator - unique_everseen(fun_bad) - - good_list = [4, 3, 2, 1, 2, 3, 4] - assert list(unique_everseen(good_list)) == [4, 3, 2, 1] - - with tmp_environ_set('HPI_CHECK_UNIQUE_EVERSEEN', None): - assert list(unique_everseen(fun_bad)) == [{1, 2}, {1, 3}] diff --git a/my/core/warnings.py b/my/core/warnings.py index d67ec7d..7051f34 100644 --- a/my/core/warnings.py +++ b/my/core/warnings.py @@ -5,16 +5,14 @@ since who looks at the terminal output? E.g. would be nice to propagate the warnings in the UI (it's even a subclass of Exception!) ''' -from __future__ import annotations - import sys +from typing import Optional import warnings -from typing import TYPE_CHECKING import click -def _colorize(x: str, color: str | None = None) -> str: +def _colorize(x: str, color: Optional[str]=None) -> str: if color is None: return x @@ -26,10 +24,10 @@ def _colorize(x: str, color: str | None = None) -> str: return click.style(x, fg=color) -def _warn(message: str, *args, color: str | None = None, **kwargs) -> None: +def _warn(message: str, *args, color: Optional[str]=None, **kwargs) -> None: stacklevel = kwargs.get('stacklevel', 1) - kwargs['stacklevel'] = stacklevel + 2 # +1 for this function, +1 for medium/high wrapper - warnings.warn(_colorize(message, color=color), *args, **kwargs) # noqa: B028 + kwargs['stacklevel'] = stacklevel + 2 # +1 for this function, +1 for medium/high wrapper + warnings.warn(_colorize(message, color=color), *args, **kwargs) def low(message: str, *args, **kwargs) -> None: @@ -50,11 +48,5 @@ def high(message: str, *args, **kwargs) -> None: _warn(message, *args, **kwargs) -if not TYPE_CHECKING: - from .compat import deprecated - - @deprecated('use warnings.warn directly instead') - def warn(*args, **kwargs): - import warnings - - return warnings.warn(*args, **kwargs) # noqa: B028 +# NOTE: deprecated -- legacy import +from warnings import warn \ No newline at end of file diff --git a/my/demo.py b/my/demo.py index fa80b2a..75954d6 100644 --- a/my/demo.py +++ b/my/demo.py @@ -1,77 +1,69 @@ ''' Just a demo module for testing and documentation purposes ''' -from __future__ import annotations -import json -from collections.abc import Iterable, Sequence +from .core import Paths, PathIsh + +from typing import Optional +from datetime import tzinfo, timezone + +from my.config import demo as user_config from dataclasses import dataclass -from datetime import datetime, timezone, tzinfo -from pathlib import Path -from typing import Protocol - -from my.core import Json, PathIsh, Paths, get_files -class config(Protocol): +@dataclass +class demo(user_config): data_path: Paths - - # this is to check required attribute handling username: str - - # this is to check optional attribute handling timezone: tzinfo = timezone.utc - external: PathIsh | None = None + external: Optional[PathIsh] = None @property def external_module(self): rpath = self.external if rpath is not None: - from my.core.utils.imports import import_dir - + from .core.common import import_dir return import_dir(rpath) - import my.config.repos.external as m # type: ignore - + import my.config.repos.external as m # type: ignore return m -def make_config() -> config: - from my.config import demo as user_config +from .core import make_config +config = make_config(demo) - class combined_config(user_config, config): ... +# TODO not sure about type checking? +external = config.external_module - return combined_config() +from pathlib import Path +from typing import Sequence, Iterable +from datetime import datetime +from .core import Json, get_files @dataclass class Item: ''' Some completely arbitrary artificial stuff, just for testing ''' - username: str raw: Json dt: datetime def inputs() -> Sequence[Path]: - cfg = make_config() - return get_files(cfg.data_path) + return get_files(config.data_path) +import json def items() -> Iterable[Item]: - cfg = make_config() - - transform = (lambda i: i) if cfg.external is None else cfg.external_module.transform - for f in inputs(): - dt = datetime.fromtimestamp(f.stat().st_mtime, tz=cfg.timezone) + dt = datetime.fromtimestamp(f.stat().st_mtime, tz=config.timezone) j = json.loads(f.read_text()) for raw in j: yield Item( - username=cfg.username, - raw=transform(raw), + username=config.username, + raw=external.identity(raw), dt=dt, ) diff --git a/my/emfit/__init__.py b/my/emfit/__init__.py index 0d50b06..1ec3341 100644 --- a/my/emfit/__init__.py +++ b/my/emfit/__init__.py @@ -4,33 +4,31 @@ Consumes data exported by https://github.com/karlicoss/emfitexport """ -from __future__ import annotations - REQUIRES = [ 'git+https://github.com/karlicoss/emfitexport', ] -import dataclasses -import inspect -from collections.abc import Iterable, Iterator from contextlib import contextmanager +import dataclasses from datetime import datetime, time, timedelta +import inspect from pathlib import Path -from typing import Any - -import emfitexport.dal as dal +from typing import Any, Dict, Iterable, Iterator, List, Optional from my.core import ( - Res, - Stats, get_files, stat, + Res, + Stats, ) -from my.core.cachew import cache_dir, mcachew -from my.core.error import extract_error_datetime, set_error_datetime +from my.core.common import mcachew +from my.core.cachew import cache_dir +from my.core.error import set_error_datetime, extract_error_datetime from my.core.pandas import DataFrameT -from my.config import emfit as config # isort: skip +from my.config import emfit as config + +import emfitexport.dal as dal Emfit = dal.Emfit @@ -88,7 +86,7 @@ def datas() -> Iterable[Res[Emfit]]: # TODO should be used for jawbone data as well? def pre_dataframe() -> Iterable[Res[Emfit]]: # TODO shit. I need some sort of interrupted sleep detection? - g: list[Emfit] = [] + g: List[Emfit] = [] def flush() -> Iterable[Res[Emfit]]: if len(g) == 0: @@ -115,10 +113,10 @@ def pre_dataframe() -> Iterable[Res[Emfit]]: def dataframe() -> DataFrameT: - dicts: list[dict[str, Any]] = [] - last: Emfit | None = None + dicts: List[Dict[str, Any]] = [] + last: Optional[Emfit] = None for s in pre_dataframe(): - d: dict[str, Any] + d: Dict[str, Any] if isinstance(s, Exception): edt = extract_error_datetime(s) d = { @@ -158,9 +156,9 @@ def dataframe() -> DataFrameT: last = s # meh dicts.append(d) - import pandas as pd + import pandas - return pd.DataFrame(dicts) + return pandas.DataFrame(dicts) def stats() -> Stats: @@ -169,11 +167,8 @@ def stats() -> Stats: @contextmanager def fake_data(nights: int = 500) -> Iterator: - from tempfile import TemporaryDirectory - - import pytz - from my.core.cfg import tmp_config + from tempfile import TemporaryDirectory with TemporaryDirectory() as td: tdir = Path(td) @@ -183,17 +178,15 @@ def fake_data(nights: int = 500) -> Iterator: class override: class emfit: export_path = tdir - excluded_sids = () - timezone = pytz.timezone('Europe/London') # meh with tmp_config(modules=__name__, config=override) as cfg: yield cfg # TODO remove/deprecate it? I think used by timeline -def get_datas() -> list[Emfit]: +def get_datas() -> List[Emfit]: # todo ugh. run lint properly - return sorted(datas(), key=lambda e: e.start) # type: ignore + return list(sorted(datas(), key=lambda e: e.start)) # type: ignore # TODO move away old entries if there is a diff?? diff --git a/my/endomondo.py b/my/endomondo.py index 7732c00..d314e97 100644 --- a/my/endomondo.py +++ b/my/endomondo.py @@ -7,14 +7,13 @@ REQUIRES = [ ] # todo use ast in setup.py or doctor to extract the corresponding pip packages? -from collections.abc import Iterable, Sequence from dataclasses import dataclass from pathlib import Path - -from my.config import endomondo as user_config +from typing import Sequence, Iterable from .core import Paths, get_files +from my.config import endomondo as user_config @dataclass class endomondo(user_config): @@ -32,22 +31,20 @@ def inputs() -> Sequence[Path]: # todo add a doctor check for pip endoexport module import endoexport.dal as dal -from endoexport.dal import Point, Workout # noqa: F401 +from endoexport.dal import Point, Workout + from .core import Res - - # todo cachew? def workouts() -> Iterable[Res[Workout]]: _dal = dal.DAL(inputs()) yield from _dal.workouts() -from .core.pandas import DataFrameT, check_dataframe - +from .core.pandas import check_dataframe, DataFrameT @check_dataframe -def dataframe(*, defensive: bool=True) -> DataFrameT: +def dataframe(defensive: bool=True) -> DataFrameT: def it(): for w in workouts(): if isinstance(w, Exception): @@ -78,9 +75,7 @@ def dataframe(*, defensive: bool=True) -> DataFrameT: return df -from .core import Stats, stat - - +from .core import stat, Stats def stats() -> Stats: return { # todo pretty print stats? @@ -91,16 +86,13 @@ def stats() -> Stats: # TODO make sure it's possible to 'advise' functions and override stuff -from collections.abc import Iterator from contextlib import contextmanager - - +from typing import Iterator @contextmanager def fake_data(count: int=100) -> Iterator: - import json - from tempfile import TemporaryDirectory - from my.core.cfg import tmp_config + from tempfile import TemporaryDirectory + import json with TemporaryDirectory() as td: tdir = Path(td) fd = dal.FakeData() diff --git a/my/error.py b/my/error.py index e3c1e11..c0b734c 100644 --- a/my/error.py +++ b/my/error.py @@ -1,6 +1,6 @@ from .core.warnings import high - high("DEPRECATED! Please use my.core.error instead.") from .core import __NOT_HPI_MODULE__ + from .core.error import * diff --git a/my/experimental/destructive_parsing.py b/my/experimental/destructive_parsing.py deleted file mode 100644 index 0c4092a..0000000 --- a/my/experimental/destructive_parsing.py +++ /dev/null @@ -1,60 +0,0 @@ -from collections.abc import Iterator -from dataclasses import dataclass -from typing import Any - -from my.core.compat import NoneType, assert_never - - -# TODO Popper? not sure -@dataclass -class Helper: - manager: 'Manager' - item: Any # todo realistically, list or dict? could at least type as indexable or something - path: tuple[str, ...] - - def pop_if_primitive(self, *keys: str) -> None: - """ - The idea that primitive TODO - """ - item = self.item - for k in keys: - v = item[k] - if isinstance(v, (str, bool, float, int, NoneType)): - item.pop(k) # todo kinda unfortunate to get dict item twice.. but not sure if can avoid? - - def check(self, key: str, expected: Any) -> None: - actual = self.item.pop(key) - assert actual == expected, (key, actual, expected) - - def zoom(self, key: str) -> 'Helper': - return self.manager.helper(item=self.item.pop(key), path=(*self.path, key)) - - -def is_empty(x) -> bool: - if isinstance(x, dict): - return len(x) == 0 - elif isinstance(x, list): - return all(map(is_empty, x)) - else: - assert_never(x) # noqa: RET503 - - -class Manager: - def __init__(self) -> None: - self.helpers: list[Helper] = [] - - def helper(self, item: Any, *, path: tuple[str, ...] = ()) -> Helper: - res = Helper(manager=self, item=item, path=path) - self.helpers.append(res) - return res - - def check(self) -> Iterator[Exception]: - remaining = [] - for h in self.helpers: - # TODO recursively check it's primitive? - if is_empty(h.item): - continue - remaining.append((h.path, h.item)) - if len(remaining) == 0: - return - yield RuntimeError(f'Unparsed items remaining: {remaining}') diff --git a/my/fbmessenger/__init__.py b/my/fbmessenger/__init__.py index e5e417c..40fb235 100644 --- a/my/fbmessenger/__init__.py +++ b/my/fbmessenger/__init__.py @@ -9,7 +9,7 @@ since that allows for easier overriding using namespace packages See https://github.com/karlicoss/HPI/blob/master/doc/MODULE_DESIGN.org#allpy for more info. """ -# prevent it from appearing in modules list/doctor +# prevent it from apprearing in modules list/doctor from ..core import __NOT_HPI_MODULE__ # kinda annoying to keep it, but it's so legacy 'hpi module install my.fbmessenger' works @@ -20,7 +20,6 @@ REQUIRES = [ from my.core.hpi_compat import handle_legacy_import - is_legacy_import = handle_legacy_import( parent_module_name=__name__, legacy_submodule_name='export', diff --git a/my/fbmessenger/all.py b/my/fbmessenger/all.py index a057dca..13689db 100644 --- a/my/fbmessenger/all.py +++ b/my/fbmessenger/all.py @@ -1,10 +1,10 @@ -from collections.abc import Iterator - -from my.core import Res, Stats +from typing import Iterator +from my.core import Res, stat, Stats from my.core.source import import_source from .common import Message, _merge_messages + src_export = import_source(module_name='my.fbmessenger.export') src_android = import_source(module_name='my.fbmessenger.android') diff --git a/my/fbmessenger/android.py b/my/fbmessenger/android.py index f6fdb82..fa313ea 100644 --- a/my/fbmessenger/android.py +++ b/my/fbmessenger/android.py @@ -1,23 +1,20 @@ """ Messenger data from Android app database (in =/data/data/com.facebook.orca/databases/threads_db2=) """ - from __future__ import annotations -import sqlite3 -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from typing import Union +import sqlite3 +from typing import Iterator, Sequence, Optional, Dict, Union, List -from my.core import LazyLogger, Paths, Res, datetime_aware, get_files, make_config +from my.core import get_files, Paths, datetime_aware, Res, assert_never, LazyLogger, make_config from my.core.common import unique_everseen -from my.core.compat import assert_never from my.core.error import echain -from my.core.sqlite import sqlite_connection, SqliteTool +from my.core.sqlite import sqlite_connection -from my.config import fbmessenger as user_config # isort: skip +from my.config import fbmessenger as user_config logger = LazyLogger(__name__) @@ -28,7 +25,7 @@ class Config(user_config.android): # paths[s]/glob to the exported sqlite databases export_path: Paths - facebook_id: str | None = None + facebook_id: Optional[str] = None # hmm. this is necessary for default value (= None) to work @@ -43,13 +40,13 @@ def inputs() -> Sequence[Path]: @dataclass(unsafe_hash=True) class Sender: id: str - name: str | None + name: Optional[str] @dataclass(unsafe_hash=True) class Thread: id: str - name: str | None # isn't set for groups or one to one messages + name: Optional[str] # isn't set for groups or one to one messages # todo not sure about order of fields... @@ -57,14 +54,14 @@ class Thread: class _BaseMessage: id: str dt: datetime_aware - text: str | None + text: Optional[str] @dataclass(unsafe_hash=True) class _Message(_BaseMessage): thread_id: str sender_id: str - reply_to_id: str | None + reply_to_id: Optional[str] # todo hmm, on the one hand would be kinda nice to inherit common.Message protocol here @@ -73,7 +70,7 @@ class _Message(_BaseMessage): class Message(_BaseMessage): thread: Thread sender: Sender - reply_to: Message | None + reply_to: Optional[Message] Entity = Union[Sender, Thread, _Message] @@ -86,8 +83,8 @@ def _entities() -> Iterator[Res[Entity]]: for idx, path in enumerate(paths): logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}') with sqlite_connection(path, immutable=True, row_factory='row') as db: - use_msys = "logging_events_v2" in SqliteTool(db).get_table_names() try: + use_msys = len(list(db.execute('SELECT * FROM sqlite_master WHERE name = "logging_events_v2"'))) > 0 if use_msys: yield from _process_db_msys(db) else: @@ -111,38 +108,29 @@ def _normalise_thread_id(key) -> str: # NOTE: this is sort of copy pasted from other _process_db method # maybe later could unify them def _process_db_msys(db: sqlite3.Connection) -> Iterator[Res[Entity]]: - senders: dict[str, Sender] = {} + senders: Dict[str, Sender] = {} for r in db.execute('SELECT CAST(id AS TEXT) AS id, name FROM contacts'): s = Sender( - id=r['id'], # looks like it's server id? same used on facebook site + id=r['id'], name=r['name'], ) - # NOTE https://www.messenger.com/t/{contant_id} for permalink senders[s.id] = s yield s - # TODO what is fb transport?? - # TODO what are client_contacts?? has pk or something - - # TODO client_threads/client_messages -- possibly for end to end encryption or something? - # TODO can we get it from db? could infer as the most common id perhaps? self_id = config.facebook_id - thread_users: dict[str, list[Sender]] = {} + thread_users: Dict[str, List[Sender]] = {} for r in db.execute('SELECT CAST(thread_key AS TEXT) AS thread_key, CAST(contact_id AS TEXT) AS contact_id FROM participants'): thread_key = r['thread_key'] user_key = r['contact_id'] + if self_id is not None and user_key == self_id: + # exclude yourself, otherwise it's just spammy to show up in all participants + continue ll = thread_users.get(thread_key) if ll is None: ll = [] thread_users[thread_key] = ll - - if self_id is not None and user_key == self_id: - # exclude yourself, otherwise it's just spammy to show up in all participants - # TODO not sure about that, maybe change later - continue - ll.append(senders[user_key]) # 15 is a weird thread that doesn't have any participants and messages @@ -169,15 +157,6 @@ def _process_db_msys(db: sqlite3.Connection) -> Iterator[Res[Entity]]: CAST(sender_id AS TEXT) AS sender_id, reply_source_id FROM messages - WHERE - /* Regular message_id conforms to mid.* regex. - However seems that when message is not sent yet it doesn't have this server id yet - (happened only once, but could be just luck of course!) - We exclude these messages to avoid duplication. - However poisitive filter (e.g. message_id LIKE 'mid%') feels a bit wrong, e.g. what if message ids change or something - So instead this excludes only such unsent messages. - */ - message_id != offline_threading_id ORDER BY timestamp_ms /* they aren't in order in the database, so need to sort */ ''' ): @@ -194,7 +173,7 @@ def _process_db_msys(db: sqlite3.Connection) -> Iterator[Res[Entity]]: def _process_db_threads_db2(db: sqlite3.Connection) -> Iterator[Res[Entity]]: - senders: dict[str, Sender] = {} + senders: Dict[str, Sender] = {} for r in db.execute('''SELECT * FROM thread_users'''): # for messaging_actor_type == 'REDUCED_MESSAGING_ACTOR', name is None # but they are still referenced, so need to keep @@ -208,7 +187,7 @@ def _process_db_threads_db2(db: sqlite3.Connection) -> Iterator[Res[Entity]]: yield s self_id = config.facebook_id - thread_users: dict[str, list[Sender]] = {} + thread_users: Dict[str, List[Sender]] = {} for r in db.execute('SELECT * from thread_participants'): thread_key = r['thread_key'] user_key = r['user_key'] @@ -238,7 +217,7 @@ def _process_db_threads_db2(db: sqlite3.Connection) -> Iterator[Res[Entity]]: for r in db.execute( ''' - SELECT *, json_extract(sender, "$.user_key") AS user_key FROM messages + SELECT *, json_extract(sender, "$.user_key") AS user_key FROM messages WHERE msg_type NOT IN ( -1, /* these don't have any data at all, likely immediately deleted or something? */ 2 /* these are 'left group' system messages, also a bit annoying since they might reference nonexistent users */ @@ -258,19 +237,10 @@ def _process_db_threads_db2(db: sqlite3.Connection) -> Iterator[Res[Entity]]: ) -def contacts() -> Iterator[Res[Sender]]: - for x in unique_everseen(_entities): - if isinstance(x, Exception): - yield x - continue - if isinstance(x, Sender): - yield x - - def messages() -> Iterator[Res[Message]]: - senders: dict[str, Sender] = {} - msgs: dict[str, Message] = {} - threads: dict[str, Thread] = {} + senders: Dict[str, Sender] = {} + msgs: Dict[str, Message] = {} + threads: Dict[str, Thread] = {} for x in unique_everseen(_entities): if isinstance(x, Exception): yield x diff --git a/my/fbmessenger/common.py b/my/fbmessenger/common.py index 0f5a374..33d1b20 100644 --- a/my/fbmessenger/common.py +++ b/my/fbmessenger/common.py @@ -1,9 +1,6 @@ -from __future__ import annotations +from my.core import __NOT_HPI_MODULE__ -from my.core import __NOT_HPI_MODULE__ # isort: skip - -from collections.abc import Iterator -from typing import Protocol +from typing import Iterator, Optional, Protocol from my.core import datetime_aware @@ -13,7 +10,7 @@ class Thread(Protocol): def id(self) -> str: ... @property - def name(self) -> str | None: ... + def name(self) -> Optional[str]: ... class Sender(Protocol): @@ -21,7 +18,7 @@ class Sender(Protocol): def id(self) -> str: ... @property - def name(self) -> str | None: ... + def name(self) -> Optional[str]: ... class Message(Protocol): @@ -32,7 +29,7 @@ class Message(Protocol): def dt(self) -> datetime_aware: ... @property - def text(self) -> str | None: ... + def text(self) -> Optional[str]: ... @property def thread(self) -> Thread: ... @@ -42,11 +39,8 @@ class Message(Protocol): from itertools import chain - from more_itertools import unique_everseen - -from my.core import Res, warn_if_empty - +from my.core import warn_if_empty, Res @warn_if_empty def _merge_messages(*sources: Iterator[Res[Message]]) -> Iterator[Res[Message]]: diff --git a/my/fbmessenger/export.py b/my/fbmessenger/export.py index 3b06618..201fad8 100644 --- a/my/fbmessenger/export.py +++ b/my/fbmessenger/export.py @@ -7,15 +7,16 @@ REQUIRES = [ 'git+https://github.com/karlicoss/fbmessengerexport', ] -from collections.abc import Iterator from contextlib import ExitStack, contextmanager from dataclasses import dataclass +from typing import Iterator + +from my.core import PathIsh, Res, stat, Stats +from my.core.warnings import high +from my.config import fbmessenger as user_config import fbmessengerexport.dal as messenger -from my.config import fbmessenger as user_config -from my.core import PathIsh, Res, Stats, stat -from my.core.warnings import high ### # support old style config diff --git a/my/foursquare.py b/my/foursquare.py index 3b418aa..b50ab0e 100644 --- a/my/foursquare.py +++ b/my/foursquare.py @@ -2,16 +2,18 @@ Foursquare/Swarm checkins ''' -import json -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone, timedelta from itertools import chain - -from my.config import foursquare as config +from pathlib import Path +import json # TODO pytz for timezone??? -from my.core import get_files, make_logger -logger = make_logger(__name__) +from .core.common import get_files, LazyLogger +from my.config import foursquare as config + + +logger = LazyLogger(__name__) def inputs(): diff --git a/my/github/all.py b/my/github/all.py index f5e13cf..f885dde 100644 --- a/my/github/all.py +++ b/my/github/all.py @@ -3,7 +3,8 @@ Unified Github data (merged from GDPR export and periodic API updates) """ from . import gdpr, ghexport -from .common import Results, merge_events + +from .common import merge_events, Results def events() -> Results: diff --git a/my/github/common.py b/my/github/common.py index 22ba47e..e54bc4d 100644 --- a/my/github/common.py +++ b/my/github/common.py @@ -1,27 +1,24 @@ """ Github events and their metadata: comments/issues/pull requests """ - -from __future__ import annotations - -from my.core import __NOT_HPI_MODULE__ # isort: skip +from ..core import __NOT_HPI_MODULE__ -from collections.abc import Iterable from datetime import datetime, timezone -from typing import NamedTuple, Optional +from typing import Optional, NamedTuple, Iterable, Set, Tuple -from my.core import make_logger, warn_if_empty -from my.core.error import Res +from ..core import warn_if_empty, LazyLogger +from ..core.error import Res -logger = make_logger(__name__) + +logger = LazyLogger(__name__) class Event(NamedTuple): dt: datetime summary: str eid: str link: Optional[str] - body: Optional[str] = None + body: Optional[str]=None is_bot: bool = False @@ -30,7 +27,7 @@ Results = Iterable[Res[Event]] @warn_if_empty def merge_events(*sources: Results) -> Results: from itertools import chain - emitted: set[tuple[datetime, str]] = set() + emitted: Set[Tuple[datetime, str]] = set() for e in chain(*sources): if isinstance(e, Exception): yield e @@ -55,7 +52,7 @@ def parse_dt(s: str) -> datetime: # experimental way of supportint event ids... not sure class EventIds: @staticmethod - def repo_created(*, dts: str, name: str, ref_type: str, ref: str | None) -> str: + def repo_created(*, dts: str, name: str, ref_type: str, ref: Optional[str]) -> str: return f'{dts}_repocreated_{name}_{ref_type}_{ref}' @staticmethod diff --git a/my/github/gdpr.py b/my/github/gdpr.py index be56454..1fde7c9 100644 --- a/my/github/gdpr.py +++ b/my/github/gdpr.py @@ -1,43 +1,36 @@ """ Github data (uses [[https://github.com/settings/admin][official GDPR export]]) """ - -from __future__ import annotations - +from dataclasses import dataclass import json -from abc import abstractmethod -from collections.abc import Iterator, Sequence from pathlib import Path -from typing import Any +import tarfile +from typing import Iterable, Any, Sequence, Dict, Optional -from my.core import Paths, Res, Stats, get_files, make_logger, stat, warnings -from my.core.error import echain +from my.core import get_files, Res, PathIsh, stat, Stats, make_logger +from my.core.cfg import make_config +from my.core.error import notnone, echain + +from .common import Event, parse_dt, EventIds + +# TODO later, use a separate user config? (github_gdpr) +from my.config import github as user_config + + +@dataclass +class github(user_config): + gdpr_dir: PathIsh # path to unpacked GDPR archive + + +config = make_config(github) -from .common import Event, EventIds, parse_dt logger = make_logger(__name__) -class config: - @property - @abstractmethod - def gdpr_dir(self) -> Paths: - raise NotImplementedError - - -def make_config() -> config: - # TODO later, use a separate user config? (github_gdpr) - from my.config import github as user_config - - class combined_config(user_config, config): - pass - - return combined_config() - - def inputs() -> Sequence[Path]: - gdpr_dir = make_config().gdpr_dir - res = get_files(gdpr_dir) + gdir = config.gdpr_dir + res = get_files(gdir) schema_json = [f for f in res if f.name == 'schema.json'] was_unpacked = len(schema_json) > 0 if was_unpacked: @@ -50,37 +43,22 @@ def inputs() -> Sequence[Path]: return res -def events() -> Iterator[Res[Event]]: +def events() -> Iterable[Res[Event]]: last = max(inputs()) logger.info(f'extracting data from {last}') - root: Path | None = None - - if last.is_dir(): # if it's already CPath, this will match it - root = last + # a bit naughty and ad-hoc, but we will generify reading from tar.gz. once we have more examples + # another one is zulip archive + if last.is_dir(): + files = list(sorted(last.glob('*.json'))) # looks like all files are in the root + open_file = lambda f: f.open() else: - try: - from kompress import CPath - - root = CPath(last) - assert len(list(root.iterdir())) > 0 # trigger to check if we have the kompress version with targz support - except Exception as e: - logger.exception(e) - warnings.high("Upgrade 'kompress' to latest version with native .tar.gz support. Falling back to unpacking to tmp dir.") - - if root is None: - from my.core.structure import match_structure - - with match_structure(last, expected=()) as res: # expected=() matches it regardless any patterns - [root] = res - yield from _process_one(root) - else: - yield from _process_one(root) - - -def _process_one(root: Path) -> Iterator[Res[Event]]: - files = sorted(root.glob('*.json')) # looks like all files are in the root + # treat as .tar.gz + tfile = tarfile.open(last) + files = list(sorted(map(Path, tfile.getnames()))) + files = [p for p in files if len(p.parts) == 1 and p.suffix == '.json'] + open_file = lambda p: notnone(tfile.extractfile(f'./{p}')) # NOTE odd, doesn't work without ./ # fmt: off handler_map = { @@ -122,7 +100,8 @@ def _process_one(root: Path) -> Iterator[Res[Event]]: # ignored continue - j = json.loads(f.read_text()) + with open_file(f) as fo: + j = json.load(fo) for r in j: try: yield handler(r) @@ -137,7 +116,7 @@ def stats() -> Stats: # TODO typing.TypedDict could be handy here.. -def _parse_common(d: dict) -> dict: +def _parse_common(d: Dict) -> Dict: url = d['url'] body = d.get('body') return { @@ -147,7 +126,7 @@ def _parse_common(d: dict) -> dict: } -def _parse_repository(d: dict) -> Event: +def _parse_repository(d: Dict) -> Event: pref = 'https://github.com/' url = d['url'] dts = d['created_at'] @@ -163,13 +142,13 @@ def _parse_repository(d: dict) -> Event: # user may be None if the user was deleted -def _is_bot(user: str | None) -> bool: +def _is_bot(user: Optional[str]) -> bool: if user is None: return False - return "[bot]" in user + return "[bot]" in "user" -def _parse_issue_comment(d: dict) -> Event: +def _parse_issue_comment(d: Dict) -> Event: url = d['url'] return Event( **_parse_common(d), @@ -179,7 +158,7 @@ def _parse_issue_comment(d: dict) -> Event: ) -def _parse_issue(d: dict) -> Event: +def _parse_issue(d: Dict) -> Event: url = d['url'] title = d['title'] return Event( @@ -190,7 +169,7 @@ def _parse_issue(d: dict) -> Event: ) -def _parse_pull_request(d: dict) -> Event: +def _parse_pull_request(d: Dict) -> Event: dts = d['created_at'] url = d['url'] title = d['title'] @@ -204,7 +183,7 @@ def _parse_pull_request(d: dict) -> Event: ) -def _parse_project(d: dict) -> Event: +def _parse_project(d: Dict) -> Event: url = d['url'] title = d['name'] is_bot = "[bot]" in d["creator"] @@ -219,7 +198,7 @@ def _parse_project(d: dict) -> Event: ) -def _parse_release(d: dict) -> Event: +def _parse_release(d: Dict) -> Event: tag = d['tag_name'] return Event( **_parse_common(d), @@ -228,7 +207,7 @@ def _parse_release(d: dict) -> Event: ) -def _parse_commit_comment(d: dict) -> Event: +def _parse_commit_comment(d: Dict) -> Event: url = d['url'] return Event( **_parse_common(d), diff --git a/my/github/ghexport.py b/my/github/ghexport.py index 3e17c10..d446c35 100644 --- a/my/github/ghexport.py +++ b/my/github/ghexport.py @@ -1,17 +1,13 @@ """ Github data: events, comments, etc. (API data) """ - -from __future__ import annotations - REQUIRES = [ 'git+https://github.com/karlicoss/ghexport', ] - from dataclasses import dataclass -from my.config import github as user_config from my.core import Paths +from my.config import github as user_config @dataclass @@ -25,9 +21,7 @@ class github(user_config): ### -from my.core.cfg import Attrs, make_config - - +from my.core.cfg import make_config, Attrs def migration(attrs: Attrs) -> Attrs: export_dir = 'export_dir' if export_dir in attrs: # legacy name @@ -47,14 +41,14 @@ except ModuleNotFoundError as e: ############################ -from collections.abc import Sequence from functools import lru_cache -from pathlib import Path +from typing import Tuple, Dict, Sequence, Optional -from my.core import LazyLogger, get_files -from my.core.cachew import mcachew +from my.core import get_files, Path, LazyLogger +from my.core.common import mcachew + +from .common import Event, parse_dt, Results, EventIds -from .common import Event, EventIds, Results, parse_dt logger = LazyLogger(__name__) @@ -70,10 +64,11 @@ def _dal() -> dal.DAL: @mcachew(depends_on=inputs) def events() -> Results: - # key = lambda e: object() if isinstance(e, Exception) else e.eid + from my.core.common import ensure_unique + key = lambda e: object() if isinstance(e, Exception) else e.eid # crap. sometimes API events can be repeated with exactly the same payload and different id # yield from ensure_unique(_events(), key=key) - return _events() + yield from _events() def _events() -> Results: @@ -87,9 +82,7 @@ def _events() -> Results: yield e -from my.core import Stats, stat - - +from my.core import stat, Stats def stats() -> Stats: return { **stat(events), @@ -106,7 +99,7 @@ def _log_if_unhandled(e) -> None: Link = str EventId = str Body = str -def _get_summary(e) -> tuple[str, Link | None, EventId | None, Body | None]: +def _get_summary(e) -> Tuple[str, Optional[Link], Optional[EventId], Optional[Body]]: # TODO would be nice to give access to raw event within timeline dts = e['created_at'] eid = e['id'] @@ -202,7 +195,7 @@ def _get_summary(e) -> tuple[str, Link | None, EventId | None, Body | None]: return tp, None, None, None -def _parse_event(d: dict) -> Event: +def _parse_event(d: Dict) -> Event: summary, link, eid, body = _get_summary(d) if eid is None: eid = d['id'] # meh diff --git a/my/goodreads.py b/my/goodreads.py index 559efda..acf2bb9 100644 --- a/my/goodreads.py +++ b/my/goodreads.py @@ -7,18 +7,15 @@ REQUIRES = [ from dataclasses import dataclass - +from my.core import Paths from my.config import goodreads as user_config -from my.core import Paths, datetime_aware - @dataclass class goodreads(user_config): # paths[s]/glob to the exported JSON data export_path: Paths -from my.core.cfg import Attrs, make_config - +from my.core.cfg import make_config, Attrs def _migration(attrs: Attrs) -> Attrs: export_dir = 'export_dir' @@ -32,19 +29,18 @@ config = make_config(goodreads, migration=_migration) #############################3 -from collections.abc import Iterator, Sequence -from pathlib import Path - from my.core import get_files - +from typing import Sequence, Iterator +from pathlib import Path def inputs() -> Sequence[Path]: return get_files(config.export_path) from datetime import datetime - import pytz + + from goodrexport import dal @@ -65,6 +61,7 @@ def books() -> Iterator[dal.Book]: ####### # todo ok, not sure these really belong here... +from my.core.common import datetime_aware @dataclass class Event: dt: datetime_aware diff --git a/my/google/maps/_android_protobuf.py b/my/google/maps/_android_protobuf.py deleted file mode 100644 index 615623d..0000000 --- a/my/google/maps/_android_protobuf.py +++ /dev/null @@ -1,113 +0,0 @@ -from my.core import __NOT_HPI_MODULE__ # isort: skip - -# NOTE: this tool was quite useful https://github.com/aj3423/aproto - -from google.protobuf import descriptor_pb2, descriptor_pool, message_factory - -TYPE_STRING = descriptor_pb2.FieldDescriptorProto.TYPE_STRING -TYPE_BYTES = descriptor_pb2.FieldDescriptorProto.TYPE_BYTES -TYPE_UINT64 = descriptor_pb2.FieldDescriptorProto.TYPE_UINT64 -TYPE_MESSAGE = descriptor_pb2.FieldDescriptorProto.TYPE_MESSAGE - -OPTIONAL = descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL -REQUIRED = descriptor_pb2.FieldDescriptorProto.LABEL_REQUIRED - - -def get_place_protos(): - f1 = descriptor_pb2.DescriptorProto(name='xf1') - # TODO 2 -> 5 is address? 2 -> 6 is a pair of coordinates - f1.field.add(name='title', number=3, type=TYPE_STRING, label=REQUIRED) - f1.field.add(name='note' , number=4, type=TYPE_STRING, label=OPTIONAL) - # TODO what's the difference between required and optional? doesn't impact decoding? - - ts = descriptor_pb2.DescriptorProto(name='Timestamp') - ts.field.add(name='seconds', number=1, type=TYPE_UINT64, label=REQUIRED) - ts.field.add(name='nanos' , number=2, type=TYPE_UINT64, label=REQUIRED) - - f1.field.add(name='created', number=10 ,type=TYPE_MESSAGE, label=REQUIRED, type_name=ts.name) - f1.field.add(name='updated', number=11 ,type=TYPE_MESSAGE, label=REQUIRED, type_name=ts.name) - - f2 = descriptor_pb2.DescriptorProto(name='xf2') - f2.field.add(name='addr1', number=2, type=TYPE_STRING, label=REQUIRED) - f2.field.add(name='addr2', number=3, type=TYPE_STRING, label=REQUIRED) - f2.field.add(name='f21' , number=4, type=TYPE_BYTES , label=REQUIRED) - f2.field.add(name='f22' , number=5, type=TYPE_UINT64, label=REQUIRED) - f2.field.add(name='f23' , number=6, type=TYPE_STRING, label=REQUIRED) - # NOTE: this also contains place ID - - f3 = descriptor_pb2.DescriptorProto(name='xf3') - # NOTE: looks like it's the same as 'updated' from above?? - f3.field.add(name='f31', number=1, type=TYPE_UINT64, label=OPTIONAL) - - descriptor_proto = descriptor_pb2.DescriptorProto(name='PlaceParser') - descriptor_proto.field.add(name='f1', number=1, type=TYPE_MESSAGE, label=REQUIRED, type_name=f1.name) - descriptor_proto.field.add(name='f2', number=2, type=TYPE_MESSAGE, label=REQUIRED, type_name=f2.name) - descriptor_proto.field.add(name='f3', number=3, type=TYPE_MESSAGE, label=OPTIONAL, type_name=f3.name) - descriptor_proto.field.add(name='f4', number=4, type=TYPE_STRING , label=OPTIONAL) - # NOTE: f4 is the list id - - return [descriptor_proto, ts, f1, f2, f3] - - -def get_labeled_protos(): - address = descriptor_pb2.DescriptorProto(name='address') - # 1: address - # 2: parts of address (multiple) - # 3: full address - address.field.add(name='full', number=3, type=TYPE_STRING, label=REQUIRED) - - main = descriptor_pb2.DescriptorProto(name='LabeledParser') - # field 1 contains item type and item id - main.field.add(name='title' , number=3, type=TYPE_STRING , label=REQUIRED) - main.field.add(name='address', number=5, type=TYPE_MESSAGE, label=OPTIONAL, type_name=address.name) - - return [main, address] - - -def get_list_protos(): - f1 = descriptor_pb2.DescriptorProto(name='xf1') - f1.field.add(name='name', number=5, type=TYPE_STRING, label=REQUIRED) - - main = descriptor_pb2.DescriptorProto(name='ListParser') - main.field.add(name='f1', number=1, type=TYPE_MESSAGE, label=REQUIRED, type_name=f1.name) - main.field.add(name='f2', number=2, type=TYPE_STRING , label=REQUIRED) - - return [main, f1] - - -def make_parser(main, *extras): - file_descriptor_proto = descriptor_pb2.FileDescriptorProto(name='dynamic.proto', package='dynamic_package') - for proto in [main, *extras]: - file_descriptor_proto.message_type.add().CopyFrom(proto) - - pool = descriptor_pool.DescriptorPool() - file_descriptor = pool.Add(file_descriptor_proto) - - message_descriptor = pool.FindMessageTypeByName(f'{file_descriptor_proto.package}.{main.name}') - factory = message_factory.MessageFactory(pool) - dynamic_message_class = factory.GetPrototype(message_descriptor) - - return dynamic_message_class - - -place_parser_class = make_parser(*get_place_protos()) -labeled_parser_class = make_parser(*get_labeled_protos()) -list_parser_class = make_parser(*get_list_protos()) - - -def parse_place(blob: bytes): - m = place_parser_class() - m.ParseFromString(blob) - return m - - -def parse_labeled(blob: bytes): - m = labeled_parser_class() - m.ParseFromString(blob) - return m - - -def parse_list(blob: bytes): - msg = list_parser_class() - msg.ParseFromString(blob) - return msg diff --git a/my/google/maps/android.py b/my/google/maps/android.py deleted file mode 100644 index 95ecacf..0000000 --- a/my/google/maps/android.py +++ /dev/null @@ -1,202 +0,0 @@ -""" -Extracts data from the official Google Maps app for Android (uses gmm_sync.db for now) -""" -from __future__ import annotations - -REQUIRES = [ - "protobuf", # for parsing blobs from the database -] - -from collections.abc import Iterator, Sequence -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Any -from urllib.parse import quote - -from my.core import LazyLogger, Paths, Res, datetime_aware, get_files -from my.core.common import unique_everseen -from my.core.sqlite import sqlite_connection - -from ._android_protobuf import parse_labeled, parse_list, parse_place - -import my.config # isort: skip - -logger = LazyLogger(__name__) - - -@dataclass -class config(my.config.google.maps.android): - # paths[s]/glob to the exported sqlite databases - export_path: Paths - - -def inputs() -> Sequence[Path]: - # TODO note sure if need to use all dbs? possibly the last one contains everything? - return get_files(config.export_path) - - -PlaceId = str -ListId = str -ListName = str - - -@dataclass(eq=True, frozen=True) -class Location: - lat: float - lon: float - - @property - def url(self) -> str: - return f'https://maps.google.com/?q={self.lat},{self.lon}' - - -@dataclass(unsafe_hash=True) -class Place: - id: PlaceId - list_name: ListName # TODO maybe best to keep list id? - created_at: datetime_aware # TODO double check it's utc? - updated_at: datetime_aware # TODO double check it's utc? - title: str - location: Location - address: str | None - note: str | None - - @property - def place_url(self) -> str: - title = quote(self.title) - return f'https://www.google.com/maps/place/{title}/data=!4m2!3m1!1s{self.id}' - - @property - def location_url(self) -> str: - return self.location.url - - -def _process_one(f: Path): - with sqlite_connection(f, row_factory='row') as conn: - msg: Any - - lists: dict[ListId, ListName] = {} - for row in conn.execute('SELECT * FROM sync_item_data WHERE corpus == 13'): # 13 looks like lists (e.g. saved/favorited etc) - server_id = row['server_id'] - - if server_id is None: - # this is the case for Travel plans, Followed places, Offers - # todo alternatively could use string_index column instead maybe? - continue - - blob = row['item_proto'] - msg = parse_list(blob) - name = msg.f1.name - lists[server_id] = name - - for row in conn.execute('SELECT * FROM sync_item_data WHERE corpus == 11'): # this looks like 'Labeled' list - ts = row['timestamp'] / 1000 - created = datetime.fromtimestamp(ts, tz=timezone.utc) - - server_id = row['server_id'] - [item_type, item_id] = server_id.split(':') - if item_type != '3': - # the ones that are not 3 are home/work address? - continue - - blob = row['item_proto'] - msg = parse_labeled(blob) - address = msg.address.full - if address == '': - address = None - - location = Location(lat=row['latitude_e6'] / 1e6, lon=row['longitude_e6'] / 1e6) - - yield Place( - id=item_id, - list_name='Labeled', - created_at=created, - updated_at=created, # doesn't look like it has 'updated'? - title=msg.title, - location=location, - address=address, - note=None, # don't think these allow notes - ) - - for row in conn.execute('SELECT * FROM sync_item_data WHERE corpus == 14'): # this looks like actual individual places - server_id = row['server_id'] - [list_id, _, id1, id2] = server_id.split(':') - item_id = f'{id1}:{id2}' - - list_name = lists[list_id] - - blob = row['item_proto'] - msg = parse_place(blob) - title = msg.f1.title - note = msg.f1.note - if note == '': # seems that protobuf does that? - note = None - - # TODO double check timezone - created = datetime.fromtimestamp(msg.f1.created.seconds, tz=timezone.utc).replace(microsecond=msg.f1.created.nanos // 1000) - - # NOTE: this one seems to be the same as row['timestamp'] - updated = datetime.fromtimestamp(msg.f1.updated.seconds, tz=timezone.utc).replace(microsecond=msg.f1.updated.nanos // 1000) - - address = msg.f2.addr1 # NOTE: there is also addr2, but they seem identical :shrug: - if address == '': - address = None - - location = Location(lat=row['latitude_e6'] / 1e6, lon=row['longitude_e6'] / 1e6) - - place = Place( - id=item_id, - list_name=list_name, - created_at=created, - updated_at=updated, - title=title, - location=location, - address=address, - note=note, - ) - - # ugh. in my case it's violated by one place by about 1 second?? - # assert place.created_at <= place.updated_at - yield place - - -def saved() -> Iterator[Res[Place]]: - def it() -> Iterator[Res[Place]]: - paths = inputs() - total = len(paths) - width = len(str(total)) - for idx, path in enumerate(paths): - logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}') - yield from _process_one(path) - return unique_everseen(it) - - -# Summary of databases on Android (as of 20240101) -# -1_optimized_threads.notifications.db -- empty -# 1_optimized_threads.notifications.db -- empty -# 1_tasks.notifications.db -- empty -# -1_threads.notifications.db -- empty -# 1_threads.notifications.db -- doesn't look like anything interested, some trip anniversaries etc? -# 1_thread_surveys.notifications.db -- empty -# 2_threads.notifications.db -- empty -# accounts.notifications.db -- just one row with account id -# brella_example_store -- empty -# gmm_myplaces.db -- contains just a few places? I think it's a subset of "Labeled" -# gmm_storage.db -- pretty huge, like 50Mb. I suspect it contains cache for places on maps or something -# gmm_sync.db -- processed above -# gnp_fcm_database -- list of accounts -# google_app_measurement_local.db -- empty -# inbox_notifications.db -- nothing interesting -# _room_notifications.db -- trip anniversaties? -# lighter_messaging_1.db -- empty -# lighter_messaging_2.db -- empty -# lighter_registration.db -- empty -# peopleCache__com.google_14.db -- contacts cache or something -# portable_geller_.db -- looks like analytics -# primes_example_store -- looks like analytics -# pseudonymous_room_notifications.db -- looks like analytics -# ue3.db -- empty -# ugc_photos_location_data.db -- empty -# ugc-sync.db -- empty -# updates-tab-visit.db -- empty diff --git a/my/google/takeout/html.py b/my/google/takeout/html.py index 3f2b5db..5d65a86 100644 --- a/my/google/takeout/html.py +++ b/my/google/takeout/html.py @@ -2,22 +2,17 @@ Google Takeout exports: browsing history, search/youtube/google play activity ''' -from __future__ import annotations - -from my.core import __NOT_HPI_MODULE__ # isort: skip - -import re -from collections.abc import Iterable -from datetime import datetime from enum import Enum -from html.parser import HTMLParser +import re from pathlib import Path -from typing import Any, Callable +from datetime import datetime, timezone +from html.parser import HTMLParser +from typing import List, Optional, Any, Callable, Iterable, Tuple +from collections import OrderedDict from urllib.parse import unquote -import pytz +from ...core.time import abbr_to_timezone -from my.core.time import abbr_to_timezone # NOTE: https://bugs.python.org/issue22377 %Z doesn't work properly _TIME_FORMATS = [ @@ -34,13 +29,12 @@ def parse_dt(s: str) -> datetime: # old takeouts didn't have timezone # hopefully it was utc? Legacy, so no that much of an issue anymore.. # todo although maybe worth adding timezone from location provider? - # note: need to use pytz here for localize call later - tz = pytz.utc + tz = timezone.utc else: s, tzabbr = s.rsplit(maxsplit=1) tz = abbr_to_timezone(tzabbr) - dt: datetime | None = None + dt: Optional[datetime] = None for fmt in _TIME_FORMATS: try: dt = datetime.strptime(s, fmt) @@ -77,7 +71,7 @@ class State(Enum): Url = str Title = str -Parsed = tuple[datetime, Url, Title] +Parsed = Tuple[datetime, Url, Title] Callback = Callable[[datetime, Url, Title], None] @@ -87,9 +81,9 @@ class TakeoutHTMLParser(HTMLParser): super().__init__() self.state: State = State.OUTSIDE - self.title_parts: list[str] = [] - self.title: str | None = None - self.url: str | None = None + self.title_parts: List[str] = [] + self.title: Optional[str] = None + self.url: Optional[str] = None self.callback = callback @@ -97,8 +91,8 @@ class TakeoutHTMLParser(HTMLParser): def handle_starttag(self, tag, attrs): if self.state == State.INSIDE and tag == 'a': self.state = State.PARSING_LINK - [hr] = (v for k, v in attrs if k == 'href') - assert hr is not None + attrs = OrderedDict(attrs) + hr = attrs['href'] # sometimes it's starts with this prefix, it's apparently clicks from google search? or visits from chrome address line? who knows... # TODO handle http? @@ -126,7 +120,7 @@ class TakeoutHTMLParser(HTMLParser): # JamiexxVEVO # Jun 21, 2018, 5:48:34 AM # Products: - # YouTube + #  YouTube def handle_data(self, data): if self.state == State.OUTSIDE: if data[:-1].strip() in ("Watched", "Visited"): @@ -152,7 +146,7 @@ class TakeoutHTMLParser(HTMLParser): def read_html(tpath: Path, file: str) -> Iterable[Parsed]: - results: list[Parsed] = [] + results: List[Parsed] = [] def cb(dt: datetime, url: Url, title: Title) -> None: results.append((dt, url, title)) parser = TakeoutHTMLParser(callback=cb) @@ -160,3 +154,5 @@ def read_html(tpath: Path, file: str) -> Iterable[Parsed]: data = fo.read() parser.feed(data) return results + +from ...core import __NOT_HPI_MODULE__ diff --git a/my/google/takeout/parser.py b/my/google/takeout/parser.py index 13fd04a..96acfff 100644 --- a/my/google/takeout/parser.py +++ b/my/google/takeout/parser.py @@ -1,7 +1,7 @@ """ -Parses Google Takeout using [[https://github.com/purarue/google_takeout_parser][google_takeout_parser]] +Parses Google Takeout using [[https://github.com/seanbreckenridge/google_takeout_parser][google_takeout_parser]] -See [[https://github.com/purarue/google_takeout_parser][google_takeout_parser]] for more information +See [[https://github.com/seanbreckenridge/google_takeout_parser][google_takeout_parser]] for more information about how to export and organize your takeouts If the DISABLE_TAKEOUT_CACHE environment variable is set, this won't cache individual @@ -12,31 +12,26 @@ zip files of the exports, which are temporarily unpacked while creating the cachew cache """ -REQUIRES = ["git+https://github.com/purarue/google_takeout_parser"] +REQUIRES = ["git+https://github.com/seanbreckenridge/google_takeout_parser"] -import os -from collections.abc import Sequence from contextlib import ExitStack -from dataclasses import dataclass +import os +from typing import List, Sequence, cast from pathlib import Path -from typing import cast - -from google_takeout_parser.parse_html.html_time_utils import ABBR_TIMEZONES - -from my.core import Paths, Stats, get_files, make_config, make_logger, stat -from my.core.cachew import mcachew +from my.core import make_config, dataclass +from my.core.common import Stats, LazyLogger, mcachew, get_files, Paths from my.core.error import ErrorPolicy from my.core.structure import match_structure -from my.core.time import user_forced +from my.core.time import user_forced +from google_takeout_parser.parse_html.html_time_utils import ABBR_TIMEZONES ABBR_TIMEZONES.extend(user_forced()) import google_takeout_parser -from google_takeout_parser.merge import CacheResults, GoogleEventSet -from google_takeout_parser.models import BaseEvent from google_takeout_parser.path_dispatch import TakeoutParser +from google_takeout_parser.merge import GoogleEventSet, CacheResults -# see https://github.com/purarue/dotfiles/blob/master/.config/my/my/config/__init__.py for an example +# see https://github.com/seanbreckenridge/dotfiles/blob/master/.config/my/my/config/__init__.py for an example from my.config import google as user_config @@ -55,11 +50,10 @@ class google(user_config): config = make_config(google) -logger = make_logger(__name__, level="warning") +logger = LazyLogger(__name__, level="warning") # patch the takeout parser logger to match the computed loglevel from google_takeout_parser.log import setup as setup_takeout_logger - setup_takeout_logger(logger.level) @@ -70,24 +64,18 @@ def inputs() -> Sequence[Path]: return get_files(config.takeout_path) -try: - from google_takeout_parser.locales.main import get_paths_for_functions - - EXPECTED = tuple(get_paths_for_functions()) - -except ImportError: - EXPECTED = ( - "My Activity", - "Chrome", - "Location History", - "Youtube", - "YouTube and YouTube Music", - ) +EXPECTED = ( + "My Activity", + "Chrome", + "Location History", + "Youtube", + "YouTube and YouTube Music", +) google_takeout_version = str(getattr(google_takeout_parser, '__version__', 'unknown')) -def _cachew_depends_on() -> list[str]: +def _cachew_depends_on() -> List[str]: exports = sorted([str(p) for p in inputs()]) # add google takeout parser pip version to hash, so this re-creates on breaking changes exports.insert(0, f"google_takeout_version: {google_takeout_version}") @@ -96,21 +84,10 @@ def _cachew_depends_on() -> list[str]: # ResultsType is a Union of all of the models in google_takeout_parser @mcachew(depends_on=_cachew_depends_on, logger=logger, force_file=True) -def events(disable_takeout_cache: bool = DISABLE_TAKEOUT_CACHE) -> CacheResults: # noqa: FBT001 +def events(disable_takeout_cache: bool = DISABLE_TAKEOUT_CACHE) -> CacheResults: error_policy = config.error_policy count = 0 emitted = GoogleEventSet() - - try: - emitted_add = emitted.add_if_not_present - except AttributeError: - # compat for older versions of google_takeout_parser which didn't have this method - def emitted_add(other: BaseEvent) -> bool: - if other in emitted: - return False - emitted.add(other) - return True - # reversed shouldn't really matter? but logic is to use newer # takeouts if they're named according to date, since JSON Activity # is nicer than HTML Activity @@ -123,7 +100,7 @@ def events(disable_takeout_cache: bool = DISABLE_TAKEOUT_CACHE) -> CacheResults: else: results = exit_stack.enter_context(match_structure(path, expected=EXPECTED, partial=True)) for m in results: - # e.g. /home/username/data/google_takeout/Takeout-1634932457.zip") -> 'Takeout-1634932457' + # e.g. /home/sean/data/google_takeout/Takeout-1634932457.zip") -> 'Takeout-1634932457' # means that zipped takeouts have nice filenames from cachew cw_id, _, _ = path.name.rpartition(".") # each takeout result is cached as well, in individual databases per-type @@ -139,15 +116,18 @@ def events(disable_takeout_cache: bool = DISABLE_TAKEOUT_CACHE) -> CacheResults: elif error_policy == 'drop': pass continue - - if emitted_add(event): - yield event # type: ignore[misc] + if event in emitted: + continue + emitted.add(event) + yield event # type: ignore[misc] logger.debug( f"HPI Takeout merge: from a total of {count} events, removed {count - len(emitted)} duplicates" ) def stats() -> Stats: + from my.core import stat + return { **stat(events), } diff --git a/my/google/takeout/paths.py b/my/google/takeout/paths.py index 6a523e2..5b53149 100644 --- a/my/google/takeout/paths.py +++ b/my/google/takeout/paths.py @@ -2,57 +2,44 @@ Module for locating and accessing [[https://takeout.google.com][Google Takeout]] data ''' -from __future__ import annotations +from dataclasses import dataclass +from ...core.common import Paths, get_files +from ...core.util import __NOT_HPI_MODULE__ -from my.core import __NOT_HPI_MODULE__ # isort: skip - -from abc import abstractmethod -from collections.abc import Iterable -from pathlib import Path +from my.config import google as user_config from more_itertools import last -from my.core import Paths, get_files - - -class config: - """ - path/paths/glob for the takeout zips - """ - - @property - @abstractmethod - def takeout_path(self) -> Paths: - raise NotImplementedError - +@dataclass +class google(user_config): + takeout_path: Paths # path/paths/glob for the takeout zips +### # TODO rename 'google' to 'takeout'? not sure +from ...core.cfg import make_config +config = make_config(google) -def make_config() -> config: - from my.config import google as user_config - - class combined_config(user_config, config): ... - - return combined_config() +from pathlib import Path +from typing import Optional, Iterable -def get_takeouts(*, path: str | None = None) -> Iterable[Path]: +def get_takeouts(*, path: Optional[str]=None) -> Iterable[Path]: """ Sometimes google splits takeout into multiple archives, so we need to detect the ones that contain the path we need """ - # TODO zip is not great.. + # TODO FIXME zip is not great.. # allow a lambda expression? that way the user could restrict it - cfg = make_config() - for takeout in get_files(cfg.takeout_path, glob='*.zip'): + for takeout in get_files(config.takeout_path, glob='*.zip'): if path is None or (takeout / path).exists(): yield takeout -def get_last_takeout(*, path: str | None = None) -> Path | None: +def get_last_takeout(*, path: Optional[str]=None) -> Optional[Path]: return last(get_takeouts(path=path), default=None) # TODO might be a good idea to merge across multiple takeouts... # perhaps even a special takeout module that deals with all of this automatically? # e.g. accumulate, filter and maybe report useless takeouts? + diff --git a/my/hackernews/common.py b/my/hackernews/common.py index 6990987..0c5ff9b 100644 --- a/my/hackernews/common.py +++ b/my/hackernews/common.py @@ -1,6 +1,6 @@ from typing import Protocol -from my.core import datetime_aware +from my.core import datetime_aware, Json def hackernews_link(id: str) -> str: diff --git a/my/hackernews/dogsheep.py b/my/hackernews/dogsheep.py index 8303284..de6c58d 100644 --- a/my/hackernews/dogsheep.py +++ b/my/hackernews/dogsheep.py @@ -3,14 +3,14 @@ Hackernews data via Dogsheep [[hacker-news-to-sqlite][https://github.com/dogshee """ from __future__ import annotations -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path +from typing import Iterator, Sequence, Optional -import my.config -from my.core import Paths, Res, datetime_aware, get_files +from my.core import get_files, Paths, Res, datetime_aware from my.core.sqlite import sqlite_connection +import my.config from .common import hackernews_link @@ -33,9 +33,9 @@ class Item: id: str type: str created: datetime_aware # checked and it's utc - title: str | None # only present for Story - text_html: str | None # should be present for Comment and might for Story - url: str | None # might be present for Story + title: Optional[str] # only present for Story + text_html: Optional[str] # should be present for Comment and might for Story + url: Optional[str] # might be present for Story # todo process 'deleted'? fields? # todo process 'parent'? diff --git a/my/hackernews/harmonic.py b/my/hackernews/harmonic.py index 08a82e6..6070510 100644 --- a/my/hackernews/harmonic.py +++ b/my/hackernews/harmonic.py @@ -1,22 +1,17 @@ """ [[https://play.google.com/store/apps/details?id=com.simon.harmonichackernews][Harmonic]] app for Hackernews """ +REQUIRES = ['lxml'] -from __future__ import annotations - -REQUIRES = ['lxml', 'orjson'] - -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime, timezone +import json from pathlib import Path -from typing import Any, TypedDict, cast +from typing import Any, Dict, Iterator, List, Optional, Sequence, TypedDict, cast -import orjson from lxml import etree -from more_itertools import one +from more_itertools import unique_everseen, one -import my.config from my.core import ( Paths, Res, @@ -26,18 +21,15 @@ from my.core import ( make_logger, stat, ) -from my.core.common import unique_everseen - -from .common import SavedBase, hackernews_link - -import my.config # isort: skip +from .common import hackernews_link, SavedBase +from my.config import harmonic as user_config logger = make_logger(__name__) @dataclass -class harmonic(my.config.harmonic): +class harmonic(user_config): export_path: Paths @@ -50,7 +42,7 @@ class Cached(TypedDict): created_at_i: int id: str points: int - test: str | None + test: Optional[str] title: str type: str # TODO Literal['story', 'comment']? comments are only in 'children' field tho url: str @@ -84,10 +76,6 @@ class Saved(SavedBase): def hackernews_link(self) -> str: return hackernews_link(self.uid) - def __hash__(self) -> int: - # meh. but seems like the easiest and fastest way to hash a dict? - return hash(orjson.dumps(self.raw)) - _PREFIX = 'com.simon.harmonichackernews.KEY_SHARED_PREFERENCES' @@ -101,16 +89,16 @@ def _saved() -> Iterator[Res[Saved]]: # TODO defensive for each item! tr = etree.parse(path) - res = one(cast(list[Any], tr.xpath(f'//*[@name="{_PREFIX}_CACHED_STORIES_STRINGS"]'))) + res = one(cast(List[Any], tr.xpath(f'//*[@name="{_PREFIX}_CACHED_STORIES_STRINGS"]'))) cached_ids = [x.text.split('-')[0] for x in res] - cached: dict[str, Cached] = {} + cached: Dict[str, Cached] = {} for sid in cached_ids: - res = one(cast(list[Any], tr.xpath(f'//*[@name="{_PREFIX}_CACHED_STORY{sid}"]'))) - j = orjson.loads(res.text) + res = one(cast(List[Any], tr.xpath(f'//*[@name="{_PREFIX}_CACHED_STORY{sid}"]'))) + j = json.loads(res.text) cached[sid] = j - res = one(cast(list[Any], tr.xpath(f'//*[@name="{_PREFIX}_BOOKMARKS"]'))) + res = one(cast(List[Any], tr.xpath(f'//*[@name="{_PREFIX}_BOOKMARKS"]'))) for x in res.text.split('-'): ids, item_timestamp = x.split('q') # not sure if timestamp is any useful? @@ -124,7 +112,7 @@ def _saved() -> Iterator[Res[Saved]]: def saved() -> Iterator[Res[Saved]]: - yield from unique_everseen(_saved) + yield from unique_everseen(_saved()) def stats() -> Stats: diff --git a/my/hackernews/materialistic.py b/my/hackernews/materialistic.py index ccf285b..4d5cd47 100644 --- a/my/hackernews/materialistic.py +++ b/my/hackernews/materialistic.py @@ -1,20 +1,19 @@ """ [[https://play.google.com/store/apps/details?id=io.github.hidroh.materialistic][Materialistic]] app for Hackernews """ -from collections.abc import Iterator, Sequence from datetime import datetime, timezone from pathlib import Path -from typing import Any, NamedTuple +from typing import Any, Dict, Iterator, NamedTuple, Sequence from more_itertools import unique_everseen -from my.core import datetime_aware, get_files, make_logger +from my.core import get_files, datetime_aware, make_logger from my.core.sqlite import sqlite_connection +from my.config import materialistic as config # todo migrate config to my.hackernews.materialistic + from .common import hackernews_link -# todo migrate config to my.hackernews.materialistic -from my.config import materialistic as config # isort: skip logger = make_logger(__name__) @@ -23,7 +22,7 @@ def inputs() -> Sequence[Path]: return get_files(config.export_path) -Row = dict[str, Any] +Row = Dict[str, Any] class Saved(NamedTuple): diff --git a/my/hypothesis.py b/my/hypothesis.py index 15e854b..55fff64 100644 --- a/my/hypothesis.py +++ b/my/hypothesis.py @@ -4,22 +4,20 @@ REQUIRES = [ 'git+https://github.com/karlicoss/hypexport', ] -from collections.abc import Iterator, Sequence from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING +from typing import Iterator, Sequence, TYPE_CHECKING from my.core import ( + get_files, + stat, Paths, Res, Stats, - get_files, - stat, ) from my.core.cfg import make_config from my.core.hpi_compat import always_supports_sequence - -import my.config # isort: skip +import my.config @dataclass @@ -43,7 +41,6 @@ except ModuleNotFoundError as e: dal = pre_pip_dal_handler('hypexport', e, config, requires=REQUIRES) -DAL = dal.DAL Highlight = dal.Highlight Page = dal.Page @@ -52,8 +49,8 @@ def inputs() -> Sequence[Path]: return get_files(config.export_path) -def _dal() -> DAL: - return DAL(inputs()) +def _dal() -> dal.DAL: + return dal.DAL(inputs()) # TODO they are in reverse chronological order... diff --git a/my/instagram/all.py b/my/instagram/all.py index ce78409..8007399 100644 --- a/my/instagram/all.py +++ b/my/instagram/all.py @@ -1,10 +1,11 @@ -from collections.abc import Iterator +from typing import Iterator -from my.core import Res, Stats, stat +from my.core import Res, stat, Stats from my.core.source import import_source from .common import Message, _merge_messages + src_gdpr = import_source(module_name='my.instagram.gdpr') @src_gdpr def _messages_gdpr() -> Iterator[Res[Message]]: @@ -23,7 +24,7 @@ def messages() -> Iterator[Res[Message]]: # TODO in general best to prefer android, it has more data # - message ids # - usernames are correct for Android data - # - thread ids more meaningful? + # - thread ids more meaninful? # but for now prefer gdpr prefix since it makes a bit things a bit more consistent? # e.g. a new batch of android exports can throw off ids if we rely on it for mapping yield from _merge_messages( diff --git a/my/instagram/android.py b/my/instagram/android.py index 12c11d3..ea5ee35 100644 --- a/my/instagram/android.py +++ b/my/instagram/android.py @@ -3,29 +3,30 @@ Bumble data from Android app database (in =/data/data/com.instagram.android/data """ from __future__ import annotations -import json -import sqlite3 -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime +import json from pathlib import Path +import sqlite3 +from typing import Iterator, Sequence, Optional, Dict, Union from my.core import ( - Json, - Paths, - Res, - assert_never, - datetime_naive, get_files, + Paths, make_config, make_logger, + datetime_naive, + Json, + Res, + assert_never, ) -from my.core.cachew import mcachew from my.core.common import unique_everseen +from my.core.cachew import mcachew from my.core.error import echain -from my.core.sqlite import select, sqlite_connect_immutable +from my.core.sqlite import sqlite_connect_immutable, select + +from my.config import instagram as user_config -from my.config import instagram as user_config # isort: skip logger = make_logger(__name__) @@ -37,8 +38,8 @@ class instagram_android_config(user_config.android): # sadly doesn't seem easy to extract user's own handle/name from the db... # todo maybe makes more sense to keep in parent class? not sure... - username: str | None = None - full_name: str | None = None + username: Optional[str] = None + full_name: Optional[str] = None config = make_config(instagram_android_config) @@ -91,7 +92,7 @@ class MessageError(RuntimeError): super().__init__(msg_id, *rest) self.rest = rest - def __hash__(self): + def __hash__(self, other): return hash(self.rest) def __eq__(self, other) -> bool: @@ -100,13 +101,13 @@ class MessageError(RuntimeError): return self.rest == other.rest -def _parse_message(j: Json) -> _Message | None: +def _parse_message(j: Json) -> Optional[_Message]: id = j['item_id'] t = j['item_type'] tid = j['thread_key']['thread_id'] uid = j['user_id'] created = datetime.fromtimestamp(int(j['timestamp']) / 1_000_000) - text: str | None = None + text: Optional[str] = None if t == 'text': text = j['text'] elif t == 'reel_share': @@ -132,7 +133,7 @@ def _parse_message(j: Json) -> _Message | None: ) -def _process_db(db: sqlite3.Connection) -> Iterator[Res[User | _Message]]: +def _process_db(db: sqlite3.Connection) -> Iterator[Res[Union[User, _Message]]]: # TODO ugh. seems like no way to extract username? # sometimes messages (e.g. media_share) contain it in message field # but generally it's not present. ugh @@ -174,7 +175,7 @@ def _process_db(db: sqlite3.Connection) -> Iterator[Res[User | _Message]]: yield e -def _entities() -> Iterator[Res[User | _Message]]: +def _entities() -> Iterator[Res[Union[User, _Message]]]: # NOTE: definitely need to merge multiple, app seems to recycle old messages # TODO: hmm hard to guarantee timestamp ordering when we use synthetic input data... # todo use TypedDict? @@ -193,7 +194,7 @@ def _entities() -> Iterator[Res[User | _Message]]: @mcachew(depends_on=inputs) def messages() -> Iterator[Res[Message]]: - id2user: dict[str, User] = {} + id2user: Dict[str, User] = {} for x in unique_everseen(_entities): if isinstance(x, Exception): yield x diff --git a/my/instagram/common.py b/my/instagram/common.py index 17d130f..36c6b83 100644 --- a/my/instagram/common.py +++ b/my/instagram/common.py @@ -1,10 +1,9 @@ -from collections.abc import Iterator from dataclasses import replace from datetime import datetime from itertools import chain -from typing import Any, Protocol +from typing import Iterator, Dict, Any, Protocol -from my.core import Res, warn_if_empty +from my.core import warn_if_empty, Res class User(Protocol): @@ -41,7 +40,7 @@ def _merge_messages(*sources: Iterator[Res[Message]]) -> Iterator[Res[Message]]: # ugh. seems that GDPR thread ids are completely uncorrelated to any android ids (tried searching over all sqlite dump) # so the only way to correlate is to try and match messages # we also can't use unique_everseen here, otherwise will never get a chance to unify threads - mmap: dict[str, Message] = {} + mmap: Dict[str, Message] = {} thread_map = {} user_map = {} @@ -61,7 +60,7 @@ def _merge_messages(*sources: Iterator[Res[Message]]) -> Iterator[Res[Message]]: user_map[m.user.id] = mm.user else: # not emitted yet, need to emit - repls: dict[str, Any] = {} + repls: Dict[str, Any] = {} tid = thread_map.get(m.thread_id) if tid is not None: repls['thread_id'] = tid @@ -69,6 +68,6 @@ def _merge_messages(*sources: Iterator[Res[Message]]) -> Iterator[Res[Message]]: if user is not None: repls['user'] = user if len(repls) > 0: - m = replace(m, **repls) # type: ignore[type-var] # ugh mypy is confused because of Protocol? + m = replace(m, **repls) # type: ignore[type-var, misc] # ugh mypy is confused because of Protocol? mmap[k] = m yield m diff --git a/my/instagram/gdpr.py b/my/instagram/gdpr.py index d417fdb..233f040 100644 --- a/my/instagram/gdpr.py +++ b/my/instagram/gdpr.py @@ -1,28 +1,26 @@ """ Instagram data (uses [[https://www.instagram.com/download/request][official GDPR export]]) """ - -from __future__ import annotations - -import json -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime +import json from pathlib import Path +from typing import Iterator, Sequence, Dict, Union from more_itertools import bucket from my.core import ( + get_files, Paths, + datetime_naive, Res, assert_never, - datetime_naive, - get_files, make_logger, ) from my.core.common import unique_everseen -from my.config import instagram as user_config # isort: skip +from my.config import instagram as user_config + logger = make_logger(__name__) @@ -71,12 +69,12 @@ def _decode(s: str) -> str: return s.encode('latin-1').decode('utf8') -def _entities() -> Iterator[Res[User | _Message]]: +def _entities() -> Iterator[Res[Union[User, _Message]]]: # it's worth processing all previous export -- sometimes instagram removes some metadata from newer ones # NOTE: here there are basically two options # - process inputs as is (from oldest to newest) # this would be more stable wrt newer exports (e.g. existing thread ids won't change) - # the downside is that newer exports seem to have better thread ids, so might be preferable to use them + # the downside is that newer exports seem to have better thread ids, so might be preferrable to use them # - process inputs reversed (from newest to oldest) # the upside is that thread ids/usernames might be better # the downside is that if for example the user renames, thread ids will change _a lot_, might be undesirable.. @@ -85,7 +83,7 @@ def _entities() -> Iterator[Res[User | _Message]]: yield from _entitites_from_path(path) -def _entitites_from_path(path: Path) -> Iterator[Res[User | _Message]]: +def _entitites_from_path(path: Path) -> Iterator[Res[Union[User, _Message]]]: # TODO make sure it works both with plan directory # idelaly get_files should return the right thing, and we won't have to force ZipPath/match_structure here # e.g. possible options are: @@ -105,12 +103,7 @@ def _entitites_from_path(path: Path) -> Iterator[Res[User | _Message]]: # old path, used up to somewhere between feb-aug 2022 personal_info = path / 'account_information' - personal_info_json = personal_info / 'personal_information.json' - if not personal_info_json.exists(): - # new path, started somewhere around april 2024 - personal_info_json = personal_info / 'personal_information' / 'personal_information.json' - - j = json.loads(personal_info_json.read_text()) + j = json.loads((personal_info / 'personal_information.json').read_text()) [profile] = j['profile_user'] pdata = profile['string_map_data'] username = pdata['Username']['value'] @@ -137,7 +130,7 @@ def _entitites_from_path(path: Path) -> Iterator[Res[User | _Message]]: j = json.loads(ffile.read_text()) id_len = 10 - # NOTE: I'm not actually sure it's other user's id.., since it corresponds to the whole conversation + # NOTE: I'm not actually sure it's other user's id.., since it corresponds to the whole converstation # but I stared a bit at these ids vs database ids and can't see any way to find the correspondence :( # so basically the only way to merge is to actually try some magic and correlate timestamps/message texts? # another option is perhaps to query user id from username with some free API @@ -203,7 +196,7 @@ def _entitites_from_path(path: Path) -> Iterator[Res[User | _Message]]: # TODO basically copy pasted from android.py... hmm def messages() -> Iterator[Res[Message]]: - id2user: dict[str, User] = {} + id2user: Dict[str, User] = {} for x in unique_everseen(_entities): if isinstance(x, Exception): yield x diff --git a/my/instapaper.py b/my/instapaper.py index d79e7e4..df1f70b 100644 --- a/my/instapaper.py +++ b/my/instapaper.py @@ -7,10 +7,10 @@ REQUIRES = [ from dataclasses import dataclass -from my.config import instapaper as user_config - from .core import Paths +from my.config import instapaper as user_config + @dataclass class instapaper(user_config): @@ -22,7 +22,6 @@ class instapaper(user_config): from .core.cfg import make_config - config = make_config(instapaper) @@ -40,12 +39,9 @@ Bookmark = dal.Bookmark Page = dal.Page -from collections.abc import Iterable, Sequence +from typing import Sequence, Iterable from pathlib import Path - from .core import get_files - - def inputs() -> Sequence[Path]: return get_files(config.export_path) diff --git a/my/ip/all.py b/my/ip/all.py index c267383..f4cdb37 100644 --- a/my/ip/all.py +++ b/my/ip/all.py @@ -3,15 +3,16 @@ An example all.py stub module that provides ip data To use this, you'd add IP providers that yield IPs to the 'ips' function -For an example of how this could be used, see https://github.com/purarue/HPI/tree/master/my/ip +For an example of how this could be used, see https://github.com/seanbreckenridge/HPI/tree/master/my/ip """ -REQUIRES = ["git+https://github.com/purarue/ipgeocache"] +REQUIRES = ["git+https://github.com/seanbreckenridge/ipgeocache"] -from collections.abc import Iterator +from typing import Iterator + +from my.core.common import Stats, warn_if_empty -from my.core import Stats, warn_if_empty from my.ip.common import IP diff --git a/my/ip/common.py b/my/ip/common.py index b551281..244ddc5 100644 --- a/my/ip/common.py +++ b/my/ip/common.py @@ -1,13 +1,12 @@ """ -Provides location/timezone data from IP addresses, using [[https://github.com/purarue/ipgeocache][ipgeocache]] +Provides location/timezone data from IP addresses, using [[https://github.com/seanbreckenridge/ipgeocache][ipgeocache]] """ -from my.core import __NOT_HPI_MODULE__ # isort: skip +from my.core import __NOT_HPI_MODULE__ import ipaddress -from collections.abc import Iterator +from typing import NamedTuple, Iterator, Tuple from datetime import datetime -from typing import NamedTuple import ipgeocache @@ -23,7 +22,7 @@ class IP(NamedTuple): return ipgeocache.get(self.addr) @property - def latlon(self) -> tuple[float, float]: + def latlon(self) -> Tuple[float, float]: loc: str = self.ipgeocache()["loc"] lat, _, lon = loc.partition(",") return float(lat), float(lon) diff --git a/my/jawbone/__init__.py b/my/jawbone/__init__.py index 463d735..4b41242 100644 --- a/my/jawbone/__init__.py +++ b/my/jawbone/__init__.py @@ -1,20 +1,18 @@ -from __future__ import annotations - +from typing import Dict, Any, List, Iterable import json -from collections.abc import Iterable -from datetime import date, datetime, time, timedelta from functools import lru_cache +from datetime import datetime, date, time, timedelta from pathlib import Path -from typing import Any import pytz -from my.core import make_logger +from ..core.common import LazyLogger -logger = make_logger(__name__) +logger = LazyLogger(__name__) from my.config import jawbone as config # type: ignore[attr-defined] + BDIR = config.export_dir PHASES_FILE = BDIR / 'phases.json' SLEEPS_FILE = BDIR / 'sleeps.json' @@ -24,7 +22,7 @@ GRAPHS_DIR = BDIR / 'graphs' XID = str # TODO how to shared with backup thing? -Phases = dict[XID, Any] +Phases = Dict[XID, Any] @lru_cache(1) def get_phases() -> Phases: return json.loads(PHASES_FILE.read_text()) @@ -89,7 +87,7 @@ class SleepEntry: # TODO might be useful to cache these?? @property - def phases(self) -> list[datetime]: + def phases(self) -> List[datetime]: # TODO make sure they are consistent with emfit? return [self._fromts(i['time']) for i in get_phases()[self.xid]] @@ -100,39 +98,32 @@ class SleepEntry: return str(self) -def load_sleeps() -> list[SleepEntry]: +def load_sleeps() -> List[SleepEntry]: sleeps = json.loads(SLEEPS_FILE.read_text()) return [SleepEntry(js) for js in sleeps] -from ..core.error import Res, extract_error_datetime, set_error_datetime - +from ..core.error import Res, set_error_datetime, extract_error_datetime def pre_dataframe() -> Iterable[Res[SleepEntry]]: - from more_itertools import bucket - sleeps = load_sleeps() # todo emit error if graph doesn't exist?? sleeps = [s for s in sleeps if s.graph.exists()] # TODO careful.. - - bucketed = bucket(sleeps, key=lambda s: s.date_) - - for dd in bucketed: - group = list(bucketed[dd]) + from ..core.common import group_by_key + for dd, group in group_by_key(sleeps, key=lambda s: s.date_).items(): if len(group) == 1: yield group[0] else: err = RuntimeError(f'Multiple sleeps per night, not supported yet: {group}') - dt = datetime.combine(dd, time.min) - set_error_datetime(err, dt=dt) + set_error_datetime(err, dt=dd) # type: ignore[arg-type] logger.exception(err) yield err def dataframe(): - dicts: list[dict[str, Any]] = [] + dicts: List[Dict[str, Any]] = [] for s in pre_dataframe(): - d: dict[str, Any] + d: Dict[str, Any] if isinstance(s, Exception): dt = extract_error_datetime(s) d = { @@ -164,6 +155,9 @@ def stats(): #### NOTE: most of the stuff below is deprecated and remnants of my old code! #### sorry for it, feel free to remove if you don't need it +import matplotlib.pyplot as plt # type: ignore +from matplotlib.figure import Figure # type: ignore +from matplotlib.axes import Axes # type: ignore def hhmm(time: datetime): return time.strftime("%H:%M") @@ -174,15 +168,14 @@ def hhmm(time: datetime): # fromstart = time - sleep.created # return fromstart / tick +import matplotlib.dates as mdates # type: ignore -def plot_one(sleep: SleepEntry, fig, axes, xlims=None, *, showtext=True): - import matplotlib.dates as mdates # type: ignore[import-not-found] - +def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None, showtext=True): span = sleep.completed - sleep.created print(f"{sleep.xid} span: {span}") # pip install imageio - from imageio import imread # type: ignore + from imageio import imread # type: ignore img = imread(sleep.graph) # all of them are 300x300 images apparently @@ -240,7 +233,7 @@ def plot_one(sleep: SleepEntry, fig, axes, xlims=None, *, showtext=True): # axes.title.set_size(10) if showtext: - axes.text(xlims[1] - timedelta(hours=1.5), 20, str(sleep)) + axes.text(xlims[1] - timedelta(hours=1.5), 20, str(sleep),) # plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) @@ -260,10 +253,7 @@ def predicate(sleep: SleepEntry): # TODO move to dashboard -def plot() -> None: - import matplotlib.pyplot as plt # type: ignore[import-not-found] - from matplotlib.figure import Figure # type: ignore[import-not-found] - +def plot(): # TODO FIXME melatonin data melatonin_data = {} # type: ignore[var-annotated] @@ -275,7 +265,7 @@ def plot() -> None: fig: Figure = plt.figure(figsize=(15, sleeps_count * 1)) axarr = fig.subplots(nrows=len(sleeps)) - for (sleep, axes) in zip(sleeps, axarr): + for i, (sleep, axes) in enumerate(zip(sleeps, axarr)): plot_one(sleep, fig, axes, showtext=True) used = melatonin_data.get(sleep.date_, None) sused: str diff --git a/my/jawbone/plots.py b/my/jawbone/plots.py index 5968412..5dcb63d 100755 --- a/my/jawbone/plots.py +++ b/my/jawbone/plots.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 # TODO this should be in dashboard +from pathlib import Path # from kython.plotting import * from csv import DictReader -from pathlib import Path -from typing import Any, NamedTuple +from itertools import islice -import matplotlib.pylab as pylab # type: ignore +from typing import Dict, Any, NamedTuple # sleep = [] # with open('2017.csv', 'r') as fo: @@ -13,14 +13,16 @@ import matplotlib.pylab as pylab # type: ignore # for line in islice(reader, 0, 10): # sleep # print(line) -import matplotlib.pyplot as plt # type: ignore + +import matplotlib.pyplot as plt # type: ignore from numpy import genfromtxt +import matplotlib.pylab as pylab # type: ignore pylab.rcParams['figure.figsize'] = (32.0, 24.0) pylab.rcParams['font.size'] = 10 jawboneDataFeatures = Path(__file__).parent / 'features.csv' # Data File Path -featureDesc: dict[str, str] = {} +featureDesc: Dict[str, str] = {} for x in genfromtxt(jawboneDataFeatures, dtype='unicode', delimiter=','): featureDesc[x[0]] = x[1] @@ -51,7 +53,7 @@ class SleepData(NamedTuple): quality: float # ??? @classmethod - def from_jawbone_dict(cls, d: dict[str, Any]): + def from_jawbone_dict(cls, d: Dict[str, Any]): return cls( date=d['DATE'], asleep_time=_safe_mins(_safe_float(d['s_asleep_time'])), @@ -74,7 +76,7 @@ class SleepData(NamedTuple): def iter_useful(data_file: str): - with Path(data_file).open() as fo: + with open(data_file) as fo: reader = DictReader(fo) for d in reader: dt = SleepData.from_jawbone_dict(d) @@ -94,7 +96,6 @@ files = [ ] from kython import concat, parse_date # type: ignore - useful = concat(*(list(iter_useful(str(f))) for f in files)) # for u in useful: @@ -108,7 +109,6 @@ dates = [parse_date(u.date, yearfirst=True, dayfirst=False) for u in useful] # TODO don't need this anymore? it's gonna be in dashboards package from kython.plotting import plot_timestamped # type: ignore - for attr, lims, mavg, fig in [ ('light', (0, 400), 5, None), ('deep', (0, 600), 5, None), diff --git a/my/kobo.py b/my/kobo.py index b4a1575..85bc50f 100644 --- a/my/kobo.py +++ b/my/kobo.py @@ -7,22 +7,21 @@ REQUIRES = [ 'kobuddy', ] -from collections.abc import Iterator from dataclasses import dataclass - -import kobuddy -from kobuddy import * -from kobuddy import Highlight, get_highlights +from typing import Iterator from my.core import ( - Paths, - Stats, get_files, stat, + Paths, + Stats, ) from my.core.cfg import make_config +import my.config -import my.config # isort: skip +import kobuddy +from kobuddy import Highlight, get_highlights +from kobuddy import * @dataclass @@ -52,7 +51,7 @@ def stats() -> Stats: ## TODO hmm. not sure if all this really belongs here?... perhaps orger? -from typing import Callable, Union +from typing import Callable, Union, List # TODO maybe type over T? _Predicate = Callable[[str], bool] @@ -70,17 +69,17 @@ def from_predicatish(p: Predicatish) -> _Predicate: return p -def by_annotation(predicatish: Predicatish, **kwargs) -> list[Highlight]: +def by_annotation(predicatish: Predicatish, **kwargs) -> List[Highlight]: pred = from_predicatish(predicatish) - res: list[Highlight] = [] + res: List[Highlight] = [] for h in get_highlights(**kwargs): if pred(h.annotation): res.append(h) return res -def get_todos() -> list[Highlight]: +def get_todos() -> List[Highlight]: def with_todo(ann): if ann is None: ann = '' diff --git a/my/kython/kompress.py b/my/kython/kompress.py index a5d9c29..01e24e4 100644 --- a/my/kython/kompress.py +++ b/my/kython/kompress.py @@ -1,4 +1,5 @@ -from my.core import __NOT_HPI_MODULE__, warnings +from my.core import __NOT_HPI_MODULE__ +from my.core import warnings warnings.high('my.kython.kompress is deprecated, please use "kompress" library directly. See https://github.com/karlicoss/kompress') diff --git a/my/lastfm.py b/my/lastfm.py index cd9fa8b..90484b4 100644 --- a/my/lastfm.py +++ b/my/lastfm.py @@ -2,10 +2,9 @@ Last.fm scrobbles ''' -from dataclasses import dataclass - +from my.core import Paths, dataclass, make_logger from my.config import lastfm as user_config -from my.core import Json, Paths, get_files, make_logger + logger = make_logger(__name__) @@ -19,17 +18,15 @@ class lastfm(user_config): from my.core.cfg import make_config - config = make_config(lastfm) -import json -from collections.abc import Iterable, Sequence from datetime import datetime, timezone +import json from pathlib import Path -from typing import NamedTuple +from typing import NamedTuple, Sequence, Iterable -from my.core.cachew import mcachew +from my.core.common import mcachew, Json, get_files def inputs() -> Sequence[Path]: @@ -78,19 +75,16 @@ def scrobbles() -> Iterable[Scrobble]: yield Scrobble(raw=raw) -from my.core import Stats, stat - - +from my.core import stat, Stats def stats() -> Stats: return stat(scrobbles) def fill_influxdb() -> None: from my.core import influxdb - # todo needs to be more automatic - sd = ({ - 'dt': x.dt, - 'track': x.track, - } for x in scrobbles()) + sd = (dict( + dt=x.dt, + track=x.track, + ) for x in scrobbles()) influxdb.fill(sd, measurement=__name__) diff --git a/my/location/all.py b/my/location/all.py index c6e8cab..fd88721 100644 --- a/my/location/all.py +++ b/my/location/all.py @@ -2,13 +2,14 @@ Merges location data from multiple sources """ -from collections.abc import Iterator +from typing import Iterator -from my.core import LazyLogger, Stats +from my.core import Stats, LazyLogger from my.core.source import import_source from .common import Location + logger = LazyLogger(__name__, level="warning") diff --git a/my/location/common.py b/my/location/common.py index 4c47ef0..7824bef 100644 --- a/my/location/common.py +++ b/my/location/common.py @@ -1,13 +1,12 @@ -from my.core import __NOT_HPI_MODULE__ # isort: skip - -from collections.abc import Iterable, Iterator -from dataclasses import dataclass from datetime import date, datetime -from typing import Optional, Protocol, TextIO, Union +from typing import Union, Tuple, Optional, Iterable, TextIO, Iterator, Protocol +from dataclasses import dataclass + +from my.core import __NOT_HPI_MODULE__ DateIsh = Union[datetime, date, str] -LatLon = tuple[float, float] +LatLon = Tuple[float, float] class LocationProtocol(Protocol): @@ -42,9 +41,9 @@ def locations_to_gpx(locations: Iterable[LocationProtocol], buffer: TextIO) -> I try: import gpxpy.gpx except ImportError as ie: - from my.core.warnings import high + from my.core.warnings import warn - high("gpxpy not installed, cannot write to gpx. 'pip install gpxpy'") + warn("gpxpy not installed, cannot write to gpx. 'pip install gpxpy'") raise ie gpx = gpxpy.gpx.GPX() @@ -71,7 +70,7 @@ def locations_to_gpx(locations: Iterable[LocationProtocol], buffer: TextIO) -> I ) except AttributeError: yield TypeError( - f"Expected a Location or Location-like object, got {type(location)} {location!r}" + f"Expected a Location or Location-like object, got {type(location)} {repr(location)}" ) continue gpx_segment.points.append(point) diff --git a/my/location/fallback/all.py b/my/location/fallback/all.py index d340148..0c7b8cd 100644 --- a/my/location/fallback/all.py +++ b/my/location/fallback/all.py @@ -1,16 +1,14 @@ # TODO: add config here which passes kwargs to estimate_from (under_accuracy) # overwritable by passing the kwarg name here to the top-level estimate_location -from __future__ import annotations - -from collections.abc import Iterator +from typing import Iterator, Optional from my.core.source import import_source from my.location.fallback.common import ( - DateExact, - FallbackLocation, - LocationEstimator, estimate_from, + FallbackLocation, + DateExact, + LocationEstimator, ) @@ -26,7 +24,7 @@ def fallback_estimators() -> Iterator[LocationEstimator]: yield _home_estimate -def estimate_location(dt: DateExact, *, first_match: bool=False, under_accuracy: int | None = None) -> FallbackLocation: +def estimate_location(dt: DateExact, first_match: bool=False, under_accuracy: Optional[int] = None) -> FallbackLocation: loc = estimate_from(dt, estimators=list(fallback_estimators()), first_match=first_match, under_accuracy=under_accuracy) # should never happen if the user has home configured if loc is None: diff --git a/my/location/fallback/common.py b/my/location/fallback/common.py index 622b2f5..fd508c6 100644 --- a/my/location/fallback/common.py +++ b/my/location/fallback/common.py @@ -1,12 +1,9 @@ from __future__ import annotations - -from collections.abc import Iterator, Sequence from dataclasses import dataclass +from typing import Optional, Callable, Sequence, Iterator, List, Union from datetime import datetime, timedelta, timezone -from typing import Callable, Union - -from ..common import Location, LocationProtocol +from ..common import LocationProtocol, Location DateExact = Union[datetime, float, int] # float/int as epoch timestamps Second = float @@ -16,12 +13,12 @@ class FallbackLocation(LocationProtocol): lat: float lon: float dt: datetime - duration: Second | None = None - accuracy: float | None = None - elevation: float | None = None - datasource: str | None = None # which module provided this, useful for debugging + duration: Optional[Second] = None + accuracy: Optional[float] = None + elevation: Optional[float] = None + datasource: Optional[str] = None # which module provided this, useful for debugging - def to_location(self, *, end: bool = False) -> Location: + def to_location(self, end: bool = False) -> Location: ''' by default the start date is used for the location If end is True, the start date + duration is used @@ -46,9 +43,9 @@ class FallbackLocation(LocationProtocol): lon: float, dt: datetime, end_dt: datetime, - accuracy: float | None = None, - elevation: float | None = None, - datasource: str | None = None, + accuracy: Optional[float] = None, + elevation: Optional[float] = None, + datasource: Optional[str] = None, ) -> FallbackLocation: ''' Create FallbackLocation from a start date and an end date @@ -96,13 +93,13 @@ def estimate_from( estimators: LocationEstimators, *, first_match: bool = False, - under_accuracy: int | None = None, -) -> FallbackLocation | None: + under_accuracy: Optional[int] = None, +) -> Optional[FallbackLocation]: ''' first_match: if True, return the first location found under_accuracy: if set, only return locations with accuracy under this value ''' - found: list[FallbackLocation] = [] + found: List[FallbackLocation] = [] for loc in _iter_estimate_from(dt, estimators): if under_accuracy is not None and loc.accuracy is not None and loc.accuracy > under_accuracy: continue diff --git a/my/location/fallback/via_home.py b/my/location/fallback/via_home.py index f88fee0..259dcaa 100644 --- a/my/location/fallback/via_home.py +++ b/my/location/fallback/via_home.py @@ -2,22 +2,25 @@ Simple location provider, serving as a fallback when more detailed data isn't available ''' -from __future__ import annotations - -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime, time, timezone -from functools import cache -from typing import cast +from functools import lru_cache +from typing import Sequence, Tuple, Union, cast, List, Iterator from my.config import location as user_config -from my.location.common import DateIsh, LatLon -from my.location.fallback.common import DateExact, FallbackLocation +from my.location.common import LatLon, DateIsh +from my.location.fallback.common import FallbackLocation, DateExact @dataclass class Config(user_config): - home: LatLon | Sequence[tuple[DateIsh, LatLon]] + home: Union[ + LatLon, # either single, 'current' location + Sequence[Tuple[ # or, a sequence of location history + DateIsh, # date when you moved to + LatLon, # the location + ]] + ] # default ~30km accuracy # this is called 'home_accuracy' since it lives on the base location.config object, @@ -26,13 +29,13 @@ class Config(user_config): # TODO could make current Optional and somehow determine from system settings? @property - def _history(self) -> Sequence[tuple[datetime, LatLon]]: + def _history(self) -> Sequence[Tuple[datetime, LatLon]]: home1 = self.home # todo ugh, can't test for isnstance LatLon, it's a tuple itself - home2: Sequence[tuple[DateIsh, LatLon]] + home2: Sequence[Tuple[DateIsh, LatLon]] if isinstance(home1[0], tuple): # already a sequence - home2 = cast(Sequence[tuple[DateIsh, LatLon]], home1) + home2 = cast(Sequence[Tuple[DateIsh, LatLon]], home1) else: # must be a pair of coordinates. also doesn't really matter which date to pick? loc = cast(LatLon, home1) @@ -52,16 +55,15 @@ class Config(user_config): if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) res.append((dt, loc)) - res = sorted(res, key=lambda p: p[0]) + res = list(sorted(res, key=lambda p: p[0])) return res from ...core.cfg import make_config - config = make_config(Config) -@cache +@lru_cache(maxsize=None) def get_location(dt: datetime) -> LatLon: ''' Interpolates the location at dt @@ -72,8 +74,8 @@ def get_location(dt: datetime) -> LatLon: # TODO: in python3.8, use functools.cached_property instead? -@cache -def homes_cached() -> list[tuple[datetime, LatLon]]: +@lru_cache(maxsize=None) +def homes_cached() -> List[Tuple[datetime, LatLon]]: return list(config._history) @@ -90,12 +92,13 @@ def estimate_location(dt: DateExact) -> Iterator[FallbackLocation]: dt=datetime.fromtimestamp(d, timezone.utc), datasource='via_home') return - - # I guess the most reasonable is to fallback on the first location - lat, lon = hist[-1][1] - yield FallbackLocation( - lat=lat, - lon=lon, - accuracy=config.home_accuracy, - dt=datetime.fromtimestamp(d, timezone.utc), - datasource='via_home') + else: + # I guess the most reasonable is to fallback on the first location + lat, lon = hist[-1][1] + yield FallbackLocation( + lat=lat, + lon=lon, + accuracy=config.home_accuracy, + dt=datetime.fromtimestamp(d, timezone.utc), + datasource='via_home') + return diff --git a/my/location/fallback/via_ip.py b/my/location/fallback/via_ip.py index 8b50878..f637552 100644 --- a/my/location/fallback/via_ip.py +++ b/my/location/fallback/via_ip.py @@ -2,13 +2,12 @@ Converts IP addresses provided by my.location.ip to estimated locations """ -REQUIRES = ["git+https://github.com/purarue/ipgeocache"] +REQUIRES = ["git+https://github.com/seanbreckenridge/ipgeocache"] -from dataclasses import dataclass from datetime import timedelta +from my.core import dataclass, Stats, make_config from my.config import location -from my.core import Stats, make_config from my.core.warnings import medium @@ -24,21 +23,19 @@ class ip_config(location.via_ip): config = make_config(ip_config) -from collections.abc import Iterator from functools import lru_cache +from typing import Iterator, List -from my.core import make_logger +from my.core.common import LazyLogger from my.core.compat import bisect_left +from my.ip.all import ips from my.location.common import Location -from my.location.fallback.common import DateExact, FallbackLocation, _datetime_timestamp +from my.location.fallback.common import FallbackLocation, DateExact, _datetime_timestamp -logger = make_logger(__name__, level="warning") +logger = LazyLogger(__name__, level="warning") def fallback_locations() -> Iterator[FallbackLocation]: - # prefer late import since ips get overridden in tests - from my.ip.all import ips - dur = config.for_duration.total_seconds() for ip in ips(): lat, lon = ip.latlon @@ -60,7 +57,7 @@ def locations() -> Iterator[Location]: @lru_cache(1) -def _sorted_fallback_locations() -> list[FallbackLocation]: +def _sorted_fallback_locations() -> List[FallbackLocation]: fl = list(filter(lambda l: l.duration is not None, fallback_locations())) logger.debug(f"Fallback locations: {len(fl)}, sorting...:") fl.sort(key=lambda l: l.dt.timestamp()) diff --git a/my/location/google.py b/my/location/google.py index 750c847..ed37231 100644 --- a/my/location/google.py +++ b/my/location/google.py @@ -3,28 +3,28 @@ Location data from Google Takeout DEPRECATED: setup my.google.takeout.parser and use my.location.google_takeout instead """ -from __future__ import annotations REQUIRES = [ 'geopy', # checking that coordinates are valid 'ijson', ] -import re -from collections.abc import Iterable, Sequence from datetime import datetime, timezone from itertools import islice from pathlib import Path -from subprocess import PIPE, Popen -from typing import IO, NamedTuple, Optional +from subprocess import Popen, PIPE +from typing import Iterable, NamedTuple, Optional, Sequence, IO, Tuple +import re # pip3 install geopy -import geopy # type: ignore +import geopy # type: ignore -from my.core import Stats, make_logger, stat, warnings -from my.core.cachew import cache_dir, mcachew +from ..core.common import LazyLogger, mcachew +from ..core.cachew import cache_dir -warnings.high("Please set up my.google.takeout.parser module for better takeout support") +from my.core.warnings import high + +high("Please set up my.google.takeout.parser module for better takeout support") # otherwise uses ijson @@ -32,7 +32,7 @@ warnings.high("Please set up my.google.takeout.parser module for better takeout USE_GREP = False -logger = make_logger(__name__) +logger = LazyLogger(__name__) class Location(NamedTuple): @@ -42,7 +42,7 @@ class Location(NamedTuple): alt: Optional[float] -TsLatLon = tuple[int, int, int] +TsLatLon = Tuple[int, int, int] def _iter_via_ijson(fo) -> Iterable[TsLatLon]: @@ -50,10 +50,11 @@ def _iter_via_ijson(fo) -> Iterable[TsLatLon]: # todo extract to common? try: # pip3 install ijson cffi - import ijson.backends.yajl2_cffi as ijson # type: ignore + import ijson.backends.yajl2_cffi as ijson # type: ignore except: - warnings.medium("Falling back to default ijson because 'cffi' backend isn't found. It's up to 2x faster, you might want to check it out") - import ijson # type: ignore + import warnings + warnings.warn("Falling back to default ijson because 'cffi' backend isn't found. It's up to 2x faster, you might want to check it out") + import ijson # type: ignore for d in ijson.items(fo, 'locations.item'): yield ( @@ -104,8 +105,7 @@ def _iter_locations_fo(fit) -> Iterable[Location]: errors += 1 if float(errors) / total > 0.01: # todo make defensive? - # todo exceptiongroup? - raise RuntimeError('too many errors! aborting') # noqa: B904 + raise RuntimeError('too many errors! aborting') else: continue @@ -164,6 +164,7 @@ def locations(**kwargs) -> Iterable[Location]: return _iter_locations(path=last_takeout, **kwargs) +from ..core.common import stat, Stats def stats() -> Stats: return stat(locations) diff --git a/my/location/google_takeout.py b/my/location/google_takeout.py index 8613257..a1c1403 100644 --- a/my/location/google_takeout.py +++ b/my/location/google_takeout.py @@ -2,16 +2,14 @@ Extracts locations using google_takeout_parser -- no shared code with the deprecated my.location.google """ -REQUIRES = ["git+https://github.com/purarue/google_takeout_parser"] +REQUIRES = ["git+https://github.com/seanbreckenridge/google_takeout_parser"] -from collections.abc import Iterator +from typing import Iterator +from my.google.takeout.parser import events, _cachew_depends_on from google_takeout_parser.models import Location as GoogleLocation -from my.core import LazyLogger, Stats, stat -from my.core.cachew import mcachew -from my.google.takeout.parser import _cachew_depends_on, events - +from my.core.common import mcachew, LazyLogger, Stats from .common import Location logger = LazyLogger(__name__) @@ -35,4 +33,6 @@ def locations() -> Iterator[Location]: def stats() -> Stats: - return stat(locations) + from my.core import stat + + return {**stat(locations)} diff --git a/my/location/google_takeout_semantic.py b/my/location/google_takeout_semantic.py index e84a932..fcf7f01 100644 --- a/my/location/google_takeout_semantic.py +++ b/my/location/google_takeout_semantic.py @@ -5,26 +5,22 @@ Extracts semantic location history using google_takeout_parser # This is a separate module to prevent ImportError and a new config block from breaking # previously functional my.location.google_takeout locations -REQUIRES = ["git+https://github.com/purarue/google_takeout_parser"] +REQUIRES = ["git+https://github.com/seanbreckenridge/google_takeout_parser"] -from collections.abc import Iterator -from dataclasses import dataclass +from typing import Iterator, List +from my.google.takeout.parser import events, _cachew_depends_on as _parser_cachew_depends_on from google_takeout_parser.models import PlaceVisit as SemanticLocation -from my.core import LazyLogger, Stats, make_config, stat -from my.core.cachew import mcachew +from my.core import dataclass, make_config +from my.core.common import mcachew, LazyLogger, Stats from my.core.error import Res -from my.google.takeout.parser import _cachew_depends_on as _parser_cachew_depends_on -from my.google.takeout.parser import events - from .common import Location logger = LazyLogger(__name__) from my.config import location as user_config - @dataclass class semantic_locations_config(user_config.google_takeout_semantic): # a value between 0 and 100, 100 being the most confident @@ -39,7 +35,7 @@ config = make_config(semantic_locations_config) # add config to cachew dependency so it recomputes on config changes -def _cachew_depends_on() -> list[str]: +def _cachew_depends_on() -> List[str]: dep = _parser_cachew_depends_on() dep.insert(0, f"require_confidence={config.require_confidence} accuracy={config.accuracy}") return dep @@ -58,9 +54,8 @@ def locations() -> Iterator[Res[Location]]: for g in events(): if isinstance(g, SemanticLocation): - visitConfidence = g.visitConfidence - if visitConfidence is None or visitConfidence < require_confidence: - logger.debug(f"Skipping {g} due to low confidence ({visitConfidence}))") + if g.visitConfidence < require_confidence: + logger.debug(f"Skipping {g} due to low confidence ({g.visitConfidence}))") continue yield Location( lon=g.lng, @@ -76,4 +71,6 @@ def locations() -> Iterator[Res[Location]]: def stats() -> Stats: - return stat(locations) + from my.core import stat + + return {**stat(locations)} diff --git a/my/location/gpslogger.py b/my/location/gpslogger.py index bbbf70e..17f828f 100644 --- a/my/location/gpslogger.py +++ b/my/location/gpslogger.py @@ -4,11 +4,8 @@ Parse [[https://github.com/mendhak/gpslogger][gpslogger]] .gpx (xml) files REQUIRES = ["gpxpy"] - -from dataclasses import dataclass - from my.config import location -from my.core import Paths +from my.core import Paths, dataclass @dataclass @@ -20,21 +17,19 @@ class config(location.gpslogger): accuracy: float = 50.0 -from collections.abc import Iterator, Sequence -from datetime import datetime, timezone from itertools import chain +from datetime import datetime, timezone from pathlib import Path +from typing import Iterator, Sequence, List import gpxpy -from gpxpy.gpx import GPXXMLSyntaxException from more_itertools import unique_everseen -from my.core import LazyLogger, Stats -from my.core.cachew import mcachew -from my.core.common import get_files - +from my.core import Stats, LazyLogger +from my.core.common import get_files, mcachew from .common import Location + logger = LazyLogger(__name__, level="warning") def _input_sort_key(path: Path) -> str: @@ -49,7 +44,7 @@ def inputs() -> Sequence[Path]: return sorted(get_files(config.export_path, glob="*.gpx", sort=False), key=_input_sort_key) -def _cachew_depends_on() -> list[float]: +def _cachew_depends_on() -> List[float]: return [p.stat().st_mtime for p in inputs()] @@ -63,11 +58,7 @@ def locations() -> Iterator[Location]: def _extract_locations(path: Path) -> Iterator[Location]: with path.open("r") as gf: - try: - gpx_obj = gpxpy.parse(gf) - except GPXXMLSyntaxException as e: - logger.warning("failed to parse XML %s: %s", path, e) - return + gpx_obj = gpxpy.parse(gf) for track in gpx_obj.tracks: for segment in track.segments: for point in segment.points: diff --git a/my/location/home.py b/my/location/home.py index c82dda7..f6e6978 100644 --- a/my/location/home.py +++ b/my/location/home.py @@ -1,7 +1,7 @@ -from my.core.warnings import high - from .fallback.via_home import * +from my.core.warnings import high + high( "my.location.home is deprecated, use my.location.fallback.via_home instead, or estimate locations using the higher-level my.location.fallback.all.estimate_location" ) diff --git a/my/location/via_ip.py b/my/location/via_ip.py index 240ec5f..df48f8b 100644 --- a/my/location/via_ip.py +++ b/my/location/via_ip.py @@ -1,7 +1,7 @@ -REQUIRES = ["git+https://github.com/purarue/ipgeocache"] - -from my.core.warnings import high +REQUIRES = ["git+https://github.com/seanbreckenridge/ipgeocache"] from .fallback.via_ip import * +from my.core.warnings import high + high("my.location.via_ip is deprecated, use my.location.fallback.via_ip instead") diff --git a/my/materialistic.py b/my/materialistic.py index 45af3f9..8a6a997 100644 --- a/my/materialistic.py +++ b/my/materialistic.py @@ -1,5 +1,4 @@ from .core.warnings import high - high("DEPRECATED! Please use my.hackernews.materialistic instead.") from .hackernews.materialistic import * diff --git a/my/media/imdb.py b/my/media/imdb.py index 131f6a7..b7ecbde 100644 --- a/my/media/imdb.py +++ b/my/media/imdb.py @@ -1,12 +1,11 @@ +#!/usr/bin/env python3 import csv -from collections.abc import Iterator from datetime import datetime -from typing import NamedTuple +from typing import Iterator, List, NamedTuple -from my.core import get_files - -from my.config import imdb as config # isort: skip +from ..core import get_files +from my.config import imdb as config def _get_last(): return max(get_files(config.export_path)) @@ -23,7 +22,7 @@ def iter_movies() -> Iterator[Movie]: with last.open() as fo: reader = csv.DictReader(fo) - for line in reader: + for i, line in enumerate(reader): # TODO extract directors?? title = line['Title'] rating = int(line['You rated']) @@ -33,8 +32,8 @@ def iter_movies() -> Iterator[Movie]: yield Movie(created=created, title=title, rating=rating) -def get_movies() -> list[Movie]: - return sorted(iter_movies(), key=lambda m: m.created) +def get_movies() -> List[Movie]: + return list(sorted(iter_movies(), key=lambda m: m.created)) def test(): diff --git a/my/media/youtube.py b/my/media/youtube.py index 9a38c43..efaa74b 100644 --- a/my/media/youtube.py +++ b/my/media/youtube.py @@ -1,10 +1,5 @@ -from my.core import __NOT_HPI_MODULE__ # isort: skip - -from typing import TYPE_CHECKING - -from my.core.warnings import high - +from ..core.warnings import high high("DEPRECATED! Please use my.youtube.takeout instead.") +from ..core.util import __NOT_HPI_MODULE__ -if not TYPE_CHECKING: - from my.youtube.takeout import * +from ..youtube.takeout import * diff --git a/my/monzo/monzoexport.py b/my/monzo/monzoexport.py deleted file mode 100644 index f5e1cd1..0000000 --- a/my/monzo/monzoexport.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Monzo transactions data (using https://github.com/karlicoss/monzoexport ) -""" -REQUIRES = [ - 'git+https://github.com/karlicoss/monzoexport', -] - -from collections.abc import Iterator, Sequence -from dataclasses import dataclass -from pathlib import Path - -from my.core import ( - Paths, - get_files, - make_logger, -) - -import my.config # isort: skip - - -@dataclass -class config(my.config.monzo.monzoexport): - ''' - Uses [[https://github.com/karlicoss/monzoexport][ghexport]] outputs. - ''' - - export_path: Paths - '''path[s]/glob to the exported JSON data''' - - -logger = make_logger(__name__) - - -def inputs() -> Sequence[Path]: - return get_files(config.export_path) - - -import monzoexport.dal as dal - - -def _dal() -> dal.DAL: - return dal.DAL(inputs()) - - -def transactions() -> Iterator[dal.MonzoTransaction]: - return _dal().transactions() diff --git a/my/orgmode.py b/my/orgmode.py index 10f53c0..8293b74 100644 --- a/my/orgmode.py +++ b/my/orgmode.py @@ -1,53 +1,39 @@ ''' Programmatic access and queries to org-mode files on the filesystem ''' -from __future__ import annotations REQUIRES = [ 'orgparse', ] -import re -from collections.abc import Iterable, Sequence from datetime import datetime from pathlib import Path -from typing import NamedTuple, Optional +import re +from typing import List, Sequence, Iterable, NamedTuple, Optional, Tuple -import orgparse - -from my.core import Paths, Stats, get_files, stat -from my.core.cachew import cache_dir, mcachew +from my.core import get_files +from my.core.common import mcachew +from my.core.cachew import cache_dir from my.core.orgmode import collect +from my.config import orgmode as user_config -class config: - paths: Paths - - -def make_config() -> config: - from my.config import orgmode as user_config - - class combined_config(user_config, config): ... - - return combined_config() +import orgparse # temporary? hack to cache org-mode notes class OrgNote(NamedTuple): created: Optional[datetime] heading: str - tags: list[str] + tags: List[str] def inputs() -> Sequence[Path]: - cfg = make_config() - return get_files(cfg.paths) + return get_files(user_config.paths) _rgx = re.compile(orgparse.date.gene_timestamp_regex(brtype='inactive'), re.VERBOSE) - - -def _created(n: orgparse.OrgNode) -> tuple[datetime | None, str]: +def _created(n: orgparse.OrgNode) -> Tuple[Optional[datetime], str]: heading = n.heading # meh.. support in orgparse? pp = {} if n.is_root() else n.properties @@ -56,7 +42,7 @@ def _created(n: orgparse.OrgNode) -> tuple[datetime | None, str]: # try to guess from heading m = _rgx.search(heading) if m is not None: - createds = m.group(0) # could be None + createds = m.group(0) # could be None if createds is None: return (None, heading) assert isinstance(createds, str) @@ -70,7 +56,7 @@ def _created(n: orgparse.OrgNode) -> tuple[datetime | None, str]: def to_note(x: orgparse.OrgNode) -> OrgNote: # ugh. hack to merely make it cacheable heading = x.heading - created: datetime | None + created: Optional[datetime] try: c, heading = _created(x) if isinstance(c, datetime): @@ -82,7 +68,7 @@ def to_note(x: orgparse.OrgNode) -> OrgNote: created = None return OrgNote( created=created, - heading=heading, # todo include the body? + heading=heading, # todo include the body? tags=list(x.tags), ) @@ -99,15 +85,14 @@ def _cachew_cache_path(_self, f: Path) -> Path: def _cachew_depends_on(_self, f: Path): return (f, f.stat().st_mtime) - + class Query: def __init__(self, files: Sequence[Path]) -> None: self.files = files # TODO yield errors? @mcachew( - cache_path=_cachew_cache_path, - force_file=True, + cache_path=_cachew_cache_path, force_file=True, depends_on=_cachew_depends_on, ) def _iterate(self, f: Path) -> Iterable[OrgNote]: @@ -130,8 +115,8 @@ def query() -> Query: return Query(files=inputs()) +from my.core import Stats, stat def stats() -> Stats: def outlines(): return query().all() - return stat(outlines) diff --git a/my/pdfs.py b/my/pdfs.py index eefd573..5355d8a 100644 --- a/my/pdfs.py +++ b/my/pdfs.py @@ -1,66 +1,63 @@ ''' PDF documents and annotations on your filesystem ''' -from __future__ import annotations as _annotations - REQUIRES = [ 'git+https://github.com/0xabu/pdfannots', # todo not sure if should use pypi version? ] -import time -from collections.abc import Iterator, Sequence from datetime import datetime +from dataclasses import dataclass +import io from pathlib import Path -from typing import TYPE_CHECKING, NamedTuple, Optional, Protocol +import time +from typing import NamedTuple, List, Optional, Iterator, Sequence -import pdfannots -from more_itertools import bucket -from my.core import PathIsh, Paths, Stats, get_files, make_logger, stat -from my.core.cachew import mcachew +from my.core import LazyLogger, get_files, Paths, PathIsh +from my.core.cfg import Attrs, make_config +from my.core.common import mcachew, group_by_key from my.core.error import Res, split_errors -class config(Protocol): - @property - def paths(self) -> Paths: - return () # allowed to be empty for 'filelist' logic +import pdfannots - def is_ignored(self, p: Path) -> bool: # noqa: ARG002 + +from my.config import pdfs as user_config + +@dataclass +class pdfs(user_config): + paths: Paths = () # allowed to be empty for 'filelist' logic + + def is_ignored(self, p: Path) -> bool: """ - You can override this in user config if you want to ignore some files that are tooheavy + Used to ignore some extremely heavy files + is_ignored function taken either from config, + or if not defined, it's a function that returns False """ + user_ignore = getattr(user_config, 'is_ignored', None) + if user_ignore is not None: + return user_ignore(p) + return False - -def make_config() -> config: - from my.config import pdfs as user_config - - class migration: - @property - def paths(self) -> Paths: - roots = getattr(user_config, 'roots', None) - if roots is not None: - from my.core.warnings import high - - high('"roots" is deprecated! Use "paths" instead.') - return roots - else: - return () - - class combined_config(user_config, migration, config): ... - - return combined_config() + @staticmethod + def _migration(attrs: Attrs) -> Attrs: + roots = 'roots' + if roots in attrs: # legacy name + attrs['paths'] = attrs[roots] + from my.core.warnings import high + high(f'"{roots}" is deprecated! Use "paths" instead.') + return attrs -logger = make_logger(__name__) +config = make_config(pdfs, migration=pdfs._migration) +logger = LazyLogger(__name__) def inputs() -> Sequence[Path]: - cfg = make_config() - all_files = get_files(cfg.paths, glob='**/*.pdf') - return [p for p in all_files if not cfg.is_ignored(p)] + all_files = get_files(config.paths, glob='**/*.pdf') + return [p for p in all_files if not config.is_ignored(p)] # TODO canonical names/fingerprinting? @@ -74,7 +71,7 @@ class Annotation(NamedTuple): created: Optional[datetime] # note: can be tz unaware in some bad pdfs... @property - def date(self) -> datetime | None: + def date(self) -> Optional[datetime]: # legacy name return self.created @@ -95,11 +92,11 @@ def _as_annotation(*, raw: pdfannots.Annotation, path: str) -> Annotation: ) -def get_annots(p: Path) -> list[Annotation]: +def get_annots(p: Path) -> List[Annotation]: b = time.time() with p.open('rb') as fo: doc = pdfannots.process_file(fo, emit_progress_to=None) - annots = list(doc.iter_annots()) + annots = [a for a in doc.iter_annots()] # also has outlines are kinda like TOC, I don't really need them a = time.time() took = a - b @@ -123,13 +120,14 @@ def _iter_annotations(pdfs: Sequence[Path]) -> Iterator[Res[Annotation]]: # todo how to print to stdout synchronously? # todo global config option not to use pools? useful for debugging.. from concurrent.futures import ProcessPoolExecutor - - from my.core.utils.concurrent import DummyExecutor - + from my.core.common import DummyExecutor workers = None # use 0 for debugging Pool = DummyExecutor if workers == 0 else ProcessPoolExecutor with Pool(workers) as pool: - futures = [pool.submit(get_annots, pdf) for pdf in pdfs] + futures = [ + pool.submit(get_annots, pdf) + for pdf in pdfs + ] for f, pdf in zip(futures, pdfs): try: yield from f.result() @@ -152,41 +150,40 @@ class Pdf(NamedTuple): annotations: Sequence[Annotation] @property - def created(self) -> datetime | None: + def created(self) -> Optional[datetime]: annots = self.annotations return None if len(annots) == 0 else annots[-1].created @property - def date(self) -> datetime | None: + def date(self) -> Optional[datetime]: # legacy return self.created -def annotated_pdfs(*, filelist: Sequence[PathIsh] | None = None) -> Iterator[Res[Pdf]]: +def annotated_pdfs(*, filelist: Optional[Sequence[PathIsh]]=None) -> Iterator[Res[Pdf]]: if filelist is not None: # hacky... keeping it backwards compatible # https://github.com/karlicoss/HPI/pull/74 - from my.config import pdfs as user_config - - user_config.paths = filelist + config.paths = filelist ait = annotations() vit, eit = split_errors(ait, ET=Exception) - bucketed = bucket(vit, key=lambda a: a.path) - for k in bucketed: - g = list(bucketed[k]) + for k, g in group_by_key(vit, key=lambda a: a.path).items(): yield Pdf(path=Path(k), annotations=g) yield from eit +from my.core import stat, Stats def stats() -> Stats: return { - **stat(annotations), + **stat(annotations) , **stat(annotated_pdfs), } ### legacy/misc stuff -if not TYPE_CHECKING: - iter_annotations = annotations +iter_annotations = annotations # for backwards compatibility ### + +# can use 'hpi query my.pdfs.annotations -o pprint' to test +# diff --git a/my/photos/main.py b/my/photos/main.py index f98cb15..c491ac1 100644 --- a/my/photos/main.py +++ b/my/photos/main.py @@ -1,30 +1,26 @@ """ Photos and videos on your filesystem, their GPS and timestamps """ - -from __future__ import annotations - REQUIRES = [ 'geopy', 'magic', ] # NOTE: also uses fdfind to search photos -import json -from collections.abc import Iterable, Iterator from concurrent.futures import ProcessPoolExecutor as Pool from datetime import datetime +import json from pathlib import Path -from typing import NamedTuple, Optional +from typing import Optional, NamedTuple, Iterator, Iterable, List -from geopy.geocoders import Nominatim # type: ignore +from geopy.geocoders import Nominatim # type: ignore -from my.core import LazyLogger -from my.core.cachew import cache_dir, mcachew -from my.core.error import Res, sort_res_by -from my.core.mime import fastermime +from ..core.common import LazyLogger, mcachew, fastermime +from ..core.error import Res, sort_res_by +from ..core.cachew import cache_dir + +from my.config import photos as config # type: ignore[attr-defined] -from my.config import photos as config # type: ignore[attr-defined] # isort: skip logger = LazyLogger(__name__) @@ -46,7 +42,8 @@ class Photo(NamedTuple): for bp in config.paths: if self.path.startswith(bp): return self.path[len(bp):] - raise RuntimeError(f"Weird path {self.path}, can't match against anything") + else: + raise RuntimeError(f"Weird path {self.path}, can't match against anything") @property def name(self) -> str: @@ -58,17 +55,17 @@ class Photo(NamedTuple): return f'{config.base_url}{self._basename}' -from .utils import Exif, ExifTags, convert_ref, dt_from_path, get_exif_from_file +from .utils import get_exif_from_file, ExifTags, Exif, dt_from_path, convert_ref Result = Res[Photo] -def _make_photo_aux(*args, **kwargs) -> list[Result]: +def _make_photo_aux(*args, **kwargs) -> List[Result]: # for the process pool.. return list(_make_photo(*args, **kwargs)) -def _make_photo(photo: Path, mtype: str, *, parent_geo: LatLon | None) -> Iterator[Result]: +def _make_photo(photo: Path, mtype: str, *, parent_geo: Optional[LatLon]) -> Iterator[Result]: exif: Exif - if any(x in mtype for x in ['image/png', 'image/x-ms-bmp', 'video']): + if any(x in mtype for x in {'image/png', 'image/x-ms-bmp', 'video'}): # TODO don't remember why.. logger.debug(f"skipping exif extraction for {photo} due to mime {mtype}") exif = {} @@ -80,7 +77,7 @@ def _make_photo(photo: Path, mtype: str, *, parent_geo: LatLon | None) -> Iterat yield e exif = {} - def _get_geo() -> LatLon | None: + def _get_geo() -> Optional[LatLon]: meta = exif.get(ExifTags.GPSINFO, {}) if ExifTags.LAT in meta and ExifTags.LON in meta: return LatLon( @@ -90,7 +87,7 @@ def _make_photo(photo: Path, mtype: str, *, parent_geo: LatLon | None) -> Iterat return parent_geo # TODO aware on unaware? - def _get_dt() -> datetime | None: + def _get_dt() -> Optional[datetime]: edt = exif.get(ExifTags.DATETIME, None) if edt is not None: dtimes = edt.replace(' 24', ' 00') # jeez maybe log it? @@ -126,7 +123,7 @@ def _make_photo(photo: Path, mtype: str, *, parent_geo: LatLon | None) -> Iterat def _candidates() -> Iterable[Res[str]]: # TODO that could be a bit slow if there are to many extra files? - from subprocess import PIPE, Popen + from subprocess import Popen, PIPE # TODO could extract this to common? # TODO would be nice to reuse get_files (or even let it use find) # that way would be easier to exclude @@ -165,7 +162,7 @@ def _photos(candidates: Iterable[Res[str]]) -> Iterator[Result]: from functools import lru_cache @lru_cache(None) - def get_geo(d: Path) -> LatLon | None: + def get_geo(d: Path) -> Optional[LatLon]: geof = d / 'geo.json' if not geof.exists(): if d == d.parent: @@ -211,13 +208,11 @@ def print_all() -> None: if isinstance(p, Exception): print('ERROR!', p) else: - print(f"{p.dt!s:25} {p.path} {p.geo}") + print(f"{str(p.dt):25} {p.path} {p.geo}") # todo cachew -- improve AttributeError: type object 'tuple' has no attribute '__annotations__' -- improve errors? # todo cachew -- invalidate if function code changed? from ..core import Stats, stat - - def stats() -> Stats: return stat(photos) diff --git a/my/photos/utils.py b/my/photos/utils.py index e88def2..c614c4a 100644 --- a/my/photos/utils.py +++ b/my/photos/utils.py @@ -1,13 +1,11 @@ -from __future__ import annotations - -from ..core import __NOT_HPI_MODULE__ # isort: skip - from pathlib import Path +from typing import Dict import PIL.Image -from PIL.ExifTags import GPSTAGS, TAGS +from PIL.ExifTags import TAGS, GPSTAGS -Exif = dict + +Exif = Dict # TODO PIL.ExifTags.TAGS @@ -64,15 +62,18 @@ def convert_ref(cstr, ref: str) -> float: import re from datetime import datetime +from typing import Optional # TODO surely there is a library that does it?? # TODO this belongs to a private overlay or something # basically have a function that patches up dates after the files were yielded.. _DT_REGEX = re.compile(r'\D(\d{8})\D*(\d{6})\D') -def dt_from_path(p: Path) -> datetime | None: +def dt_from_path(p: Path) -> Optional[datetime]: name = p.stem mm = _DT_REGEX.search(name) if mm is None: return None dates = mm.group(1) + mm.group(2) return datetime.strptime(dates, "%Y%m%d%H%M%S") + +from ..core import __NOT_HPI_MODULE__ diff --git a/my/pinboard.py b/my/pinboard.py index e98dc78..354f15c 100644 --- a/my/pinboard.py +++ b/my/pinboard.py @@ -5,35 +5,22 @@ REQUIRES = [ 'git+https://github.com/karlicoss/pinbexport', ] -from collections.abc import Iterator, Sequence -from dataclasses import dataclass -from pathlib import Path +from my.config import pinboard as config + import pinbexport.dal as pinbexport -from my.core import Paths, Res, get_files - -import my.config # isort: skip - - -@dataclass -class config(my.config.pinboard): # TODO rename to pinboard.pinbexport? - # TODO rename to export_path? - export_dir: Paths - - -# TODO not sure if should keep this import here? Bookmark = pinbexport.Bookmark -def inputs() -> Sequence[Path]: - return get_files(config.export_dir) - - # yep; clearly looks that the purpose of my. package is to wire files to DAL implicitly; otherwise it's just passtrhough. def dal() -> pinbexport.DAL: - return pinbexport.DAL(inputs()) + from .core import get_files + inputs = get_files(config.export_dir) # todo rename to export_path + model = pinbexport.DAL(inputs) + return model -def bookmarks() -> Iterator[Res[pinbexport.Bookmark]]: +from typing import Iterable +def bookmarks() -> Iterable[pinbexport.Bookmark]: return dal().bookmarks() diff --git a/my/pocket.py b/my/pocket.py index ff9a788..b638fba 100644 --- a/my/pocket.py +++ b/my/pocket.py @@ -7,10 +7,10 @@ REQUIRES = [ from dataclasses import dataclass from typing import TYPE_CHECKING -from my.config import pocket as user_config - from .core import Paths +from my.config import pocket as user_config + @dataclass class pocket(user_config): @@ -23,7 +23,6 @@ class pocket(user_config): from .core.cfg import make_config - config = make_config(pocket) @@ -38,7 +37,7 @@ except ModuleNotFoundError as e: Article = dal.Article -from collections.abc import Iterable, Sequence +from typing import Sequence, Iterable # todo not sure if should be defensive against empty? @@ -52,12 +51,9 @@ def articles() -> Iterable[Article]: yield from _dal().articles() -from .core import Stats, stat - - +from .core import stat, Stats def stats() -> Stats: from itertools import chain - from more_itertools import ilen return { **stat(articles), diff --git a/my/polar.py b/my/polar.py index 2172014..fe59d00 100644 --- a/my/polar.py +++ b/my/polar.py @@ -1,12 +1,11 @@ """ [[https://github.com/burtonator/polar-bookshelf][Polar]] articles and highlights """ -from __future__ import annotations - from pathlib import Path -from typing import TYPE_CHECKING, cast +from typing import cast, TYPE_CHECKING -import my.config # isort: skip + +import my.config # todo use something similar to tz.via_location for config fallback if not TYPE_CHECKING: @@ -21,36 +20,32 @@ if user_config is None: pass -from dataclasses import dataclass - from .core import PathIsh - - +from dataclasses import dataclass @dataclass class polar(user_config): ''' Polar config is optional, you only need it if you want to specify custom 'polar_dir' ''' - polar_dir: PathIsh = Path('~/.polar').expanduser() # noqa: RUF009 + polar_dir: PathIsh = Path('~/.polar').expanduser() defensive: bool = True # pass False if you want it to fail faster on errors (useful for debugging) from .core import make_config - config = make_config(polar) # todo not sure where it keeps stuff on Windows? # https://github.com/burtonator/polar-bookshelf/issues/296 -import json -from collections.abc import Iterable, Sequence from datetime import datetime -from typing import NamedTuple +from typing import List, Dict, Iterable, NamedTuple, Sequence, Optional +import json -from .core import Json, LazyLogger, Res -from .core.compat import fromisoformat +from .core import LazyLogger, Json, Res +from .core.common import isoparse from .core.error import echain, sort_res_by -from .core.konsume import Wdict, Zoomable, wrap +from .core.konsume import wrap, Zoomable, Wdict + logger = LazyLogger(__name__) @@ -70,7 +65,7 @@ class Highlight(NamedTuple): comments: Sequence[Comment] tags: Sequence[str] page: int # 1-indexed - color: str | None = None + color: Optional[str] = None Uid = str @@ -78,7 +73,7 @@ class Book(NamedTuple): created: datetime uid: Uid path: Path - title: str | None + title: Optional[str] # TODO hmmm. I think this needs to be defensive as well... # think about it later. items: Sequence[Highlight] @@ -134,7 +129,7 @@ class Loader: pi['dimensions'].consume_all() # TODO how to make it nicer? - cmap: dict[Hid, list[Comment]] = {} + cmap: Dict[Hid, List[Comment]] = {} vals = list(comments) for v in vals: cid = v['id'].zoom() @@ -150,7 +145,7 @@ class Loader: cmap[hlid] = ccs ccs.append(Comment( cid=cid.value, - created=fromisoformat(crt.value), + created=isoparse(crt.value), text=html.value, # TODO perhaps coonvert from html to text or org? )) v.consume() @@ -168,10 +163,10 @@ class Loader: h['rects'].ignore() # TODO make it more generic.. - htags: list[str] = [] + htags: List[str] = [] if 'tags' in h: ht = h['tags'].zoom() - for _k, v in list(ht.items()): + for k, v in list(ht.items()): ctag = v.zoom() ctag['id'].consume() ct = ctag['label'].zoom() @@ -188,7 +183,7 @@ class Loader: yield Highlight( hid=hid, - created=fromisoformat(crt), + created=isoparse(crt), selection=text, comments=tuple(comments), tags=tuple(htags), @@ -204,7 +199,7 @@ class Loader: def load_items(self, metas: Json) -> Iterable[Highlight]: - for _p, meta in metas.items(): # noqa: PERF102 + for p, meta in metas.items(): with wrap(meta, throw=not config.defensive) as meta: yield from self.load_item(meta) @@ -226,7 +221,7 @@ class Loader: path = Path(config.polar_dir) / 'stash' / filename yield Book( - created=fromisoformat(added), + created=isoparse(added), uid=self.uid, path=path, title=title, @@ -247,7 +242,7 @@ def iter_entries() -> Iterable[Result]: yield err -def get_entries() -> list[Result]: +def get_entries() -> List[Result]: # sorting by first annotation is reasonable I guess??? # todo perhaps worth making it a pattern? X() returns iterable, get_X returns reasonably sorted list? return list(sort_res_by(iter_entries(), key=lambda e: e.created)) diff --git a/my/reddit/__init__.py b/my/reddit/__init__.py index 982901a..e81aaf9 100644 --- a/my/reddit/__init__.py +++ b/my/reddit/__init__.py @@ -9,7 +9,7 @@ since that allows for easier overriding using namespace packages See https://github.com/karlicoss/HPI/blob/master/doc/MODULE_DESIGN.org#allpy for more info. """ -# prevent it from appearing in modules list/doctor +# prevent it from apprearing in modules list/doctor from ..core import __NOT_HPI_MODULE__ # kinda annoying to keep it, but it's so legacy 'hpi module install my.reddit' works @@ -20,7 +20,6 @@ REQUIRES = [ from my.core.hpi_compat import handle_legacy_import - is_legacy_import = handle_legacy_import( parent_module_name=__name__, legacy_submodule_name='rexport', diff --git a/my/reddit/all.py b/my/reddit/all.py index 27e22df..a668081 100644 --- a/my/reddit/all.py +++ b/my/reddit/all.py @@ -1,9 +1,8 @@ -from collections.abc import Iterator - -from my.core import Stats, stat +from typing import Iterator +from my.core.common import Stats from my.core.source import import_source -from .common import Comment, Save, Submission, Upvote, _merge_comments +from .common import Save, Upvote, Comment, Submission, _merge_comments # Man... ideally an all.py file isn't this verbose, but # reddit just feels like that much of a complicated source and @@ -59,6 +58,7 @@ def upvoted() -> Iterator[Upvote]: yield from upvoted() def stats() -> Stats: + from my.core import stat return { **stat(saved), **stat(comments), diff --git a/my/reddit/common.py b/my/reddit/common.py index 40f9f6e..c01258b 100644 --- a/my/reddit/common.py +++ b/my/reddit/common.py @@ -2,14 +2,12 @@ This defines Protocol classes, which make sure that each different type of shared models have a standardized interface """ +from my.core import __NOT_HPI_MODULE__ -from my.core import __NOT_HPI_MODULE__ # isort: skip - -from collections.abc import Iterator +from typing import Set, Iterator, Protocol from itertools import chain -from typing import Protocol -from my.core import Json, datetime_aware +from my.core import datetime_aware, Json # common fields across all the Protocol classes, so generic code can be written @@ -51,7 +49,7 @@ class Submission(RedditBase, Protocol): def _merge_comments(*sources: Iterator[Comment]) -> Iterator[Comment]: #from .rexport import logger #ignored = 0 - emitted: set[str] = set() + emitted: Set[str] = set() for e in chain(*sources): uid = e.id if uid in emitted: diff --git a/my/reddit/pushshift.py b/my/reddit/pushshift.py index 12f592b..1c7ec8d 100644 --- a/my/reddit/pushshift.py +++ b/my/reddit/pushshift.py @@ -1,27 +1,26 @@ """ Gives you access to older comments possibly not accessible with rexport using pushshift -See https://github.com/purarue/pushshift_comment_export +See https://github.com/seanbreckenridge/pushshift_comment_export """ REQUIRES = [ - "git+https://github.com/purarue/pushshift_comment_export", + "git+https://github.com/seanbreckenridge/pushshift_comment_export", ] +from my.core.common import Paths, Stats from dataclasses import dataclass - -# note: keeping pushshift import before config import, so it's handled gracefully by import_source -from pushshift_comment_export.dal import PComment, read_file - -from my.config import reddit as uconfig -from my.core import Paths, Stats, stat from my.core.cfg import make_config +# note: keeping pushshift import before config import, so it's handled gracefully by import_source +from pushshift_comment_export.dal import read_file, PComment + +from my.config import reddit as uconfig @dataclass class pushshift_config(uconfig.pushshift): ''' - Uses [[https://github.com/purarue/pushshift_comment_export][pushshift]] to get access to old comments + Uses [[https://github.com/seanbreckenridge/pushshift_comment_export][pushshift]] to get access to old comments ''' # path[s]/glob to the exported JSON data @@ -29,10 +28,10 @@ class pushshift_config(uconfig.pushshift): config = make_config(pushshift_config) -from collections.abc import Iterator, Sequence +from my.core import get_files +from typing import Sequence, Iterator from pathlib import Path -from my.core import get_files def inputs() -> Sequence[Path]: @@ -44,6 +43,7 @@ def comments() -> Iterator[PComment]: yield from read_file(f) def stats() -> Stats: + from my.core import stat return { **stat(comments) } diff --git a/my/reddit/rexport.py b/my/reddit/rexport.py index 262635b..a7be39b 100644 --- a/my/reddit/rexport.py +++ b/my/reddit/rexport.py @@ -7,24 +7,23 @@ REQUIRES = [ 'git+https://github.com/karlicoss/rexport', ] -import inspect -from collections.abc import Iterator, Sequence from dataclasses import dataclass +import inspect from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Iterator, Sequence from my.core import ( - Paths, - Stats, get_files, make_logger, - stat, warnings, + stat, + Paths, + Stats, ) -from my.core.cachew import mcachew -from my.core.cfg import Attrs, make_config +from my.core.cfg import make_config, Attrs +from my.core.common import mcachew -from my.config import reddit as uconfig # isort: skip +from my.config import reddit as uconfig logger = make_logger(__name__) @@ -145,9 +144,9 @@ if not TYPE_CHECKING: try: # here we just check that types are available, we don't actually want to import them # fmt: off - dal.Subreddit # noqa: B018 - dal.Profile # noqa: B018 - dal.Multireddit # noqa: B018 + dal.Subreddit + dal.Profile + dal.Multireddit # fmt: on except AttributeError as ae: warnings.high(f'{ae} : please update "rexport" installation') diff --git a/my/rescuetime.py b/my/rescuetime.py index 0c9fd28..75684d9 100644 --- a/my/rescuetime.py +++ b/my/rescuetime.py @@ -5,15 +5,16 @@ REQUIRES = [ 'git+https://github.com/karlicoss/rescuexport', ] -from collections.abc import Iterable, Sequence -from datetime import timedelta from pathlib import Path +from datetime import timedelta +from typing import Sequence, Iterable -from my.core import Stats, get_files, make_logger, stat -from my.core.cachew import mcachew +from my.core import get_files, make_logger +from my.core.common import mcachew from my.core.error import Res, split_errors -from my.config import rescuetime as config # isort: skip +from my.config import rescuetime as config + logger = make_logger(__name__) @@ -23,7 +24,6 @@ def inputs() -> Sequence[Path]: import rescuexport.dal as dal - DAL = dal.DAL Entry = dal.Entry @@ -43,12 +43,11 @@ def groups(gap: timedelta=timedelta(hours=3)) -> Iterable[Res[Sequence[Entry]]]: # todo automatic dataframe interface? from .core.pandas import DataFrameT, as_dataframe - - def dataframe() -> DataFrameT: return as_dataframe(entries()) +from .core import stat, Stats def stats() -> Stats: return { **stat(groups), @@ -58,19 +57,16 @@ def stats() -> Stats: # basically, hack config and populate it with fake data? fake data generated by DAL, but the rest is handled by this? -from collections.abc import Iterator from contextlib import contextmanager - - +from typing import Iterator # todo take seed, or what? @contextmanager def fake_data(rows: int=1000) -> Iterator: # todo also disable cachew automatically for such things? - import json - from tempfile import TemporaryDirectory - - from my.core.cachew import disabled_cachew from my.core.cfg import tmp_config + from my.core.cachew import disabled_cachew + from tempfile import TemporaryDirectory + import json with disabled_cachew(), TemporaryDirectory() as td: tdir = Path(td) f = tdir / 'rescuetime.json' @@ -87,13 +83,12 @@ def fake_data(rows: int=1000) -> Iterator: def fill_influxdb() -> None: - from my.core import influxdb - - it = ({ - 'dt': e.dt, - 'duration_d': e.duration_s, - 'tags': {'activity': e.activity}, - } for e in entries() if isinstance(e, Entry)) # TODO handle errors in core.influxdb + from .core import influxdb + it = (dict( + dt=e.dt, + duration_d=e.duration_s, + tags=dict(activity=e.activity), + ) for e in entries() if isinstance(e, Entry)) # TODO handle errors in core.influxdb influxdb.fill(it, measurement=__name__) diff --git a/my/roamresearch.py b/my/roamresearch.py index 7322774..2fe06d4 100644 --- a/my/roamresearch.py +++ b/my/roamresearch.py @@ -1,19 +1,16 @@ """ [[https://roamresearch.com][Roam]] data """ -from __future__ import annotations - -import re -from collections.abc import Iterator from datetime import datetime, timezone -from itertools import chain from pathlib import Path -from typing import NamedTuple +from itertools import chain +import re +from typing import NamedTuple, Iterator, List, Optional + +from .core import get_files, LazyLogger, Json from my.config import roamresearch as config -from .core import Json, LazyLogger, get_files - logger = LazyLogger(__name__) @@ -60,15 +57,15 @@ class Node(NamedTuple): return datetime.fromtimestamp(rt / 1000, tz=timezone.utc) @property - def title(self) -> str | None: + def title(self) -> Optional[str]: return self.raw.get(Keys.TITLE) @property - def body(self) -> str | None: + def body(self) -> Optional[str]: return self.raw.get(Keys.STRING) @property - def children(self) -> list[Node]: + def children(self) -> List['Node']: # TODO cache? needs a key argument (because of Json) ch = self.raw.get(Keys.CHILDREN, []) return list(map(Node, ch)) @@ -98,7 +95,7 @@ class Node(NamedTuple): # - heading -- notes that haven't been created yet return len(self.body or '') == 0 and len(self.children) == 0 - def traverse(self) -> Iterator[Node]: + def traverse(self) -> Iterator['Node']: # not sure about __iter__, because might be a bit unintuitive that it's recursive.. yield self for c in self.children: @@ -123,7 +120,7 @@ class Node(NamedTuple): return f'Node(created={self.created}, title={self.title}, body={self.body})' @staticmethod - def make(raw: Json) -> Iterator[Node]: + def make(raw: Json) -> Iterator['Node']: is_empty = set(raw.keys()) == {Keys.EDITED, Keys.EDIT_EMAIL, Keys.TITLE} # not sure about that... but daily notes end up like that if is_empty: @@ -133,11 +130,11 @@ class Node(NamedTuple): class Roam: - def __init__(self, raw: list[Json]) -> None: + def __init__(self, raw: List[Json]) -> None: self.raw = raw @property - def notes(self) -> list[Node]: + def notes(self) -> List[Node]: return list(chain.from_iterable(map(Node.make, self.raw))) def traverse(self) -> Iterator[Node]: diff --git a/my/rss/all.py b/my/rss/all.py index e10e4d2..61f9fab 100644 --- a/my/rss/all.py +++ b/my/rss/all.py @@ -1,11 +1,10 @@ ''' Unified RSS data, merged from different services I used historically ''' - # NOTE: you can comment out the sources you're not using -from collections.abc import Iterable - from . import feedbin, feedly + +from typing import Iterable from .common import Subscription, compute_subscriptions @@ -13,5 +12,5 @@ def subscriptions() -> Iterable[Subscription]: # TODO google reader? yield from compute_subscriptions( feedbin.states(), - feedly.states(), + feedly .states(), ) diff --git a/my/rss/common.py b/my/rss/common.py index bf9506e..f3893b7 100644 --- a/my/rss/common.py +++ b/my/rss/common.py @@ -1,40 +1,36 @@ -from __future__ import annotations - -from my.core import __NOT_HPI_MODULE__ # isort: skip - -from collections.abc import Iterable, Sequence -from dataclasses import dataclass, replace -from itertools import chain - -from my.core import datetime_aware, warn_if_empty +# shared Rss stuff +from datetime import datetime +from typing import NamedTuple, Optional, List, Dict -@dataclass -class Subscription: +class Subscription(NamedTuple): title: str url: str - id: str # TODO not sure about it... + id: str # TODO not sure about it... # eh, not all of them got reasonable 'created' time - created_at: datetime_aware | None - subscribed: bool = True + created_at: Optional[datetime] + subscribed: bool=True +from typing import Iterable, Tuple, Sequence # snapshot of subscriptions at time -SubscriptionState = tuple[datetime_aware, Sequence[Subscription]] +SubscriptionState = Tuple[datetime, Sequence[Subscription]] +from ..core import warn_if_empty @warn_if_empty -def compute_subscriptions(*sources: Iterable[SubscriptionState]) -> list[Subscription]: +def compute_subscriptions(*sources: Iterable[SubscriptionState]) -> List[Subscription]: """ Keeps track of everything I ever subscribed to. In addition, keeps track of unsubscribed as well (so you'd remember when and why you unsubscribed) """ + from itertools import chain states = list(chain.from_iterable(sources)) # TODO keep 'source'/'provider'/'service' attribute? - by_url: dict[str, Subscription] = {} + by_url: Dict[str, Subscription] = {} # ah. dates are used for sorting - for _when, state in sorted(states): + for when, state in sorted(states): # TODO use 'when'? for feed in state: if feed.url not in by_url: @@ -49,5 +45,7 @@ def compute_subscriptions(*sources: Iterable[SubscriptionState]) -> list[Subscri res = [] for u, x in sorted(by_url.items()): present = u in last_urls - res.append(replace(x, subscribed=present)) + res.append(x._replace(subscribed=present)) return res + +from ..core import __NOT_HPI_MODULE__ diff --git a/my/rss/feedbin.py b/my/rss/feedbin.py index 5f4da0a..8ba25b8 100644 --- a/my/rss/feedbin.py +++ b/my/rss/feedbin.py @@ -2,40 +2,48 @@ Feedbin RSS reader """ -import json -from collections.abc import Iterator, Sequence +from my.config import feedbin as config + from pathlib import Path +from typing import Sequence -from my.core import Stats, get_files, stat -from my.core.compat import fromisoformat +from ..core.common import listify, get_files, isoparse +from .common import Subscription -from .common import Subscription, SubscriptionState - -from my.config import feedbin as config # isort: skip def inputs() -> Sequence[Path]: return get_files(config.export_path) -def parse_file(f: Path) -> Iterator[Subscription]: +import json + +@listify +def parse_file(f: Path): raw = json.loads(f.read_text()) for r in raw: yield Subscription( - created_at=fromisoformat(r['created_at']), + created_at=isoparse(r['created_at']), title=r['title'], url=r['site_url'], id=r['id'], ) -def states() -> Iterator[SubscriptionState]: +from typing import Iterable +from .common import SubscriptionState +def states() -> Iterable[SubscriptionState]: + # meh + from dateutil.parser import isoparse for f in inputs(): # TODO ugh. depends on my naming. not sure if useful? dts = f.stem.split('_')[-1] - dt = fromisoformat(dts) - subs = list(parse_file(f)) + dt = isoparse(dts) + subs = parse_file(f) yield dt, subs -def stats() -> Stats: - return stat(states) +def stats(): + from more_itertools import ilen, last + return { + 'subscriptions': ilen(last(states())[1]) + } diff --git a/my/rss/feedly.py b/my/rss/feedly.py index 9bf5429..4611ced 100644 --- a/my/rss/feedly.py +++ b/my/rss/feedly.py @@ -1,45 +1,28 @@ """ Feedly RSS reader """ +from my.config import feedly as config -import json -from abc import abstractmethod -from collections.abc import Iterator, Sequence from datetime import datetime, timezone +import json from pathlib import Path -from typing import Protocol - -from my.core import Paths, get_files +from typing import Iterable, Sequence +from ..core.common import listify, get_files from .common import Subscription, SubscriptionState -class config(Protocol): - @property - @abstractmethod - def export_path(self) -> Paths: - raise NotImplementedError - - -def make_config() -> config: - from my.config import feedly as user_config - - class combined_config(user_config, config): ... - - return combined_config() - - def inputs() -> Sequence[Path]: - cfg = make_config() - return get_files(cfg.export_path) + return get_files(config.export_path) -def parse_file(f: Path) -> Iterator[Subscription]: +@listify +def parse_file(f: Path): raw = json.loads(f.read_text()) for r in raw: # err, some even don't have website.. rid = r['id'] - website = r.get('website', rid) # meh + website = r.get('website', rid) # meh yield Subscription( created_at=None, title=r['title'], @@ -48,9 +31,9 @@ def parse_file(f: Path) -> Iterator[Subscription]: ) -def states() -> Iterator[SubscriptionState]: +def states() -> Iterable[SubscriptionState]: for f in inputs(): dts = f.stem.split('_')[-1] dt = datetime.strptime(dts, '%Y%m%d%H%M%S').replace(tzinfo=timezone.utc) - subs = list(parse_file(f)) + subs = parse_file(f) yield dt, subs diff --git a/my/rtm.py b/my/rtm.py index 217c969..8d41e7a 100644 --- a/my/rtm.py +++ b/my/rtm.py @@ -6,21 +6,21 @@ REQUIRES = [ 'icalendar', ] -import re -from collections.abc import Iterator from datetime import datetime from functools import cached_property +import re +from typing import Dict, List, Iterator -import icalendar # type: ignore -from icalendar.cal import Todo # type: ignore -from more_itertools import bucket +from .core.common import LazyLogger, get_files, group_by_key, make_dict -from my.core import get_files, make_logger -from my.core.utils.itertools import make_dict +from my.config import rtm as config -from my.config import rtm as config # isort: skip -logger = make_logger(__name__) +import icalendar # type: ignore +from icalendar.cal import Todo # type: ignore + + +logger = LazyLogger(__name__) # TODO extract in a module to parse RTM's ical? @@ -30,14 +30,14 @@ class MyTodo: self.revision = revision @cached_property - def notes(self) -> list[str]: + def notes(self) -> List[str]: # TODO can there be multiple?? desc = self.todo['DESCRIPTION'] notes = re.findall(r'---\n\n(.*?)\n\nUpdated:', desc, flags=re.DOTALL) return notes @cached_property - def tags(self) -> list[str]: + def tags(self) -> List[str]: desc = self.todo['DESCRIPTION'] [tags_str] = re.findall(r'\nTags: (.*?)\n', desc, flags=re.DOTALL) if tags_str == 'none': @@ -56,7 +56,7 @@ class MyTodo: def get_status(self) -> str: if 'STATUS' not in self.todo: return None # type: ignore - # TODO 'COMPLETED'? + # TODO 'COMPLETED'? return str(self.todo['STATUS']) # TODO tz? @@ -90,14 +90,13 @@ class DAL: for t in self.cal.walk('VTODO'): yield MyTodo(t, self.revision) - def get_todos_by_uid(self) -> dict[str, MyTodo]: + def get_todos_by_uid(self) -> Dict[str, MyTodo]: todos = self.all_todos() return make_dict(todos, key=lambda t: t.uid) - def get_todos_by_title(self) -> dict[str, list[MyTodo]]: + def get_todos_by_title(self) -> Dict[str, List[MyTodo]]: todos = self.all_todos() - bucketed = bucket(todos, lambda todo: todo.title) - return {k: list(bucketed[k]) for k in bucketed} + return group_by_key(todos, lambda todo: todo.title) def dal(): diff --git a/my/runnerup.py b/my/runnerup.py index f5d7d1e..f12d9b3 100644 --- a/my/runnerup.py +++ b/my/runnerup.py @@ -6,15 +6,17 @@ REQUIRES = [ 'python-tcxparser', ] -from collections.abc import Iterable from datetime import timedelta from pathlib import Path +from typing import Iterable + +from .core import Res, get_files +from .core.common import isoparse, Json import tcxparser # type: ignore[import-untyped] from my.config import runnerup as config -from my.core import Json, Res, get_files -from my.core.compat import fromisoformat + # TODO later, use a proper namedtuple? Workout = Json @@ -42,7 +44,7 @@ def _parse(f: Path) -> Workout: return { 'id' : f.name, # not sure? - 'start_time' : fromisoformat(tcx.started_at), + 'start_time' : isoparse(tcx.started_at), 'duration' : timedelta(seconds=tcx.duration), 'sport' : sport, 'heart_rate_avg': tcx.hr_avg, @@ -56,7 +58,7 @@ def _parse(f: Path) -> Workout: # tcx.hr_values(), # # todo cadence? # ): - # t = fromisoformat(ts) + # t = isoparse(ts) def workouts() -> Iterable[Res[Workout]]: @@ -68,8 +70,6 @@ def workouts() -> Iterable[Res[Workout]]: from .core.pandas import DataFrameT, check_dataframe, error_to_row - - @check_dataframe def dataframe() -> DataFrameT: def it(): @@ -85,8 +85,6 @@ def dataframe() -> DataFrameT: return df -from .core import Stats, stat - - +from .core import stat, Stats def stats() -> Stats: return stat(dataframe) diff --git a/my/simple.py b/my/simple.py index b7f25cd..7462291 100644 --- a/my/simple.py +++ b/my/simple.py @@ -1,11 +1,12 @@ ''' Just a demo module for testing and documentation purposes ''' -from collections.abc import Iterator from dataclasses import dataclass +from typing import Iterator + +from my.core import make_config from my.config import simple as user_config -from my.core import make_config @dataclass diff --git a/my/smscalls.py b/my/smscalls.py index 27d08be..dbcf8b2 100644 --- a/my/smscalls.py +++ b/my/smscalls.py @@ -2,34 +2,29 @@ Phone calls and SMS messages Exported using https://play.google.com/store/apps/details?id=com.riteshsahu.SMSBackupRestore&hl=en_US """ -from __future__ import annotations # See: https://www.synctech.com.au/sms-backup-restore/fields-in-xml-backup-files/ for schema REQUIRES = ['lxml'] -from dataclasses import dataclass - +from .core import Paths, dataclass from my.config import smscalls as user_config -from my.core import Paths, Stats, get_files, stat - @dataclass class smscalls(user_config): # path[s] that SMSBackupRestore syncs XML files to export_path: Paths -from my.core.cfg import make_config - +from .core.cfg import make_config config = make_config(smscalls) -from collections.abc import Iterator from datetime import datetime, timezone from pathlib import Path -from typing import Any, NamedTuple +from typing import NamedTuple, Iterator, Set, Tuple, Optional -import lxml.etree as etree +from lxml import etree +from my.core.common import get_files, Stats from my.core.error import Res @@ -37,8 +32,7 @@ class Call(NamedTuple): dt: datetime dt_readable: str duration_s: int - phone_number: str - who: str | None + who: Optional[str] # type - 1 = Incoming, 2 = Outgoing, 3 = Missed, 4 = Voicemail, 5 = Rejected, 6 = Refused List. call_type: int @@ -55,27 +49,23 @@ class Call(NamedTuple): # All the field values are read as-is from the underlying database and no conversion is done by the app in most cases. # # The '(Unknown)' is just what my android phone does, not sure if there are others -UNKNOWN: set[str] = {'(Unknown)'} - -def _parse_xml(xml: Path) -> Any: - return etree.parse(str(xml), parser=etree.XMLParser(huge_tree=True)) +UNKNOWN: Set[str] = {'(Unknown)'} def _extract_calls(path: Path) -> Iterator[Res[Call]]: - tr = _parse_xml(path) + tr = etree.parse(str(path)) for cxml in tr.findall('call'): dt = cxml.get('date') dt_readable = cxml.get('readable_date') duration = cxml.get('duration') who = cxml.get('contact_name') call_type = cxml.get('type') - number = cxml.get('number') # if name is missing, its not None (its some string), depends on the phone/message app if who is not None and who in UNKNOWN: who = None - if dt is None or dt_readable is None or duration is None or call_type is None or number is None: + if dt is None or dt_readable is None or duration is None or call_type is None: call_str = etree.tostring(cxml).decode('utf-8') - yield RuntimeError(f"Missing one or more required attributes [date, readable_date, duration, type, number] in {call_str}") + yield RuntimeError(f"Missing one or more required attributes [date, readable_date, duration, type] in {call_str}") continue # TODO we've got local tz here, not sure if useful.. # ok, so readable date is local datetime, changing throughout the backup @@ -83,7 +73,6 @@ def _extract_calls(path: Path) -> Iterator[Res[Call]]: dt=_parse_dt_ms(dt), dt_readable=dt_readable, duration_s=int(duration), - phone_number=number, who=who, call_type=int(call_type), ) @@ -93,7 +82,7 @@ def calls() -> Iterator[Res[Call]]: files = get_files(config.export_path, glob='calls-*.xml') # TODO always replacing with the latter is good, we get better contact names?? - emitted: set[datetime] = set() + emitted: Set[datetime] = set() for p in files: for c in _extract_calls(p): if isinstance(c, Exception): @@ -108,7 +97,7 @@ def calls() -> Iterator[Res[Call]]: class Message(NamedTuple): dt: datetime dt_readable: str - who: str | None + who: Optional[str] message: str phone_number: str # type - 1 = Received, 2 = Sent, 3 = Draft, 4 = Outbox, 5 = Failed, 6 = Queued @@ -122,7 +111,7 @@ class Message(NamedTuple): def messages() -> Iterator[Res[Message]]: files = get_files(config.export_path, glob='sms-*.xml') - emitted: set[tuple[datetime, str | None, bool]] = set() + emitted: Set[Tuple[datetime, Optional[str], bool]] = set() for p in files: for c in _extract_messages(p): if isinstance(c, Exception): @@ -136,7 +125,7 @@ def messages() -> Iterator[Res[Message]]: def _extract_messages(path: Path) -> Iterator[Res[Message]]: - tr = _parse_xml(path) + tr = etree.parse(str(path)) for mxml in tr.findall('sms'): dt = mxml.get('date') dt_readable = mxml.get('readable_date') @@ -161,160 +150,6 @@ def _extract_messages(path: Path) -> Iterator[Res[Message]]: ) -class MMSContentPart(NamedTuple): - sequence_index: int - content_type: str - filename: str - text: str | None - data: str | None - - -class MMS(NamedTuple): - dt: datetime - dt_readable: str - parts: list[MMSContentPart] - # NOTE: these is often something like 'Name 1, Name 2', but might be different depending on your client - who: str | None - # NOTE: This can be a single phone number, or multiple, split by '~' or ','. Its better to think - # of this as a 'key' or 'conversation ID', phone numbers are also present in 'addresses' - phone_number: str - addresses: list[tuple[str, int]] - # 1 = Received, 2 = Sent, 3 = Draft, 4 = Outbox - message_type: int - - @property - def from_user(self) -> str: - # since these can be group messages, we can't just check message_type, - # we need to iterate through and find who sent it - # who is CC/'To' is not obvious in many message clients - # - # 129 = BCC, 130 = CC, 151 = To, 137 = From - for (addr, _type) in self.addresses: - if _type == 137: - return addr - # hmm, maybe return instead? but this probably shouldn't happen, means - # something is very broken - raise RuntimeError(f'No from address matching 137 found in {self.addresses}') - - @property - def from_me(self) -> bool: - return self.message_type == 2 - - -def mms() -> Iterator[Res[MMS]]: - files = get_files(config.export_path, glob='sms-*.xml') - - emitted: set[tuple[datetime, str | None, str]] = set() - for p in files: - for c in _extract_mms(p): - if isinstance(c, Exception): - yield c - continue - key = (c.dt, c.phone_number, c.from_user) - if key in emitted: - continue - emitted.add(key) - yield c - - -def _resolve_null_str(value: str | None) -> str | None: - if value is None: - return None - # hmm.. there's some risk of the text actually being 'null', but there's - # no way to distinguish that from XML values - if value == 'null': - return None - return value - - -def _extract_mms(path: Path) -> Iterator[Res[MMS]]: - tr = _parse_xml(path) - for mxml in tr.findall('mms'): - dt = mxml.get('date') - dt_readable = mxml.get('readable_date') - message_type = mxml.get('msg_box') - - who = mxml.get('contact_name') - if who is not None and who in UNKNOWN: - who = None - phone_number = mxml.get('address') - - if dt is None or dt_readable is None or message_type is None or phone_number is None: - mxml_str = etree.tostring(mxml).decode('utf-8') - yield RuntimeError(f'Missing one or more required attributes [date, readable_date, msg_box, address] in {mxml_str}') - continue - - addresses: list[tuple[str, int]] = [] - for addr_parent in mxml.findall('addrs'): - for addr in addr_parent.findall('addr'): - addr_data = addr.attrib - user_address = addr_data.get('address') - user_type = addr_data.get('type') - if user_address is None or user_type is None: - addr_str = etree.tostring(addr_parent).decode() - yield RuntimeError(f'Missing one or more required attributes [address, type] in {addr_str}') - continue - if not user_type.isdigit(): - yield RuntimeError(f'Invalid address type {user_type} {type(user_type)}, cannot convert to number') - continue - addresses.append((user_address, int(user_type))) - - content: list[MMSContentPart] = [] - - for part_root in mxml.findall('parts'): - - for part in part_root.findall('part'): - - # the first item is an SMIL XML element encoded as a string which describes - # how the rest of the parts are laid out - # https://www.w3.org/TR/SMIL3/smil-timing.html#Timing-TimeContainerSyntax - # An example: - # - # - # This seems pretty useless, so we should try and skip it, and just return the - # text/images/data - part_data: dict[str, Any] = part.attrib - seq: str | None = part_data.get('seq') - if seq == '-1': - continue - - if seq is None or not seq.isdigit(): - yield RuntimeError(f'seq must be a number, was seq={seq} {type(seq)} in {part_data}') - continue - - charset_type: str | None = _resolve_null_str(part_data.get('ct')) - filename: str | None = _resolve_null_str(part_data.get('name')) - # in some cases (images, cards), the filename is set in 'cl' instead - if filename is None: - filename = _resolve_null_str(part_data.get('cl')) - text: str | None = _resolve_null_str(part_data.get('text')) - data: str | None = _resolve_null_str(part_data.get('data')) - - if charset_type is None or filename is None or (text is None and data is None): - yield RuntimeError(f'Missing one or more required attributes [ct, name, (text, data)] must be present in {part_data}') - continue - - content.append( - MMSContentPart( - sequence_index=int(seq), - content_type=charset_type, - filename=filename, - text=text, - data=data - ) - ) - - yield MMS( - dt=_parse_dt_ms(dt), - dt_readable=dt_readable, - who=who, - phone_number=phone_number, - message_type=int(message_type), - parts=content, - addresses=addresses, - ) - - # See https://github.com/karlicoss/HPI/pull/90#issuecomment-702422351 # for potentially parsing timezone from the readable_date def _parse_dt_ms(d: str) -> datetime: @@ -322,8 +157,9 @@ def _parse_dt_ms(d: str) -> datetime: def stats() -> Stats: + from .core import stat + return { **stat(calls), **stat(messages), - **stat(mms), } diff --git a/my/stackexchange/gdpr.py b/my/stackexchange/gdpr.py index 8ed0d30..18b2b4d 100644 --- a/my/stackexchange/gdpr.py +++ b/my/stackexchange/gdpr.py @@ -5,12 +5,8 @@ Stackexchange data (uses [[https://stackoverflow.com/legal/gdpr/request][officia # TODO need to merge gdpr and stexport ### config -from dataclasses import dataclass - from my.config import stackexchange as user_config -from my.core import Json, PathIsh, get_files, make_config - - +from ..core import dataclass, PathIsh, make_config, get_files @dataclass class stackexchange(user_config): gdpr_path: PathIsh # path to GDPR zip file @@ -20,20 +16,16 @@ config = make_config(stackexchange) # TODO just merge all of them and then filter?.. not sure -from collections.abc import Iterable +from ..core.common import Json, isoparse +from typing import NamedTuple, Iterable from datetime import datetime -from typing import NamedTuple - -from my.core.compat import fromisoformat - - class Vote(NamedTuple): j: Json # todo ip? @property def when(self) -> datetime: - return fromisoformat(self.j['eventTime']) + return isoparse(self.j['eventTime']) # todo Url return type? @property @@ -49,7 +41,7 @@ class Vote(NamedTuple): # hmm, this loads very raw comments without the rest of the page? # - https://meta.stackexchange.com/posts/27319/comments#comment-57475 # - # parentPostId is the original question + # parentPostId is the original quesion # TODO is not always present? fucking hell # seems like there is no way to get a hierarchical comment link.. guess this needs to be handled in Promnesia normalisation... # postId is the answer @@ -69,10 +61,7 @@ class Vote(NamedTuple): # todo expose vote type? import json - from ..core.error import Res - - def votes() -> Iterable[Res[Vote]]: # TODO there is also some site specific stuff in qa/ directory.. not sure if its' more detailed # todo should be defensive? not sure if present when user has no votes @@ -84,8 +73,6 @@ def votes() -> Iterable[Res[Vote]]: yield Vote(r) -from ..core import Stats, stat - - +from ..core import stat, Stats def stats() -> Stats: return stat(votes) diff --git a/my/stackexchange/stexport.py b/my/stackexchange/stexport.py index 111ed28..812a155 100644 --- a/my/stackexchange/stexport.py +++ b/my/stackexchange/stexport.py @@ -16,8 +16,7 @@ from my.core import ( make_config, stat, ) - -import my.config # isort: skip +import my.config @dataclass diff --git a/my/taplog.py b/my/taplog.py index 5e64a72..51eeb72 100644 --- a/my/taplog.py +++ b/my/taplog.py @@ -1,26 +1,24 @@ ''' [[https://play.google.com/store/apps/details?id=com.waterbear.taglog][Taplog]] app data ''' -from __future__ import annotations - -from collections.abc import Iterable from datetime import datetime -from typing import NamedTuple +from typing import NamedTuple, Dict, Optional, Iterable + +from my.core import get_files, stat, Stats +from my.core.sqlite import sqlite_connection from my.config import taplog as user_config -from my.core import Stats, get_files, stat -from my.core.sqlite import sqlite_connection class Entry(NamedTuple): - row: dict + row: Dict @property def id(self) -> str: return str(self.row['_id']) @property - def number(self) -> float | None: + def number(self) -> Optional[float]: ns = self.row['number'] # TODO ?? if isinstance(ns, str): diff --git a/my/telegram/telegram_backup.py b/my/telegram/telegram_backup.py index eea7e50..0617501 100644 --- a/my/telegram/telegram_backup.py +++ b/my/telegram/telegram_backup.py @@ -1,39 +1,39 @@ """ Telegram data via [fabianonline/telegram_backup](https://github.com/fabianonline/telegram_backup) tool """ -from __future__ import annotations -import sqlite3 -from collections.abc import Iterator from dataclasses import dataclass from datetime import datetime, timezone -from struct import calcsize, unpack_from +from struct import unpack_from, calcsize +import sqlite3 +from typing import Dict, Iterator, Optional + +from my.core import datetime_aware, PathIsh +from my.core.sqlite import sqlite_connection from my.config import telegram as user_config -from my.core import PathIsh, datetime_aware -from my.core.sqlite import sqlite_connection @dataclass class config(user_config.telegram_backup): # path to the export database.sqlite export_path: PathIsh - + @dataclass class Chat: id: str - name: str | None + name: Optional[str] # not all users have short handle + groups don't have them either? # TODO hmm some groups have it -- it's just the tool doesn't dump them?? - handle: str | None + handle: Optional[str] # not sure if need type? @dataclass class User: id: str - name: str | None + name: Optional[str] @dataclass @@ -44,7 +44,7 @@ class Message: chat: Chat sender: User text: str - extra_media_info: str | None = None + extra_media_info: Optional[str] = None @property def permalink(self) -> str: @@ -61,7 +61,7 @@ class Message: -Chats = dict[str, Chat] +Chats = Dict[str, Chat] def _message_from_row(r: sqlite3.Row, *, chats: Chats, with_extra_media_info: bool) -> Message: ts = r['time'] # desktop export uses UTC (checked by exporting in winter time vs summer time) @@ -70,7 +70,7 @@ def _message_from_row(r: sqlite3.Row, *, chats: Chats, with_extra_media_info: bo chat = chats[r['source_id']] sender = chats[r['sender_id']] - extra_media_info: str | None = None + extra_media_info: Optional[str] = None if with_extra_media_info and r['has_media'] == 1: # also it's quite hacky, so at least for now it's just an optional attribute behind the flag # defensive because it's a bit tricky to correctly parse without a proper api parser.. @@ -90,7 +90,7 @@ def _message_from_row(r: sqlite3.Row, *, chats: Chats, with_extra_media_info: bo ) -def messages(*, extra_where: str | None=None, with_extra_media_info: bool=False) -> Iterator[Message]: +def messages(*, extra_where: Optional[str]=None, with_extra_media_info: bool=False) -> Iterator[Message]: messages_query = 'SELECT * FROM messages WHERE message_type NOT IN ("service_message", "empty_message")' if extra_where is not None: messages_query += ' AND ' + extra_where @@ -106,7 +106,7 @@ def messages(*, extra_where: str | None=None, with_extra_media_info: bool=False) for r in db.execute('SELECT * FROM users ORDER BY id'): first = r["first_name"] last = r["last_name"] - name: str | None + name: Optional[str] if first is not None and last is not None: name = f'{first} {last}' else: @@ -121,7 +121,7 @@ def messages(*, extra_where: str | None=None, with_extra_media_info: bool=False) yield _message_from_row(r, chats=chats, with_extra_media_info=with_extra_media_info) -def _extract_extra_media_info(data: bytes) -> str | None: +def _extract_extra_media_info(data: bytes) -> Optional[str]: # ugh... very hacky, but it does manage to extract from 90% of messages that have media pos = 0 diff --git a/my/tests/body/weight.py b/my/tests/body/weight.py deleted file mode 100644 index f26ccf2..0000000 --- a/my/tests/body/weight.py +++ /dev/null @@ -1,59 +0,0 @@ -from pathlib import Path - -import pytest -import pytz - -from my.body.weight import from_orgmode -from my.core.cfg import tmp_config - - -def test_body_weight() -> None: - weights = [0.0 if isinstance(x, Exception) else x.value for x in from_orgmode()] - - assert weights == [ - 0.0, - 62.0, - 0.0, - 61.0, - 62.0, - 0.0, - ] - - -@pytest.fixture(autouse=True) -def prepare(tmp_path: Path): - ndir = tmp_path / 'notes' - ndir.mkdir() - logs = ndir / 'logs.org' - logs.write_text( - ''' -#+TITLE: Stuff I'm logging - -* Weight (org-capture) :weight: -** [2020-05-01 Fri 09:00] 62 -** 63 - this should be ignored, got no timestamp -** [2020-05-03 Sun 08:00] 61 -** [2020-05-04 Mon 10:00] 62 -''' - ) - misc = ndir / 'misc.org' - misc.write_text( - ''' -Some misc stuff - -* unrelated note :weight:whatever: -''' - ) - - class orgmode: - paths = [ndir] - - class weight: - # TODO ugh. this belongs to tz provider or global config or something - default_timezone = pytz.timezone('Europe/London') - - with tmp_config() as cfg: - cfg.orgmode = orgmode - cfg.weight = weight - yield diff --git a/my/tests/calendar.py b/my/tests/calendar.py deleted file mode 100644 index b5f856c..0000000 --- a/my/tests/calendar.py +++ /dev/null @@ -1,9 +0,0 @@ -from my.calendar.holidays import is_holiday - -from .shared_tz_config import config # autoused fixture - - -def test_is_holiday() -> None: - assert is_holiday('20190101') - assert not is_holiday('20180601') - assert is_holiday('20200906') # national holiday in Bulgaria diff --git a/my/tests/commits.py b/my/tests/commits.py index 48e349f..c967027 100644 --- a/my/tests/commits.py +++ b/my/tests/commits.py @@ -1,11 +1,14 @@ import os from pathlib import Path -import pytest from more_itertools import bucket +import pytest + + +from my.core.cfg import tmp_config from my.coding.commits import commits -from my.core.cfg import tmp_config + pytestmark = pytest.mark.skipif( os.name == 'nt', diff --git a/my/tests/common.py b/my/tests/common.py index cf5c632..c8d88ff 100644 --- a/my/tests/common.py +++ b/my/tests/common.py @@ -1,5 +1,7 @@ import os from pathlib import Path +import re +import sys import pytest @@ -7,15 +9,21 @@ V = 'HPI_TESTS_KARLICOSS' skip_if_not_karlicoss = pytest.mark.skipif( V not in os.environ, - reason=f'test only works on @karlicoss data for now. Set env variable {V}=true to override.', + reason=f'test only works on @karlicoss data for now. Set evn variable {V}=true to override.', ) +def reset_modules() -> None: + ''' + A hack to 'unload' HPI modules, otherwise some modules might cache the config + TODO: a bit crap, need a better way.. + ''' + to_unload = [m for m in sys.modules if re.match(r'my[.]?', m)] + for m in to_unload: + del sys.modules[m] + + def testdata() -> Path: d = Path(__file__).absolute().parent.parent.parent / 'testdata' assert d.exists(), d return d - - -# prevent pytest from treating this as test -testdata.__test__ = False # type: ignore[attr-defined] diff --git a/my/tests/conftest.py b/my/tests/conftest.py deleted file mode 100644 index cc7bb7e..0000000 --- a/my/tests/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -import pytest - - -# I guess makes sense by default -@pytest.fixture(autouse=True) -def without_cachew(): - from my.core.cachew import disabled_cachew - - with disabled_cachew(): - yield diff --git a/my/tests/location/google.py b/my/tests/location/google.py deleted file mode 100644 index 43b8646..0000000 --- a/my/tests/location/google.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Tests for LEGACY location provider - -Keeping for now for backwards compatibility -""" - -from pathlib import Path - -import pytest -from more_itertools import one - -from my.core.cfg import tmp_config -from my.location.google import locations - - -def test_google_locations() -> None: - locs = list(locations()) - assert len(locs) == 3810, len(locs) - - last = locs[-1] - assert last.dt.strftime('%Y%m%d %H:%M:%S') == '20170802 13:01:56' # should be utc - # todo approx - assert last.lat == 46.5515350 - assert last.lon == 16.4742742 - # todo check altitude - - -@pytest.fixture(autouse=True) -def prepare(tmp_path: Path): - - # TODO could just pick a part of shared config? not sure - _takeout_path = _prepare_takeouts_dir(tmp_path) - - class google: - takeout_path = _takeout_path - - with tmp_config() as config: - config.google = google - yield - - -def _prepare_takeouts_dir(tmp_path: Path) -> Path: - from ..common import testdata - - try: - track = one(testdata().rglob('italy-slovenia-2017-07-29.json')) - except ValueError as e: - raise RuntimeError('testdata not found, setup git submodules?') from e - - # todo ugh. unnecessary zipping, but at the moment takeout provider doesn't support plain dirs - import zipfile - - with zipfile.ZipFile(tmp_path / 'takeout.zip', 'w') as zf: - zf.writestr('Takeout/Location History/Location History.json', track.read_bytes()) - return tmp_path diff --git a/my/tests/reddit.py b/my/tests/reddit.py index 4ddccf8..4af95ae 100644 --- a/my/tests/reddit.py +++ b/my/tests/reddit.py @@ -1,14 +1,14 @@ +from my.core.cfg import tmp_config +from my.core.common import make_dict + +# todo ugh, it's discovered as a test??? +from .common import testdata + import pytest -from more_itertools import consume # deliberately use mixed style imports on the top level and inside the methods to test tmp_config stuff -# todo won't really be necessary once we migrate to lazy user config -import my.reddit.all as my_reddit_all import my.reddit.rexport as my_reddit_rexport -from my.core.cfg import tmp_config -from my.core.utils.itertools import ensure_unique - -from .common import testdata +import my.reddit.all as my_reddit_all def test_basic_1() -> None: @@ -36,8 +36,8 @@ def test_saves() -> None: saves = list(saved()) assert len(saves) > 0 - # will throw if not unique - consume(ensure_unique(saves, key=lambda s: s.sid)) + # just check that they are unique (makedict will throw) + make_dict(saves, key=lambda s: s.sid) def test_preserves_extra_attr() -> None: diff --git a/my/tests/tz.py b/my/tests/tz.py deleted file mode 100644 index 92d8f3b..0000000 --- a/my/tests/tz.py +++ /dev/null @@ -1,107 +0,0 @@ -import sys -from datetime import datetime, timedelta - -import pytest -import pytz - -import my.time.tz.main as tz_main -import my.time.tz.via_location as tz_via_location -from my.core import notnone -from my.core.compat import fromisoformat - -from .shared_tz_config import config # autoused fixture - - -def getzone(dt: datetime) -> str: - tz = notnone(dt.tzinfo) - return getattr(tz, 'zone') - - -@pytest.mark.parametrize('fast', [False, True]) -def test_iter_tzs(*, fast: bool, config) -> None: - # TODO hmm.. maybe need to make sure we start with empty config? - config.time.tz.via_location.fast = fast - - ll = list(tz_via_location._iter_tzs()) - zones = [x.zone for x in ll] - - if fast: - assert zones == [ - 'Europe/Rome', - 'Europe/Rome', - 'Europe/Vienna', - 'Europe/Vienna', - 'Europe/Vienna', - ] - else: - assert zones == [ - 'Europe/Rome', - 'Europe/Rome', - 'Europe/Ljubljana', - 'Europe/Ljubljana', - 'Europe/Ljubljana', - ] - - -def test_past() -> None: - """ - Should fallback to the 'home' location provider - """ - dt = fromisoformat('2000-01-01 12:34:45') - dt = tz_main.localize(dt) - assert getzone(dt) == 'America/New_York' - - -def test_future() -> None: - """ - For locations in the future should rely on 'home' location - """ - fut = datetime.now() + timedelta(days=100) - fut = tz_main.localize(fut) - assert getzone(fut) == 'Europe/Moscow' - - -def test_get_tz(config) -> None: - # todo hmm, the way it's implemented at the moment, never returns None? - get_tz = tz_via_location.get_tz - - # not present in the test data - tz = get_tz(fromisoformat('2020-01-01 10:00:00')) - assert notnone(tz).zone == 'Europe/Sofia' - - tz = get_tz(fromisoformat('2017-08-01 11:00:00')) - assert notnone(tz).zone == 'Europe/Vienna' - - tz = get_tz(fromisoformat('2017-07-30 10:00:00')) - assert notnone(tz).zone == 'Europe/Rome' - - tz = get_tz(fromisoformat('2020-10-01 14:15:16')) - assert tz is not None - - on_windows = sys.platform == 'win32' - if not on_windows: - tz = get_tz(datetime.min) - assert tz is not None - else: - # seems this fails because windows doesn't support same date ranges - # https://stackoverflow.com/a/41400321/ - with pytest.raises(OSError): - get_tz(datetime.min) - - -def test_policies() -> None: - naive = fromisoformat('2017-07-30 10:00:00') - assert naive.tzinfo is None # just in case - - # actual timezone at the time - assert getzone(tz_main.localize(naive)) == 'Europe/Rome' - - z = pytz.timezone('America/New_York') - aware = z.localize(naive) - - assert getzone(tz_main.localize(aware)) == 'America/New_York' - - assert getzone(tz_main.localize(aware, policy='convert')) == 'Europe/Rome' - - with pytest.raises(RuntimeError): - assert tz_main.localize(aware, policy='throw') diff --git a/my/time/tz/common.py b/my/time/tz/common.py index c0dd262..e2c428d 100644 --- a/my/time/tz/common.py +++ b/my/time/tz/common.py @@ -1,7 +1,8 @@ from datetime import datetime -from typing import Callable, Literal, cast +from typing import Callable, cast + +from ...core.common import tzdatetime, Literal -from my.core import datetime_aware ''' Depending on the specific data provider and your level of paranoia you might expect different behaviour.. E.g.: @@ -29,11 +30,7 @@ def default_policy() -> TzPolicy: return 'keep' -def localize_with_policy( - lfun: Callable[[datetime], datetime_aware], - dt: datetime, - policy: TzPolicy=default_policy() # noqa: B008 -) -> datetime_aware: +def localize_with_policy(lfun: Callable[[datetime], tzdatetime], dt: datetime, policy: TzPolicy=default_policy()) -> tzdatetime: tz = dt.tzinfo if tz is None: return lfun(dt) diff --git a/my/time/tz/main.py b/my/time/tz/main.py index bdd36b1..624d7aa 100644 --- a/my/time/tz/main.py +++ b/my/time/tz/main.py @@ -1,14 +1,11 @@ ''' Timezone data provider, used to localize timezone-unaware timestamps for other modules ''' - from datetime import datetime - -from my.core import datetime_aware - +from ...core.common import tzdatetime # todo hmm, kwargs isn't mypy friendly.. but specifying types would require duplicating default args. uhoh -def localize(dt: datetime, **kwargs) -> datetime_aware: +def localize(dt: datetime, **kwargs) -> tzdatetime: # todo document patterns for combining multiple data sources # e.g. see https://github.com/karlicoss/HPI/issues/89#issuecomment-716495136 from . import via_location as L diff --git a/my/time/tz/via_location.py b/my/time/tz/via_location.py index 1b2275b..1ed1ba7 100644 --- a/my/time/tz/via_location.py +++ b/my/time/tz/via_location.py @@ -1,39 +1,36 @@ ''' Timezone data provider, guesses timezone based on location data (e.g. GPS) ''' - -from __future__ import annotations - REQUIRES = [ # for determining timezone by coordinate 'timezonefinder', ] -import heapq -import os -from collections import Counter -from collections.abc import Iterable, Iterator -from dataclasses import dataclass -from datetime import date, datetime -from functools import lru_cache -from itertools import groupby -from typing import ( - TYPE_CHECKING, - Any, - Protocol, -) -import pytz +## user might not have tz config section, so makes sense to be more defensive about it +# todo might be useful to extract a helper for this +try: + from my.config import time +except ImportError as ie: + if ie.name != 'time': + raise ie +else: + try: + user_config = time.tz.via_location + except AttributeError as ae: + if not ("'tz'" in str(ae) or "'via_location'"): + raise ae -from my.core import Stats, datetime_aware, make_logger, stat -from my.core.cachew import mcachew -from my.core.compat import TypeAlias -from my.core.source import import_source -from my.core.warnings import high -from my.location.common import LatLon +# deliberately dynamic to prevent confusing mypy +if 'user_config' not in globals(): + globals()['user_config'] = object +## -class config(Protocol): +from my.core import dataclass + +@dataclass +class config(user_config): # less precise, but faster fast: bool = True @@ -49,94 +46,69 @@ class config(Protocol): _iter_tz_refresh_time: int = 6 -def _get_user_config(): - ## user might not have tz config section, so makes sense to be more defensive about it +from collections import Counter +from datetime import date, datetime +from functools import lru_cache +from itertools import groupby +from typing import Iterator, NamedTuple, Optional, Tuple, Any, List, Iterable, Set - class empty_config: ... +import heapq +import pytz +from more_itertools import seekable - try: - from my.config import time - except ImportError as ie: - if "'time'" not in str(ie): - raise ie - return empty_config +from my.core.common import LazyLogger, mcachew, tzdatetime +from my.core.source import import_source - try: - user_config = time.tz.via_location - except AttributeError as ae: - if not ("'tz'" in str(ae) or "'via_location'" in str(ae)): - raise ae - return empty_config +logger = LazyLogger(__name__, level='warning') - return user_config - - -def make_config() -> config: - if TYPE_CHECKING: - import my.config - - user_config: TypeAlias = my.config.time.tz.via_location - else: - user_config = _get_user_config() - - class combined_config(user_config, config): ... - - return combined_config() - - -logger = make_logger(__name__) - - -@lru_cache(None) -def _timezone_finder(*, fast: bool) -> Any: +@lru_cache(2) +def _timezone_finder(fast: bool) -> Any: if fast: # less precise, but faster from timezonefinder import TimezoneFinderL as Finder else: - from timezonefinder import TimezoneFinder as Finder # type: ignore + from timezonefinder import TimezoneFinder as Finder # type: ignore return Finder(in_memory=True) -# for backwards compatibility -def _locations() -> Iterator[tuple[LatLon, datetime_aware]]: - try: - import my.location.all - - for loc in my.location.all.locations(): - if loc.accuracy is not None and loc.accuracy > config.require_accuracy: - continue - yield ((loc.lat, loc.lon), loc.dt) - - except Exception as e: - logger.exception( - "Could not setup via_location using my.location.all provider, falling back to legacy google implementation", exc_info=e - ) - high("Setup my.google.takeout.parser, then my.location.all for better google takeout/location data") - - import my.location.google - - for gloc in my.location.google.locations(): - yield ((gloc.lat, gloc.lon), gloc.dt) - - -# TODO: could use heapmerge or sort the underlying iterators somehow? -# see https://github.com/karlicoss/HPI/pull/237#discussion_r858372934 -def _sorted_locations() -> list[tuple[LatLon, datetime_aware]]: - return sorted(_locations(), key=lambda x: x[1]) - - # todo move to common? Zone = str # NOTE: for now only daily resolution is supported... later will implement something more efficient -@dataclass(unsafe_hash=True) -class DayWithZone: +class DayWithZone(NamedTuple): day: date zone: Zone -def _find_tz_for_locs(finder: Any, locs: Iterable[tuple[LatLon, datetime]]) -> Iterator[DayWithZone]: +from my.location.common import LatLon + +# for backwards compatibility +def _locations() -> Iterator[Tuple[LatLon, datetime]]: + try: + import my.location.all + for loc in my.location.all.locations(): + if loc.accuracy is not None and loc.accuracy > config.require_accuracy: + continue + yield ((loc.lat, loc.lon), loc.dt) + + except Exception as e: + from my.core.warnings import high + logger.exception("Could not setup via_location using my.location.all provider, falling back to legacy google implementation", exc_info=e) + high("Setup my.google.takeout.parser, then my.location.all for better google takeout/location data") + + import my.location.google + + for gloc in my.location.google.locations(): + yield ((gloc.lat, gloc.lon), gloc.dt) + +# TODO: could use heapmerge or sort the underlying iterators somehow? +# see https://github.com/karlicoss/HPI/pull/237#discussion_r858372934 +def _sorted_locations() -> List[Tuple[LatLon, datetime]]: + return list(sorted(_locations(), key=lambda x: x[1])) + + +def _find_tz_for_locs(finder: Any, locs: Iterable[Tuple[LatLon, datetime]]) -> Iterator[DayWithZone]: for (lat, lon), dt in locs: # TODO right. its _very_ slow... zone = finder.timezone_at(lat=lat, lng=lon) @@ -148,28 +120,25 @@ def _find_tz_for_locs(finder: Any, locs: Iterable[tuple[LatLon, datetime]]) -> I # TODO this is probably a bit expensive... test & benchmark ldt = dt.astimezone(tz) ndate = ldt.date() - # if pdt is not None and ndate < pdt.date(): + #if pdt is not None and ndate < pdt.date(): # # TODO for now just drop and collect the stats # # I guess we'd have minor drops while air travel... # warnings.append("local time goes backwards {ldt} ({tz}) < {pdt}") # continue - # pdt = ldt - z = tz.zone - assert z is not None + #pdt = ldt + z = tz.zone; assert z is not None yield DayWithZone(day=ndate, zone=z) - # Note: this takes a while, as the upstream since _locations isn't sorted, so this # has to do an iterative sort of the entire my.locations.all list def _iter_local_dates() -> Iterator[DayWithZone]: - cfg = make_config() - finder = _timezone_finder(fast=cfg.fast) # rely on the default - # pdt = None + finder = _timezone_finder(fast=config.fast) # rely on the default + #pdt = None # TODO: warnings doesn't actually warn? # warnings = [] - locs: Iterable[tuple[LatLon, datetime]] - locs = _sorted_locations() if cfg.sort_locations else _locations() + locs: Iterable[Tuple[LatLon, datetime]] + locs = _sorted_locations() if config.sort_locations else _locations() yield from _find_tz_for_locs(finder, locs) @@ -181,16 +150,14 @@ def _iter_local_dates() -> Iterator[DayWithZone]: def _iter_local_dates_fallback() -> Iterator[DayWithZone]: from my.location.fallback.all import fallback_locations as flocs - cfg = make_config() - - def _fallback_locations() -> Iterator[tuple[LatLon, datetime]]: + def _fallback_locations() -> Iterator[Tuple[LatLon, datetime]]: for loc in sorted(flocs(), key=lambda x: x.dt): yield ((loc.lat, loc.lon), loc.dt) - yield from _find_tz_for_locs(_timezone_finder(fast=cfg.fast), _fallback_locations()) + yield from _find_tz_for_locs(_timezone_finder(fast=config.fast), _fallback_locations()) -def most_common(lst: Iterator[DayWithZone]) -> DayWithZone: +def most_common(lst: List[DayWithZone]) -> DayWithZone: res, _ = Counter(lst).most_common(1)[0] return res @@ -205,59 +172,68 @@ def _iter_tz_depends_on() -> str: 2022-04-26_12 2022-04-26_18 """ - cfg = make_config() - mod = cfg._iter_tz_refresh_time + mod = config._iter_tz_refresh_time assert mod >= 1 day = str(date.today()) hr = datetime.now().hour hr_truncated = hr // mod * mod - return f"{day}_{hr_truncated}" + return "{}_{}".format(day, hr_truncated) # refresh _iter_tzs every few hours -- don't think a better depends_on is possible dynamically -@mcachew(depends_on=_iter_tz_depends_on) +@mcachew(logger=logger, depends_on=_iter_tz_depends_on) def _iter_tzs() -> Iterator[DayWithZone]: # since we have no control over what order the locations are returned, # we need to sort them first before we can do a groupby - by_day = lambda p: p.day - - local_dates: list[DayWithZone] = sorted(_iter_local_dates(), key=by_day) + local_dates: List[DayWithZone] = list(_iter_local_dates()) + local_dates.sort(key=lambda p: p.day) logger.debug(f"no. of items using exact locations: {len(local_dates)}") - local_dates_fallback: list[DayWithZone] = sorted(_iter_local_dates_fallback(), key=by_day) + local_dates_fallback: List[DayWithZone] = list(_iter_local_dates_fallback()) + local_dates_fallback.sort(key=lambda p: p.day) # find days that are in fallback but not in local_dates (i.e., missing days) - local_dates_set: set[date] = {d.day for d in local_dates} - use_fallback_days: list[DayWithZone] = [d for d in local_dates_fallback if d.day not in local_dates_set] + local_dates_set: Set[date] = set(d.day for d in local_dates) + use_fallback_days: List[DayWithZone] = [d for d in local_dates_fallback if d.day not in local_dates_set] logger.debug(f"no. of items being used from fallback locations: {len(use_fallback_days)}") # combine local_dates and missing days from fallback into a sorted list - all_dates = heapq.merge(local_dates, use_fallback_days, key=by_day) - # todo could probably use heapify here instead of heapq.merge? + all_dates = heapq.merge(local_dates, use_fallback_days, key=lambda p: p.day) - for d, gr in groupby(all_dates, key=by_day): - logger.debug(f"processed {d}{', using fallback' if d in local_dates_set else ''}") - zone = most_common(gr).zone + for d, gr in groupby(all_dates, key=lambda p: p.day): + logger.info(f"processed {d}{', using fallback' if d in local_dates_set else ''}") + zone = most_common(list(gr)).zone yield DayWithZone(day=d, zone=zone) @lru_cache(1) -def _day2zone() -> dict[date, pytz.BaseTzInfo]: - # NOTE: kinda unfortunate that this will have to process all days before returning result for just one - # however otherwise cachew cache might never be initialized properly - # so we'll always end up recomputing everything during subsequent runs - return {dz.day: pytz.timezone(dz.zone) for dz in _iter_tzs()} +def loc_tz_getter() -> Iterator[DayWithZone]: + # seekable makes it cache the emitted values + return seekable(_iter_tzs()) -def _get_day_tz(d: date) -> pytz.BaseTzInfo | None: - return _day2zone().get(d) +# todo expose zone names too? +@lru_cache(maxsize=None) +def _get_day_tz(d: date) -> Optional[pytz.BaseTzInfo]: + sit = loc_tz_getter() + # todo hmm. seeking is not super efficient... might need to use some smarter dict-based cache + # hopefully, this method itself caches stuff forthe users, so won't be too bad + sit.seek(0) # type: ignore + + zone: Optional[str] = None + for x, tz in sit: + if x == d: + zone = tz + if x >= d: + break + return None if zone is None else pytz.timezone(zone) # ok to cache, there are only a few home locations? -@lru_cache(None) -def _get_home_tz(loc: LatLon) -> pytz.BaseTzInfo | None: +@lru_cache(maxsize=None) +def _get_home_tz(loc: LatLon) -> Optional[pytz.BaseTzInfo]: (lat, lng) = loc - finder = _timezone_finder(fast=False) # ok to use slow here for better precision + finder = _timezone_finder(fast=False) # ok to use slow here for better precision zone = finder.timezone_at(lat=lat, lng=lng) if zone is None: # TODO shouldn't really happen, warn? @@ -266,7 +242,7 @@ def _get_home_tz(loc: LatLon) -> pytz.BaseTzInfo | None: return pytz.timezone(zone) -def get_tz(dt: datetime) -> pytz.BaseTzInfo | None: +def _get_tz(dt: datetime) -> Optional[pytz.BaseTzInfo]: ''' Given a datetime, returns the timezone for that date. ''' @@ -282,14 +258,16 @@ def get_tz(dt: datetime) -> pytz.BaseTzInfo | None: # that datetime is between, else fallback on your first home location, so it acts # as a last resort from my.location.fallback import via_home as home - loc = list(home.estimate_location(dt)) assert len(loc) == 1, f"should only have one home location, received {loc}" return _get_home_tz(loc=(loc[0].lat, loc[0].lon)) +# expose as 'public' function +get_tz = _get_tz -def localize(dt: datetime) -> datetime_aware: - tz = get_tz(dt) + +def localize(dt: datetime) -> tzdatetime: + tz = _get_tz(dt) if tz is None: # TODO -- this shouldn't really happen.. think about it carefully later return dt @@ -297,17 +275,20 @@ def localize(dt: datetime) -> datetime_aware: return tz.localize(dt) -def stats(*, quick: bool = False) -> Stats: +from ...core import stat, Stats +def stats(quick: bool=False) -> Stats: if quick: prev, config.sort_locations = config.sort_locations, False - res = {'first': next(_iter_local_dates())} + res = { + 'first': next(_iter_local_dates()) + } config.sort_locations = prev return res # TODO not sure what would be a good stat() for this module... # might be nice to print some actual timezones? # there aren't really any great iterables to expose + import os VIA_LOCATION_START_YEAR = int(os.environ.get("VIA_LOCATION_START_YEAR", 1990)) - def localized_years(): last = datetime.now().year + 2 # note: deliberately take + 2 years, so the iterator exhausts. otherwise stuff might never get cached @@ -315,17 +296,4 @@ def stats(*, quick: bool = False) -> Stats: for Y in range(VIA_LOCATION_START_YEAR, last): dt = datetime.fromisoformat(f'{Y}-01-01 01:01:01') yield localize(dt) - return stat(localized_years) - - -## deprecated -- keeping for now as might be used in other modules? -if not TYPE_CHECKING: - from my.core.compat import deprecated - - @deprecated('use get_tz function instead') - def _get_tz(*args, **kwargs): - return get_tz(*args, **kwargs) - - -## diff --git a/my/tinder/android.py b/my/tinder/android.py index 5a5d887..7e5f535 100644 --- a/my/tinder/android.py +++ b/my/tinder/android.py @@ -3,22 +3,19 @@ Tinder data from Android app database (in =/data/data/com.tinder/databases/tinde """ from __future__ import annotations -import sqlite3 -from collections import Counter, defaultdict -from collections.abc import Iterator, Mapping, Sequence +from collections import defaultdict, Counter from dataclasses import dataclass from datetime import datetime, timezone from itertools import chain from pathlib import Path -from typing import Union +import sqlite3 +from typing import Sequence, Iterator, Union, Dict, List, Mapping -from my.core import Paths, Res, Stats, datetime_aware, get_files, make_logger, stat +from my.core import Paths, get_files, Res, assert_never, stat, Stats, datetime_aware, make_logger from my.core.common import unique_everseen -from my.core.compat import assert_never from my.core.error import echain from my.core.sqlite import sqlite_connection - -import my.config # isort: skip +import my.config logger = make_logger(__name__) @@ -95,10 +92,7 @@ def _entities() -> Iterator[Res[_Entity]]: for idx, path in enumerate(paths): logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}') with sqlite_connection(path, immutable=True, row_factory='row') as db: - try: - yield from _handle_db(db) - except Exception as e: - yield e + yield from _handle_db(db) def _handle_db(db: sqlite3.Connection) -> Iterator[Res[_Entity]]: @@ -106,12 +100,11 @@ def _handle_db(db: sqlite3.Connection) -> Iterator[Res[_Entity]]: user_profile_rows = list(db.execute('SELECT * FROM profile_user_view')) if len(user_profile_rows) == 0: - # shit, sometime in 2023 profile_user_view stopped containing user profile.. + # shit, sometime in 2023 profile_user_view stoppped containing user profile.. # presumably the most common from_id/to_id would be our own username counter = Counter([id_ for (id_,) in db.execute('SELECT from_id FROM message UNION ALL SELECT to_id FROM message')]) - if len(counter) > 0: # this might happen if db is empty (e.g. user got logged out) - [(you_id, _)] = counter.most_common(1) - yield Person(id=you_id, name='you') + [(you_id, _)] = counter.most_common(1) + yield Person(id=you_id, name='you') for row in chain( user_profile_rows, @@ -166,8 +159,8 @@ def _parse_msg(row: sqlite3.Row) -> _Message: # todo maybe it's rich_entities method? def entities() -> Iterator[Res[Entity]]: - id2person: dict[str, Person] = {} - id2match: dict[str, Match] = {} + id2person: Dict[str, Person] = {} + id2match: Dict[str, Match] = {} for x in unique_everseen(_entities): if isinstance(x, Exception): yield x @@ -219,7 +212,7 @@ def messages() -> Iterator[Res[Message]]: # todo not sure, maybe it's not fundamental enough to keep here... def match2messages() -> Iterator[Res[Mapping[Match, Sequence[Message]]]]: - res: dict[Match, list[Message]] = defaultdict(list) + res: Dict[Match, List[Message]] = defaultdict(list) for x in entities(): if isinstance(x, Exception): yield x diff --git a/my/topcoder.py b/my/topcoder.py deleted file mode 100644 index 40df77c..0000000 --- a/my/topcoder.py +++ /dev/null @@ -1,92 +0,0 @@ -import json -from collections.abc import Iterator, Sequence -from dataclasses import dataclass -from functools import cached_property -from pathlib import Path - -from my.core import Res, datetime_aware, get_files -from my.core.compat import fromisoformat -from my.experimental.destructive_parsing import Manager - -from my.config import topcoder as config # type: ignore[attr-defined] # isort: skip - - -def inputs() -> Sequence[Path]: - return get_files(config.export_path) - - -@dataclass -class Competition: - contest_id: str - contest: str - percentile: float - date_str: str - - @cached_property - def uid(self) -> str: - return self.contest_id - - @cached_property - def when(self) -> datetime_aware: - return fromisoformat(self.date_str) - - @classmethod - def make(cls, j) -> Iterator[Res['Competition']]: - assert isinstance(j.pop('rating'), float) - assert isinstance(j.pop('placement'), int) - - cid = j.pop('challengeId') - cname = j.pop('challengeName') - percentile = j.pop('percentile') - date_str = j.pop('date') - - yield cls( - contest_id=cid, - contest=cname, - percentile=percentile, - date_str=date_str, - ) - - -def _parse_one(p: Path) -> Iterator[Res[Competition]]: - d = json.loads(p.read_text()) - - # TODO manager should be a context manager? - m = Manager() - - h = m.helper(d) - h.pop_if_primitive('version', 'id') - - h = h.zoom('result') - h.check('success', expected=True) - h.check('status', 200) - h.pop_if_primitive('metadata') - - h = h.zoom('content') - h.pop_if_primitive('handle', 'handleLower', 'userId', 'createdAt', 'updatedAt', 'createdBy', 'updatedBy') - - # NOTE at the moment it's empty for me, but it will result in an error later if there is some data here - h.zoom('DEVELOP').zoom('subTracks') - - h = h.zoom('DATA_SCIENCE') - # TODO multi zoom? not sure which axis, e.g. - # zoom('SRM', 'history') or zoom('SRM', 'MARATHON_MATCH') - # or zoom(('SRM', 'history'), ('MARATHON_MATCH', 'history')) - srms = h.zoom('SRM').zoom('history') - mms = h.zoom('MARATHON_MATCH').zoom('history') - - for c in srms.item + mms.item: - # NOTE: so here we are actually just using pure dicts in .make method - # this is kinda ok since it will be checked by parent Helper - # but also expects cooperation from .make method (e.g. popping items from the dict) - # could also wrap in helper and pass to .make .. not sure - # an argument could be made that .make isn't really a class methond.. - # it's pretty specific to this parser only - yield from Competition.make(j=c) - - yield from m.check() - - -def data() -> Iterator[Res[Competition]]: - *_, last = inputs() - return _parse_one(last) diff --git a/my/twitter/all.py b/my/twitter/all.py index c2c471e..4714021 100644 --- a/my/twitter/all.py +++ b/my/twitter/all.py @@ -1,11 +1,11 @@ """ Unified Twitter data (merged from the archive and periodic updates) """ -from collections.abc import Iterator - +from typing import Iterator from ..core import Res from ..core.source import import_source -from .common import Tweet, merge_tweets +from .common import merge_tweets, Tweet + # NOTE: you can comment out the sources you don't need src_twint = import_source(module_name='my.twitter.twint') diff --git a/my/twitter/android.py b/my/twitter/android.py deleted file mode 100644 index 7e8f170..0000000 --- a/my/twitter/android.py +++ /dev/null @@ -1,318 +0,0 @@ -""" -Twitter data from official app for Android -""" - -from __future__ import annotations - -import re -from collections.abc import Iterator, Sequence -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from struct import unpack_from - -from my.core import LazyLogger, Paths, Res, datetime_aware, get_files -from my.core.common import unique_everseen -from my.core.sqlite import sqlite_connect_immutable - -from .common import permalink - -import my.config # isort: skip - -logger = LazyLogger(__name__) - - -@dataclass -class config(my.config.twitter.android): - # paths[s]/glob to the exported sqlite databases - export_path: Paths - - -def inputs() -> Sequence[Path]: - # NOTE: individual databases are very patchy. - # e.g. some contain hundreds of my bookmarks, whereas other contain just a few - # good motivation for synthetic exports - return get_files(config.export_path) - - -@dataclass(unsafe_hash=True) -class Tweet: - id_str: str - created_at: datetime_aware - screen_name: str - text: str - - @property - def permalink(self) -> str: - return permalink(screen_name=self.screen_name, id=self.id_str) - - -def _parse_content(data: bytes) -> str: - pos = 0 - - def skip(count: int) -> None: - nonlocal pos - pos += count - - def getstring(slen: int) -> str: - if slen == 1: - lfmt = '>B' - elif slen == 2: - lfmt = '>H' - else: - raise RuntimeError - - (sz,) = unpack_from(lfmt, data, offset=pos) - skip(slen) - assert sz > 0 - assert sz <= 10000 # sanity check? - - # soo, this is how it should ideally work: - # (ss,) = unpack_from(f'{sz}s', data, offset=pos) - # skip(sz) - # however sometimes there is a discrepancy between string length in header and actual length (if you stare at the data) - # example is 1725868458246570412 - # wtf??? (see logging below) - - # ughhhh - seps = [ - b'I\x08', - b'I\x09', - ] - sep_idxs = [data[pos:].find(sep) for sep in seps] - sep_idxs = [i for i in sep_idxs if i != -1] - assert len(sep_idxs) > 0 - sep_idx = min(sep_idxs) - - # print("EXPECTED LEN", sz, "GOT", sep_idx, "DIFF", sep_idx - sz) - - zz = data[pos : pos + sep_idx] - skip(sep_idx) - return zz.decode('utf8') - - skip(2) # always starts with 4a03? - - (xx,) = unpack_from('B', data, offset=pos) - skip(1) - - # wtf is this... maybe it's a bitmask? - slen = { - 66: 1, - 67: 2, - 106: 1, - 107: 2, - }[xx] - - text = getstring(slen=slen) - - # after the main tweet text it contains entities (e.g. shortened urls) - # however couldn't reverse engineer the schema properly, the links are kinda all over the place - - # TODO this also contains image alt descriptions? - # see 1665029077034565633 - - extracted = [] - linksep = 0x6A - while True: - m = re.search(b'\x6a.http', data[pos:]) - if m is None: - break - - qq = m.start() - pos += qq - - while True: - if data[pos] != linksep: - break - pos += 1 - (sz,) = unpack_from('B', data, offset=pos) - pos += 1 - (ss,) = unpack_from(f'{sz}s', data, offset=pos) - pos += sz - extracted.append(ss) - - replacements = {} - i = 0 - while i < len(extracted): - if b'https://t.co/' in extracted[i]: - key = extracted[i].decode('utf8') - value = extracted[i + 1].decode('utf8') - i += 2 - replacements[key] = value - else: - i += 1 - - for k, v in replacements.items(): - text = text.replace(k, v) - assert 'https://t.co/' not in text # make sure we detected all links - - return text - - -_SELECT_OWN_TWEETS = '_SELECT_OWN_TWEETS' - - -def get_own_user_id(conn) -> str: - # unclear what's the reliable way to query it, so we use multiple different ones and arbitrate - # NOTE: 'SELECT DISTINCT ev_owner_id FROM lists' doesn't work, might include lists from other people? - res: set[str] = set() - # need to cast as it's int by default - for q in [ - 'SELECT DISTINCT CAST(list_mapping_user_id AS TEXT) FROM list_mapping', - 'SELECT DISTINCT CAST(owner_id AS TEXT) FROM cursors', - 'SELECT DISTINCT CAST(user_id AS TEXT) FROM users WHERE _id == 1', - # ugh, sometimes all of the above are empty... - # for the rest it seems: - # - is_active_creator is NULL - # - is_graduated is NULL - # - profile_highlighted_info is NULL - 'SELECT DISTINCT CAST(user_id AS TEXT) FROM users WHERE is_active_creator == 0 AND is_graduated == 1 AND profile_highlights_info IS NOT NULL', - ]: - res |= {r for (r,) in conn.execute(q)} - - assert len(res) <= 1, res - if len(res) == 0: - # sometimes even all of the above doesn't help... - # last resort is trying to get from status_groups table - # however we can't always use it because it might contain multiple different owner_id? - # not sure, maybe it will break as well and we'll need to fallback on the most common or something.. - res |= {r for (r,) in conn.execute('SELECT DISTINCT CAST(owner_id AS TEXT) FROM status_groups')} - assert len(res) == 1, res - [r] = res - return r - - -# NOTE: -# - it also has statuses_r_ent_content which has entities' links replaced -# but they are still ellipsized (e.g. check 1692905005479580039) -# so let's just uses statuses_content -# - there is also timeline_created_at, but they look like made up timestamps -# don't think they represent bookmarking time -# - timeline_type -# 7, 8, 9: some sort of notifications or cursors, should exclude -# 14: some converstaionthread stuff? -# 17: ??? some cursors but also tweets NOTE: they seem to contribute to user's tweets data, so make sure not to delete -# 18: ??? relatively few, maybe 20 of them, also they all have timeline_is_preview=1? -# most of them have our own id as timeline_sender? -# I think it might actually be 'replies' tab -- also contains some retweets etc -# 26: ??? very low sort index -# 28: weird, contains lots of our own tweets, but also a bunch of unrelated.. -# 29: seems like it contains the favorites! -# 30: seems like it contains the bookmarks -# 34: contains some tweets -- not sure.. -# 63: contains the bulk of data -# 69: ??? just a few tweets -# - timeline_data_type -# 1 : the bulk of tweets, but also some notifications etc?? -# 2 : who-to-follow/community-to-join. contains a couple of tweets, but their corresponding status_id is NULL -# 8 : who-to-follow/notification -# 13: semantic-core/who-to-follow -# 14: cursor -# 17: trends -# 27: notification -# 31: some superhero crap -# 37: semantic-core -# 42: community-to-join -# - timeline_entity_type -# 1 : contains the bulk of data -- either tweet-*/promoted-tweet-*. However some notification-* and some just contain raw ids?? -# 11: some sort of 'superhero-superhero' crap -# 13: always cursors -# 15: tweet-*/tweet:*/home-conversation-*/trends-*/and lots of other crap -# 31: always notification-* -# - timeline_data_type_group -# 0 : tweets? -# 6 : always notifications?? -# 42: tweets (bulk of them) -def _process_one(f: Path, *, where: str) -> Iterator[Res[Tweet]]: - # meh... maybe separate this function into special ones for tweets/bookmarks/likes - select_own = _SELECT_OWN_TWEETS in where - with sqlite_connect_immutable(f) as db: - if select_own: - own_user_id = get_own_user_id(db) - db_where = where.replace(_SELECT_OWN_TWEETS, own_user_id) - else: - db_where = where - - # NOTE: we used to get this from 'timeline_view' - # however seems that it's missing a fair amount of data that's present instatuses table... - QUERY = ''' - SELECT - CAST(statuses.status_id AS TEXT), /* int by default */ - users.username, - statuses.created, - CAST(statuses.content AS BLOB), - statuses.quoted_tweet_id - FROM statuses FULL OUTER JOIN users - ON statuses.author_id == users.user_id - WHERE - /* there are sometimes a few shitty statuses in the db with weird ids which are duplicating other tweets - don't want to filter by status_id < 10 ** 10, since there might legit be statuses with low ids? - so this is the best I came up with.. - */ - NOT (statuses.in_r_user_id == -1 AND statuses.in_r_status_id == -1 AND statuses.conversation_id == 0) - ''' - - def _query_one(*, where: str, quoted: set[int]) -> Iterator[Res[Tweet]]: - for ( - tweet_id, - user_username, - created_ms, - blob, - quoted_id, - ) in db.execute(f'{QUERY} AND {where}'): - quoted.add(quoted_id) # if no quoted tweet, id is 0 here - - try: - content = _parse_content(blob) - except Exception as e: - yield e - continue - - yield Tweet( - id_str=tweet_id, - # TODO double check it's utc? - created_at=datetime.fromtimestamp(created_ms / 1000, tz=timezone.utc), - screen_name=user_username, - text=content, - ) - - quoted: set[int] = set() - yield from _query_one(where=db_where, quoted=quoted) - # get quoted tweets 'recursively' - # TODO maybe do it for favs/bookmarks too? not sure - while select_own and len(quoted) > 0: - db_where = 'status_id IN (' + ','.join(map(str, sorted(quoted))) + ')' - quoted = set() - yield from _query_one(where=db_where, quoted=quoted) - - -def _entities(*, where: str) -> Iterator[Res[Tweet]]: - # TODO might need to sort by timeline_sort_index again? - def it() -> Iterator[Res[Tweet]]: - paths = inputs() - total = len(paths) - width = len(str(total)) - for idx, path in enumerate(paths): - logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}') - yield from _process_one(path, where=where) - - # TODO hmm maybe unique_everseen should be a decorator? - return unique_everseen(it) - - -def bookmarks() -> Iterator[Res[Tweet]]: - # NOTE: in principle we get the bulk of bookmarks via timeline_type == 30 filter - # however we still might miss on a few (I think the timeline_type 30 only refreshes when you enter bookmarks in the app) - # if you bookmarked in the home feed, it might end up as status_bookmarked == 1 but not necessarily as timeline_type 30 - return _entities(where='statuses.bookmarked == 1') - - -def likes() -> Iterator[Res[Tweet]]: - # NOTE: similarly to bookmarks, we could use timeline_type == 29, but it's only refreshed if we actually open likes tab - return _entities(where='statuses.favorited == 1') - - -def tweets() -> Iterator[Res[Tweet]]: - # NOTE: where timeline_type == 18 covers quite a few of our on tweets, but not everything - # querying by our own user id seems the most exhaustive - return _entities(where=f'users.user_id == {_SELECT_OWN_TWEETS} OR statuses.retweeted == 1') diff --git a/my/twitter/archive.py b/my/twitter/archive.py index c9d2dbc..0ea6b24 100644 --- a/my/twitter/archive.py +++ b/my/twitter/archive.py @@ -2,74 +2,73 @@ Twitter data (uses [[https://help.twitter.com/en/managing-your-account/how-to-download-your-twitter-archive][official twitter archive export]]) """ -from __future__ import annotations -import html -import json # hmm interesting enough, orjson didn't give much speedup here? -from abc import abstractmethod -from collections.abc import Iterator, Sequence +# before this config was named 'twitter', doesn't make too much sense for archive +# todo unify with other code like this, e.g. time.tz.via_location +try: + from my.config import twitter_archive as user_config +except ImportError as ie: + if not (ie.name == 'my.config' and 'twitter_archive' in str(ie)): + # must be caused by something else + raise ie + try: + from my.config import twitter as user_config # type: ignore[assignment] + except ImportError: + raise ie # raise the original exception.. must be something else + else: + from ..core import warnings + warnings.high('my.config.twitter is deprecated! Please rename it to my.config.twitter_archive in your config') +## + + from dataclasses import dataclass from datetime import datetime -from functools import cached_property from itertools import chain +import json # hmm interesting enough, orjson didn't give much speedup here? from pathlib import Path +from functools import cached_property +import html from typing import ( - TYPE_CHECKING, + Iterator, + List, + Optional, + Sequence, ) from more_itertools import unique_everseen from my.core import ( - Json, - Paths, - Res, - Stats, datetime_aware, get_files, make_logger, stat, - warnings, + Json, + Paths, + Res, + Stats, ) +from my.core import warnings +from my.core.cfg import make_config from my.core.serialize import dumps as json_dumps from .common import TweetId, permalink + +@dataclass +class twitter_archive(user_config): + export_path: Paths # path[s]/glob to the twitter archive takeout + + +### + +config = make_config(twitter_archive) + + logger = make_logger(__name__) -class config: - @property - @abstractmethod - def export_path(self) -> Paths: - """path[s]/glob to the twitter archive takeout""" - raise NotImplementedError - - -def make_config() -> config: - # before this config was named 'twitter', doesn't make too much sense for archive - # todo unify with other code like this, e.g. time.tz.via_location - try: - from my.config import twitter_archive as user_config - except ImportError as ie: - if not (ie.name == 'my.config' and 'twitter_archive' in str(ie)): - # must be caused by something else - raise ie - try: - from my.config import twitter as user_config # type: ignore[assignment] - except ImportError: - raise ie # raise the original exception.. must be something else # noqa: B904 - else: - warnings.high('my.config.twitter is deprecated! Please rename it to my.config.twitter_archive in your config') - ## - - class combined_config(user_config, config): - pass - - return combined_config() - - def inputs() -> Sequence[Path]: - return get_files(make_config().export_path) + return get_files(config.export_path) # TODO make sure it's not used anywhere else and simplify interface @@ -106,7 +105,7 @@ class Tweet: repls.append((fr, to, me['display_url'])) # todo not sure, maybe use media_url_https instead? # for now doing this for compatibility with twint - repls = sorted(repls) + repls = list(sorted(repls)) parts = [] idx = 0 for fr, to, what in repls: @@ -122,7 +121,7 @@ class Tweet: return res @property - def urls(self) -> list[str]: + def urls(self) -> List[str]: ents = self.entities us = ents['urls'] return [u['expanded_url'] for u in us] @@ -163,10 +162,10 @@ class Like: return self.raw['tweetId'] @property - def text(self) -> str | None: + def text(self) -> Optional[str]: # NOTE: likes basically don't have anything except text and url # ugh. I think none means that tweet was deleted? - res: str | None = self.raw.get('fullText') + res: Optional[str] = self.raw.get('fullText') if res is None: return None res = html.unescape(res) @@ -187,7 +186,7 @@ class ZipExport: if not (self.zpath / 'Your archive.html').exists(): self.old_format = True - def raw(self, what: str, *, fname: str | None = None) -> Iterator[Json]: + def raw(self, what: str, *, fname: Optional[str] = None) -> Iterator[Json]: logger.info(f'{self.zpath} : processing {what}') path = fname or what @@ -227,80 +226,11 @@ class ZipExport: yield Like(r, screen_name=self.screen_name) -def _cleanup_tweet_json(rj: Json) -> None: - # note: for now this isn't used, was just an attempt to normalise raw data... - - rj.pop('edit_info', None) # useless for downstream processing, but results in dupes, so let's remove it - - ## could probably just take the last one? dunno - rj.pop('retweet_count', None) - rj.pop('favorite_count', None) - ## - - entities = rj.get('entities', {}) - ext_entities = rj.get('extended_entities', {}) - - # TODO shit. unclear how to 'merge' changes to these - # links sometimes change for no apparent reason -- and sometimes old one is still valid but not the new one??? - for m in entities.get('media', {}): - m.pop('media_url', None) - m.pop('media_url_https', None) - for m in ext_entities.get('media', {}): - m.pop('media_url', None) - m.pop('media_url_https', None) - ## - - for m in entities.get('user_mentions', {}): - # changes if user renames themselves... - m.pop('name', None) - - # hmm so can change to -1? maybe if user was deleted? - # but also can change to actually something else?? second example - entities.pop('user_mentions', None) - - # TODO figure out what else is changing there later... - rj.pop('entities', None) - rj.pop('extended_entities', None) - - ## useless attributes which should be fine to exclude - rj.pop('possibly_sensitive', None) # not sure what is this.. sometimes appears with False value?? - rj.pop('withheld_in_countries', None) - rj.pop('lang', None) - ## - - # ugh. might change if the Twitter client was deleted or description renamed?? - rj.pop('source', None) - - ## ugh. sometimes trailing 0 after decimal point is present? - rj.pop('coordinates', None) - rj.get('geo', {}).pop('coordinates', None) - ## - - # ugh. this changes if user changed their name... - # or disappears if account was deleted? - rj.pop('in_reply_to_screen_name', None) - - # todo not sure about list and sorting? although can't hurt considering json is not iterative? def tweets() -> Iterator[Res[Tweet]]: _all = chain.from_iterable(ZipExport(i).tweets() for i in inputs()) - - # NOTE raw json data in archived tweets changes all the time even for same tweets - # there is an attempt to clean it up... but it's tricky since users rename themselves, twitter stats are changing - # so it's unclear how to pick up - # we should probably 'merge' tweets into a canonical version, e.g. - # - pick latest tweet stats - # - keep history of usernames we were replying to that share the same user id - # - pick 'best' media url somehow?? - # - normalise coordinates data - def key(t: Tweet): - # NOTE: not using t.text, since it actually changes if entities in tweet are changing... - # whereas full_text seems stable - text = t.raw['full_text'] - return (t.created_at, t.id_str, text) - - res = unique_everseen(_all, key=key) - yield from sorted(res, key=lambda t: t.created_at) + res = unique_everseen(_all, key=json_dumps) + yield from sorted(res, key=lambda t: t.dt) def likes() -> Iterator[Res[Like]]: @@ -318,5 +248,4 @@ def stats() -> Stats: ## Deprecated stuff -if not TYPE_CHECKING: - Tid = TweetId +Tid = TweetId diff --git a/my/twitter/common.py b/my/twitter/common.py index 8c346f6..258216f 100644 --- a/my/twitter/common.py +++ b/my/twitter/common.py @@ -1,19 +1,17 @@ -from my.core import __NOT_HPI_MODULE__ # isort: skip +from my.core import __NOT_HPI_MODULE__ -from collections.abc import Iterator from itertools import chain -from typing import Any +from typing import Iterator, Any from more_itertools import unique_everseen + # TODO add proper Protocol for Tweet Tweet = Any TweetId = str -from my.core import Res, warn_if_empty - - +from my.core import warn_if_empty, Res @warn_if_empty def merge_tweets(*sources: Iterator[Res[Tweet]]) -> Iterator[Res[Tweet]]: def key(r: Res[Tweet]): diff --git a/my/twitter/talon.py b/my/twitter/talon.py index dbf2e2e..306a735 100644 --- a/my/twitter/talon.py +++ b/my/twitter/talon.py @@ -1,17 +1,13 @@ """ Twitter data from Talon app database (in =/data/data/com.klinker.android.twitter_l/databases/=) """ - from __future__ import annotations -import re -import sqlite3 -from abc import abstractmethod -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime, timezone -from pathlib import Path -from typing import Union +import re +import sqlite3 +from typing import Iterator, Sequence, Union from my.core import Paths, Res, datetime_aware, get_files from my.core.common import unique_everseen @@ -19,25 +15,18 @@ from my.core.sqlite import sqlite_connection from .common import TweetId, permalink - -class config: - @property - @abstractmethod - def export_path(self) -> Paths: - raise NotImplementedError +from my.config import twitter as user_config -def make_config() -> config: - from my.config import twitter as user_config - - class combined_config(user_config.talon, config): - pass - - return combined_config() +@dataclass +class config(user_config.talon): + # paths[s]/glob to the exported sqlite databases + export_path: Paths +from pathlib import Path def inputs() -> Sequence[Path]: - return get_files(make_config().export_path) + return get_files(config.export_path) @dataclass(unsafe_hash=True) @@ -57,16 +46,12 @@ class Tweet: @dataclass(unsafe_hash=True) class _IsTweet: tweet: Tweet - - @dataclass(unsafe_hash=True) class _IsFavorire: tweet: Tweet Entity = Union[_IsTweet, _IsFavorire] - - def _entities() -> Iterator[Res[Entity]]: for f in inputs(): yield from _process_one(f) @@ -74,7 +59,7 @@ def _entities() -> Iterator[Res[Entity]]: def _process_one(f: Path) -> Iterator[Res[Entity]]: handlers = { - 'user_tweets.db': _process_user_tweets, + 'user_tweets.db' : _process_user_tweets, 'favorite_tweets.db': _process_favorite_tweets, } fname = f.name diff --git a/my/twitter/twint.py b/my/twitter/twint.py index 9d36a93..ceb5406 100644 --- a/my/twitter/twint.py +++ b/my/twitter/twint.py @@ -1,17 +1,17 @@ """ Twitter data (tweets and favorites). Uses [[https://github.com/twintproject/twint][Twint]] data export. """ -from collections.abc import Iterator from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from typing import NamedTuple +from typing import NamedTuple, Iterator, List -from my.core import Json, LazyLogger, Paths, Res, Stats, datetime_aware, get_files, stat + +from my.core import Paths, Res, get_files, LazyLogger, Json, datetime_aware, stat, Stats from my.core.cfg import make_config from my.core.sqlite import sqlite_connection -from my.config import twint as user_config # isort: skip +from my.config import twint as user_config # TODO move to twitter.twint config structure @@ -54,7 +54,7 @@ class Tweet(NamedTuple): # https://github.com/thomasancheriyil/Red-Tide-Detection-based-on-Twitter/blob/beb200be60cc66dcbc394e670513715509837812/python/twitterGapParse.py#L61-L62 # # twint is also saving 'timezone', but this is local machine timezone at the time of scraping? - # perhaps they thought date-time-ms was local time... or just kept it just in case (they are keeping lots on unnecessary stuff in the db) + # perhaps they thought date-time-ms was local time... or just kept it just in case (they are keepin lots on unnecessary stuff in the db) return datetime.fromtimestamp(seconds, tz=tz) @property @@ -76,7 +76,7 @@ class Tweet(NamedTuple): return text @property - def urls(self) -> list[str]: + def urls(self) -> List[str]: ustr = self.row['urls'] if len(ustr) == 0: return [] diff --git a/my/util/hpi_heartbeat.py b/my/util/hpi_heartbeat.py index 6dcac7e..84790a4 100644 --- a/my/util/hpi_heartbeat.py +++ b/my/util/hpi_heartbeat.py @@ -5,13 +5,12 @@ In particular the behaviour of import_original_module function The idea of testing is that overlays extend this module, and add their own items to items(), and the checker asserts all overlays have contributed. """ +from my.core import __NOT_HPI_MODULE__ -from my.core import __NOT_HPI_MODULE__ # isort: skip - -import sys -from collections.abc import Iterator from dataclasses import dataclass from datetime import datetime +import sys +from typing import Iterator, List NOW = datetime.now() @@ -20,10 +19,10 @@ NOW = datetime.now() class Item: dt: datetime message: str - path: list[str] + path: List[str] -def get_pkg_path() -> list[str]: +def get_pkg_path() -> List[str]: pkg = sys.modules[__package__] return list(pkg.__path__) diff --git a/my/vk/favorites.py b/my/vk/favorites.py index 5f278ff..eb1a89b 100644 --- a/my/vk/favorites.py +++ b/my/vk/favorites.py @@ -1,28 +1,29 @@ -# todo: uses my private export script?, timezone -from __future__ import annotations - +# todo: uses my private export script? +from datetime import datetime import json -from collections.abc import Iterable, Iterator -from dataclasses import dataclass -from datetime import datetime, timezone +from typing import NamedTuple, Iterable, Sequence, Optional + from my.config import vk as config # type: ignore[attr-defined] -from my.core import Json, Stats, datetime_aware, stat -from my.core.error import Res -@dataclass -class Favorite: - dt: datetime_aware +class Favorite(NamedTuple): + dt: datetime title: str - url: str | None + url: Optional[str] text: str +from ..core import Json +from ..core.error import Res + + skip = ( 'graffiti', 'poll', - 'note', # TODO could be useful.. + + # TODO could be useful.. + 'note', 'doc', 'audio', 'photo', @@ -31,11 +32,10 @@ skip = ( 'page', ) - def parse_fav(j: Json) -> Favorite: # TODO copy_history?? url = None - title = '' # TODO ??? + title = '' # TODO ??? atts = j.get('attachments', []) for a in atts: if any(k in a for k in skip): @@ -47,14 +47,14 @@ def parse_fav(j: Json) -> Favorite: # TODO would be nice to include user return Favorite( - dt=datetime.fromtimestamp(j['date'], tz=timezone.utc), + dt=datetime.utcfromtimestamp(j['date']), title=title, url=url, text=j['text'], ) -def _iter_favs() -> Iterator[Res]: +def _iter_favs() -> Iterable[Res]: jj = json.loads(config.favs_file.read_text()) for j in jj: try: @@ -65,7 +65,7 @@ def _iter_favs() -> Iterator[Res]: yield ex -def favorites() -> Iterable[Res]: +def favorites() -> Sequence[Res]: it = _iter_favs() # trick to sort errors along with the actual objects # TODO wonder if there is a shorter way? @@ -76,11 +76,12 @@ def favorites() -> Iterable[Res]: for i, f in enumerate(favs): if not isinstance(f, Exception): prev = f.dt - keys.append((prev, i)) # include index to resolve ties + keys.append((prev, i)) # include index to resolve ties sorted_items = [p[1] for p in sorted(zip(keys, favs))] # return sorted_items -def stats() -> Stats: +def stats(): + from ..core import stat return stat(favorites) diff --git a/my/vk/vk_messages_backup.py b/my/vk/vk_messages_backup.py index 4f593c8..1837385 100644 --- a/my/vk/vk_messages_backup.py +++ b/my/vk/vk_messages_backup.py @@ -2,17 +2,19 @@ VK data (exported by [[https://github.com/Totktonada/vk_messages_backup][Totktonada/vk_messages_backup]]) ''' # note: could reuse the original repo, but little point I guess since VK closed their API -import json -from collections.abc import Iterator -from dataclasses import dataclass from datetime import datetime +from dataclasses import dataclass +import json +from typing import Dict, Iterator import pytz -from my.config import vk_messages_backup as config -from my.core import Json, Res, Stats, datetime_aware, get_files, stat +from my.core import stat, Stats, Json, Res, datetime_aware, get_files from my.core.common import unique_everseen +from my.config import vk_messages_backup as config + + # I think vk_messages_backup used this tz? # not sure if vk actually used to return this tz in api? TZ = pytz.timezone('Europe/Moscow') @@ -20,7 +22,6 @@ TZ = pytz.timezone('Europe/Moscow') Uid = int - @dataclass(frozen=True) class User: id: Uid @@ -43,11 +44,9 @@ class Message: body: str -Users = dict[Uid, User] - - +Users = Dict[Uid, User] def users() -> Users: - files = get_files(config.storage_path, glob='user_*.json') + files = list(sorted(config.storage_path.glob('user_*.json'))) res = {} for f in files: j = json.loads(f.read_text()) @@ -61,8 +60,6 @@ def users() -> Users: GROUP_CHAT_MIN_ID = 2000000000 - - def _parse_chat(*, msg: Json, udict: Users) -> Chat: # exported with newer api, peer_id is a proper identifier both for users and chats peer_id = msg.get('peer_id') @@ -91,13 +88,13 @@ def _parse_chat(*, msg: Json, udict: Users) -> Chat: def _parse_msg(*, msg: Json, chat: Chat, udict: Users) -> Message: mid = msg['id'] - md = msg['date'] + md = msg['date'] dt = datetime.fromtimestamp(md, tz=TZ) # todo attachments? e.g. url could be an attachment # todo might be forwarded? - mb = msg.get('body') + mb = msg.get('body') if mb is None: mb = msg.get('text') assert mb is not None, msg @@ -106,7 +103,7 @@ def _parse_msg(*, msg: Json, chat: Chat, udict: Users) -> Message: if out: user = udict[config.user_id] else: - mu = msg.get('user_id') or msg.get('from_id') + mu = msg.get('user_id') or msg.get('from_id') assert mu is not None, msg user = udict[mu] return Message( @@ -121,7 +118,8 @@ def _parse_msg(*, msg: Json, chat: Chat, udict: Users) -> Message: def _messages() -> Iterator[Res[Message]]: udict = users() - uchats = get_files(config.storage_path, glob='userchat_*.json') + get_files(config.storage_path, glob='groupchat_*.json') + uchats = list(sorted(config.storage_path.glob('userchat_*.json' ))) + \ + list(sorted(config.storage_path.glob('groupchat_*.json'))) for f in uchats: j = json.loads(f.read_text()) # ugh. very annoying, sometimes not possible to extract title from last message diff --git a/my/whatsapp/android.py b/my/whatsapp/android.py index a8dbe8d..295d831 100644 --- a/my/whatsapp/android.py +++ b/my/whatsapp/android.py @@ -1,22 +1,20 @@ """ Whatsapp data from Android app database (in =/data/data/com.whatsapp/databases/msgstore.db=) """ - from __future__ import annotations -import sqlite3 -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from typing import Union +import sqlite3 +from typing import Sequence, Iterator, Optional -from my.core import Paths, Res, datetime_aware, get_files, make_config, make_logger +from my.core import get_files, Paths, datetime_aware, Res, make_logger, make_config from my.core.common import unique_everseen from my.core.error import echain, notnone from my.core.sqlite import sqlite_connection +import my.config -import my.config # isort: skip logger = make_logger(__name__) @@ -25,7 +23,7 @@ logger = make_logger(__name__) class Config(my.config.whatsapp.android): # paths[s]/glob to the exported sqlite databases export_path: Paths - my_user_id: str | None = None + my_user_id: Optional[str] = None config = make_config(Config) @@ -40,13 +38,13 @@ class Chat: id: str # todo not sure how to support renames? # could change Chat object itself, but this won't work well with incremental processing.. - name: str | None + name: Optional[str] @dataclass(unsafe_hash=True) class Sender: id: str - name: str | None + name: Optional[str] @dataclass(unsafe_hash=True) @@ -55,36 +53,17 @@ class Message: id: str dt: datetime_aware sender: Sender - text: str | None + text: Optional[str] -Entity = Union[Chat, Sender, Message] - - -def _process_db(db: sqlite3.Connection) -> Iterator[Entity]: +def _process_db(db: sqlite3.Connection): # TODO later, split out Chat/Sender objects separately to safe on object creation, similar to other android data sources - try: - db.execute('SELECT jid_row_id FROM chat_view') - except sqlite3.OperationalError as oe: - if 'jid_row_id' not in str(oe): - raise oe - new_version_202410 = False - else: - new_version_202410 = True - - if new_version_202410: - chat_id_col = 'jid.raw_string' - jid_join = 'JOIN jid ON jid._id == chat_view.jid_row_id' - else: - chat_id_col = 'chat_view.raw_string_jid' - jid_join = '' - chats = {} for r in db.execute( - f''' - SELECT {chat_id_col} AS chat_id, subject - FROM chat_view {jid_join} + ''' + SELECT raw_string_jid AS chat_id, subject + FROM chat_view WHERE chat_id IS NOT NULL /* seems that it might be null for chats that are 'recycled' (the db is more like an LRU cache) */ ''' ): @@ -94,7 +73,6 @@ def _process_db(db: sqlite3.Connection) -> Iterator[Entity]: id=chat_id, name=subject, ) - yield chat chats[chat.id] = chat senders = {} @@ -106,21 +84,19 @@ def _process_db(db: sqlite3.Connection) -> Iterator[Entity]: ): # TODO seems that msgstore.db doesn't have contact names # perhaps should extract from wa.db and match against wa_contacts.jid? - # TODO these can also be chats? not sure if need to include... s = Sender( id=r['raw_string'], name=None, ) - yield s senders[r['_id']] = s # NOTE: hmm, seems that message_view or available_message_view use lots of NULL as ... # so even if it seems as if it has a column (e.g. for attachment path), there is actually no such data # so makes more sense to just query message column directly for r in db.execute( - f''' + ''' SELECT - {chat_id_col} AS chat_id, + C.raw_string_jid AS chat_id, M.key_id, M.timestamp, sender_jid_row_id, M.from_me, @@ -129,9 +105,8 @@ def _process_db(db: sqlite3.Connection) -> Iterator[Entity]: MM.file_size, M.message_type FROM message AS M - LEFT JOIN chat_view ON M.chat_row_id = chat_view._id - {jid_join} - left JOIN message_media AS MM ON M._id = MM.message_row_id + LEFT JOIN chat_view AS C ON M.chat_row_id = C._id + LEFT JOIN message_media AS MM ON M._id = MM.message_row_id WHERE M.key_id != -1 /* key_id -1 is some sort of fake message where everything is null */ /* type 7 seems to be some dummy system message. sometimes contain chat name, but usually null, so ignore them @@ -145,9 +120,9 @@ def _process_db(db: sqlite3.Connection) -> Iterator[Entity]: ts: int = notnone(r['timestamp']) dt = datetime.fromtimestamp(ts / 1000, tz=timezone.utc) - text: str | None = r['text_data'] - media_file_path: str | None = r['file_path'] - media_file_size: int | None = r['file_size'] + text: Optional[str] = r['text_data'] + media_file_path: Optional[str] = r['file_path'] + media_file_size: Optional[int] = r['file_size'] message_type = r['message_type'] @@ -199,7 +174,7 @@ def _process_db(db: sqlite3.Connection) -> Iterator[Entity]: sender_row_id = r['sender_jid_row_id'] if sender_row_id == 0: # seems that it's always 0 for 1-1 chats - # for group chats our own id is still 0, but other ids are properly set + # for group chats our onw id is still 0, but other ids are properly set if from_me: myself_user_id = config.my_user_id or 'MYSELF_USER_ID' sender = Sender(id=myself_user_id, name=None) # TODO set my own name as well? @@ -212,7 +187,7 @@ def _process_db(db: sqlite3.Connection) -> Iterator[Entity]: yield m -def _entities() -> Iterator[Res[Entity]]: +def _messages() -> Iterator[Res[Message]]: paths = inputs() total = len(paths) width = len(str(total)) @@ -225,14 +200,5 @@ def _entities() -> Iterator[Res[Entity]]: yield echain(RuntimeError(f'While processing {path}'), cause=e) -def entities() -> Iterator[Res[Entity]]: - return unique_everseen(_entities) - - def messages() -> Iterator[Res[Message]]: - # TODO hmm, specify key=lambda m: m.id? - # not sure since might be useful to keep track of sender changes etc - # probably best not to, or maybe query messages/senders separately and merge later? - for e in entities(): - if isinstance(e, (Exception, Message)): - yield e + yield from unique_everseen(_messages) diff --git a/my/youtube/takeout.py b/my/youtube/takeout.py index 8eca328..79b4549 100644 --- a/my/youtube/takeout.py +++ b/my/youtube/takeout.py @@ -1,17 +1,13 @@ -from __future__ import annotations +from typing import NamedTuple, List, Iterable, TYPE_CHECKING -from collections.abc import Iterable, Iterator -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any - -from my.core import Res, Stats, datetime_aware, make_logger, stat, warnings -from my.core.compat import deprecated - -logger = make_logger(__name__) +from ..core import datetime_aware, Res, LazyLogger +from ..core.compat import removeprefix -@dataclass -class Watched: +logger = LazyLogger(__name__) + + +class Watched(NamedTuple): url: str title: str when: datetime_aware @@ -20,57 +16,19 @@ class Watched: def eid(self) -> str: return f'{self.url}-{self.when.isoformat()}' - def is_deleted(self) -> bool: - return self.title == self.url - # todo define error policy? # although it has one from google takeout module.. so not sure - -def watched() -> Iterator[Res[Watched]]: - emitted: dict[Any, Watched] = {} - for w in _watched(): - if isinstance(w, Exception): - yield w # TODO also make unique? - continue - - # older exports (e.g. html) didn't have microseconds - # whereas newer json ones do have them - # seconds resolution is enough to distinguish watched videos - # also we're processing takeouts in HPI in reverse order, so first seen watch would contain microseconds, resulting in better data - without_microsecond = w.when.replace(microsecond=0) - - key = w.url, without_microsecond - prev = emitted.get(key, None) - if prev is not None: - # NOTE: some video titles start with 'Liked ' for liked videos activity - # but they'd have different timestamp, so fine not to handle them as a special case here - if w.title in prev.title: - # often more stuff added to the title, like 'Official Video' - # in this case not worth emitting the change - # also handles the case when titles match - continue - # otherwise if title changed completely, just emit the change... not sure what else we could do? - # could merge titles in the 'titles' field and update dynamically? but a bit complicated, maybe later.. - - # TODO would also be nice to handle is_deleted here somehow... - # but for that would need to process data in direct order vs reversed.. - # not sure, maybe this could use a special mode or something? - - emitted[key] = w - yield w - - -def _watched() -> Iterator[Res[Watched]]: +def watched() -> Iterable[Res[Watched]]: try: - from google_takeout_parser.models import Activity - from ..google.takeout.parser import events + from google_takeout_parser.models import Activity except ModuleNotFoundError as ex: logger.exception(ex) - warnings.high("Please set up my.google.takeout.parser module for better youtube support. Falling back to legacy implementation.") - yield from _watched_legacy() # type: ignore[name-defined] + from ..core.warnings import high + high("Please set up my.google.takeout.parser module for better youtube support. Falling back to legacy implementation.") + yield from _watched_legacy() return YOUTUBE_VIDEO_LINK = '://www.youtube.com/watch?v=' @@ -79,18 +37,17 @@ def _watched() -> Iterator[Res[Watched]]: for e in events(): if isinstance(e, Exception): yield e - continue if not isinstance(e, Activity): continue url = e.titleUrl + header = e.header + title = e.title if url is None: continue - header = e.header - if header in {'Image Search', 'Search', 'Chrome'}: # sometimes results in youtube links.. but definitely not watch history continue @@ -103,8 +60,6 @@ def _watched() -> Iterator[Res[Watched]]: pass continue - title = e.title - if header == 'youtube.com' and title.startswith('Visited '): continue @@ -118,34 +73,18 @@ def _watched() -> Iterator[Res[Watched]]: # all titles contain it, so pointless to include 'Watched ' # also compatible with legacy titles - title = title.removeprefix('Watched ') - - # watches originating from some activity end with this, remove it for consistency - title = title.removesuffix(' - YouTube') + title = removeprefix(title, 'Watched ') if YOUTUBE_VIDEO_LINK not in url: - if 'youtube.com/post/' in url: - # some sort of channel updates? + if e.details == ['From Google Ads']: + # weird, sometimes results in odd continue - if 'youtube.com/playlist' in url: - # 'saved playlist' actions - continue - if 'music.youtube.com' in url: - # todo maybe allow it? - continue - if any('From Google Ads' in d for d in e.details): - # weird, sometimes results in odd urls - continue - - if title == 'Used YouTube': + if title == 'Used YouTube' and e.products == ['Android']: continue yield RuntimeError(f'Unexpected url: {e}') continue - # TODO contribute to takeout parser? seems that these still might happen in json data - title = title.replace("\xa0", " ") - yield Watched( url=url, title=title, @@ -153,6 +92,7 @@ def _watched() -> Iterator[Res[Watched]]: ) +from ..core import stat, Stats def stats() -> Stats: return stat(watched) @@ -160,24 +100,23 @@ def stats() -> Stats: ### deprecated stuff (keep in my.media.youtube) if not TYPE_CHECKING: + # "deprecate" by hiding from mypy + get_watched = watched - @deprecated("use 'watched' instead") - def get_watched(*args, **kwargs): - return watched(*args, **kwargs) - def _watched_legacy() -> Iterable[Watched]: - from ..google.takeout.html import read_html - from ..google.takeout.paths import get_last_takeout +def _watched_legacy() -> Iterable[Watched]: + from ..google.takeout.html import read_html + from ..google.takeout.paths import get_last_takeout - # todo looks like this one doesn't have retention? so enough to use the last - path = 'Takeout/My Activity/YouTube/MyActivity.html' - last = get_last_takeout(path=path) - if last is None: - return [] + # todo looks like this one doesn't have retention? so enough to use the last + path = 'Takeout/My Activity/YouTube/MyActivity.html' + last = get_last_takeout(path=path) + if last is None: + return [] - watches: list[Watched] = [] - for dt, url, title in read_html(last, path): - watches.append(Watched(url=url, title=title, when=dt)) + watches: List[Watched] = [] + for dt, url, title in read_html(last, path): + watches.append(Watched(url=url, title=title, when=dt)) - # todo hmm they already come sorted.. wonder if should just rely on it.. - return sorted(watches, key=lambda e: e.when) + # todo hmm they already come sorted.. wonder if should just rely on it.. + return list(sorted(watches, key=lambda e: e.when)) diff --git a/my/zotero.py b/my/zotero.py index 8eb34ba..4440aae 100644 --- a/my/zotero.py +++ b/my/zotero.py @@ -1,16 +1,14 @@ -from __future__ import annotations as _annotations - -import json -import sqlite3 -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime, timezone +import json +from typing import Iterator, Optional, Dict, Any, Sequence from pathlib import Path -from typing import Any +import sqlite3 -from my.core import Res, datetime_aware, make_logger +from my.core import make_logger, Res, datetime_aware from my.core.sqlite import sqlite_copy_and_open + logger = make_logger(__name__) @@ -28,7 +26,7 @@ class Item: """Corresponds to 'Zotero item'""" file: Path title: str - url: Url | None + url: Optional[Url] tags: Sequence[str] @@ -41,8 +39,8 @@ class Annotation: page: int """0-indexed""" - text: str | None - comment: str | None + text: Optional[str] + comment: Optional[str] tags: Sequence[str] color_hex: str """Original hex-encoded color in zotero""" @@ -99,7 +97,7 @@ WHERE ID.fieldID = 13 AND IA.itemID = ? # TODO maybe exclude 'private' methods from detection? -def _query_raw() -> Iterator[Res[dict[str, Any]]]: +def _query_raw() -> Iterator[Res[Dict[str, Any]]]: [db] = inputs() with sqlite_copy_and_open(db) as conn: @@ -159,7 +157,7 @@ def _hex2human(color_hex: str) -> str: }.get(color_hex, color_hex) -def _parse_annotation(r: dict) -> Annotation: +def _parse_annotation(r: Dict) -> Annotation: text = r['text'] comment = r['comment'] # todo use json query for this? diff --git a/my/zulip/organization.py b/my/zulip/organization.py index d0cfcb7..8725411 100644 --- a/my/zulip/organization.py +++ b/my/zulip/organization.py @@ -1,55 +1,38 @@ """ Zulip data from [[https://memex.zulipchat.com/help/export-your-organization][Organization export]] """ - -from __future__ import annotations - -import json -from abc import abstractmethod -from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime, timezone from itertools import count +import json from pathlib import Path +from typing import Sequence, Iterator, Dict, Union from my.core import ( + assert_never, + datetime_aware, + get_files, + stat, Json, Paths, Res, Stats, - assert_never, - datetime_aware, - get_files, - make_logger, - stat, - warnings, ) - -logger = make_logger(__name__) +from my.core.error import notnone +import my.config -class config: - @property - @abstractmethod - def export_path(self) -> Paths: - """paths[s]/glob to the exported JSON data""" - raise NotImplementedError - - -def make_config() -> config: - from my.config import zulip as user_config - - class combined_config(user_config.organization, config): - pass - - return combined_config() +@dataclass +class organization(my.config.zulip.organization): + # paths[s]/glob to the exported JSON data + export_path: Paths def inputs() -> Sequence[Path]: # TODO: seems like export ids are kinda random.. # not sure what's the best way to figure out the last without renaming? # could use mtime perhaps? - return get_files(make_config().export_path, sort=False) + return get_files(organization.export_path, sort=False) @dataclass(frozen=True) @@ -102,39 +85,19 @@ class Message: # todo cache it -def _entities() -> Iterator[Res[Server | Sender | _Message]]: +def _entities() -> Iterator[Res[Union[Server, Sender, _Message]]]: last = max(inputs()) - logger.info(f'extracting data from {last}') + # todo would be nice to switch it to unpacked dirs as well, similar to ZipPath + # I guess makes sense to have a special implementation for .tar.gz considering how common are they + import tarfile - root: Path | None = None + tfile = tarfile.open(last) - if last.is_dir(): # if it's already CPath, this will match it - root = last - else: - try: - from kompress import CPath + subdir = tfile.getnames()[0] # there is a directory inside tar file, first name should be that - root = CPath(last) - assert len(list(root.iterdir())) > 0 # trigger to check if we have the kompress version with targz support - except Exception as e: - logger.exception(e) - warnings.high("Upgrade 'kompress' to latest version with native .tar.gz support. Falling back to unpacking to tmp dir.") - - if root is None: - from my.core.structure import match_structure - - with match_structure(last, expected=()) as res: # expected=() matches it regardless any patterns - [root] = res - yield from _process_one(root) - else: - yield from _process_one(root) - - -def _process_one(root: Path) -> Iterator[Res[Server | Sender | _Message]]: - [subdir] = root.iterdir() # there is a directory inside tar file, first name should be that - - rj = json.loads((subdir / 'realm.json').read_text()) + with notnone(tfile.extractfile(f'{subdir}/realm.json')) as fo: + rj = json.load(fo) [sj] = rj['zerver_realm'] server = Server( @@ -173,10 +136,12 @@ def _process_one(root: Path) -> Iterator[Res[Server | Sender | _Message]]: for idx in count(start=1, step=1): fname = f'messages-{idx:06}.json' - fpath = subdir / fname - if not fpath.exists(): + fpath = f'{subdir}/{fname}' + if fpath not in tfile.getnames(): + # tarfile doesn't have .exists? break - mj = json.loads(fpath.read_text()) + with notnone(tfile.extractfile(fpath)) as fo: + mj = json.load(fo) # TODO handle zerver_usermessage for j in mj['zerver_message']: try: @@ -186,8 +151,8 @@ def _process_one(root: Path) -> Iterator[Res[Server | Sender | _Message]]: def messages() -> Iterator[Res[Message]]: - id2sender: dict[int, Sender] = {} - id2server: dict[int, Server] = {} + id2sender: Dict[int, Sender] = {} + id2server: Dict[int, Server] = {} for x in _entities(): if isinstance(x, Exception): yield x diff --git a/mypy.ini b/mypy.ini index 9c34fcc..ebc81a5 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,13 +1,18 @@ [mypy] +namespace_packages = True pretty = True show_error_context = True +show_error_codes = True show_column_numbers = True show_error_end = True -warn_redundant_casts = True warn_unused_ignores = True check_untyped_defs = True -strict_equality = True enable_error_code = possibly-undefined +strict_equality = True + +# a bit annoying, it has optional ipython import which should be ignored in mypy-core configuration.. +[mypy-my.core.__main__] +warn_unused_ignores = False # todo ok, maybe it wasn't such a good idea.. # mainly because then tox picks it up and running against the user config, not the repository config diff --git a/ruff.toml b/ruff.toml index 3d803e7..0be93e0 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,55 +1,4 @@ -target-version = "py39" # NOTE: inferred from pyproject.toml if present - -lint.extend-select = [ - "F", # flakes rules -- default, but extend just in case - "E", # pycodestyle -- default, but extend just in case - "W", # various warnings - - "B", # 'bugbear' set -- various possible bugs - "C4", # flake8-comprehensions -- unnecessary list/map/dict calls - "COM", # trailing commas - "EXE", # various checks wrt executable files - # "I", # sort imports - "ICN", # various import conventions - "FBT", # detect use of boolean arguments - "FURB", # various rules - "PERF", # various potential performance speedups - "PD", # pandas rules - "PIE", # 'misc' lints - "PLC", # pylint convention rules - "PLR", # pylint refactor rules - "PLW", # pylint warnings - "PT", # pytest stuff - "PYI", # various type hinting rules - "RET", # early returns - "RUF", # various ruff-specific rules - "TID", # various imports suggestions - "TRY", # various exception handling rules - "UP", # detect deprecated python stdlib stuff - "FA", # suggest using from __future__ import annotations - "PTH", # pathlib migration - "ARG", # unused argument checks - # "A", # builtin shadowing -- TODO handle later - # "EM", # TODO hmm could be helpful to prevent duplicate err msg in traceback.. but kinda annoying - - # "ALL", # uncomment this to check for new rules! -] - -# Preserve types, even if a file imports `from __future__ import annotations` -# we need this for cachew to work with HPI types on 3.9 -# can probably remove after 3.10? -lint.pyupgrade.keep-runtime-typing = true - -lint.ignore = [ - "D", # annoying nags about docstrings - "N", # pep naming - "TCH", # type checking rules, mostly just suggests moving imports under TYPE_CHECKING - "S", # bandit (security checks) -- tends to be not very useful, lots of nitpicks - "DTZ", # datetimes checks -- complaining about missing tz and mostly false positives - "FIX", # complains about fixmes/todos -- annoying - "TD", # complains about todo formatting -- too annoying - "ANN", # missing type annotations? seems way to strict though - +ignore = [ ### too opinionated style checks "E501", # too long lines "E702", # Multiple statements on one line (semicolon) @@ -68,84 +17,9 @@ lint.ignore = [ "E402", # Module level import not at top of file ### maybe consider these soon - # sometimes it's useful to give a variable a name even if we don't use it as a documentation - # on the other hand, often is a sign of error +# sometimes it's useful to give a variable a name even if we don't use it as a documentation +# on the other hand, often is a sign of error "F841", # Local variable `count` is assigned to but never used + "F401", # imported but unused ### - - "RUF100", # unused noqa -- handle later - "RUF012", # mutable class attrs should be annotated with ClassVar... ugh pretty annoying for user configs - -### these are just nitpicky, we usually know better - "PLR0911", # too many return statements - "PLR0912", # too many branches - "PLR0913", # too many function arguments - "PLR0915", # too many statements - "PLR1714", # consider merging multiple comparisons - "PLR2044", # line with empty comment - "PLR5501", # use elif instead of else if - "PLR2004", # magic value in comparison -- super annoying in tests -### - "PLR0402", # import X.Y as Y -- TODO maybe consider enabling it, but double check - - "B009", # calling gettattr with constant attribute -- this is useful to convince mypy - "B010", # same as above, but setattr - "B011", # complains about assert False - "B017", # pytest.raises(Exception) - "B023", # seems to result in false positives? - "B028", # suggest using explicit stacklevel? TODO double check later, but not sure it's useful - - # complains about useless pass, but has sort of a false positive if the function has a docstring? - # this is common for click entrypoints (e.g. in __main__), so disable - "PIE790", - - # a bit too annoying, offers to convert for loops to list comprehension - # , which may heart readability - "PERF401", - - # suggests no using exception in for loops - # we do use this technique a lot, plus in 3.11 happy path exception handling is "zero-cost" - "PERF203", - - "RET504", # unnecessary assignment before returning -- that can be useful for readability - "RET505", # unnecessary else after return -- can hurt readability - - "PLW0603", # global variable update.. we usually know why we are doing this - "PLW2901", # for loop variable overwritten, usually this is intentional - - "PT004", # deprecated rule, will be removed later - "PT011", # pytest raises should is too broad - "PT012", # pytest raises should contain a single statement - - "COM812", # trailing comma missing -- mostly just being annoying with long multiline strings - - "PD901", # generic variable name df - - "TRY003", # suggests defining exception messages in exception class -- kinda annoying - "TRY004", # prefer TypeError -- don't see the point - "TRY201", # raise without specifying exception name -- sometimes hurts readability - "TRY400", # TODO double check this, might be useful - "TRY401", # redundant exception in logging.exception call? TODO double check, might result in excessive logging - - "PGH", # TODO force error code in mypy instead - - "TID252", # Prefer absolute imports over relative imports from parent modules - - "UP038", # suggests using | (union) in isisntance checks.. but it results in slower code - - ## too annoying - "T20", # just complains about prints and pprints - "Q", # flake quotes, too annoying - "C90", # some complexity checking - "G004", # logging statement uses f string - "ERA001", # commented out code - "SLF001", # private member accessed - "BLE001", # do not catch 'blind' Exception - "INP001", # complains about implicit namespace packages - "SIM", # some if statements crap - "RSE102", # complains about missing parens in exceptions - ## - - "ARG001", # ugh, kinda annoying when using pytest fixtures - "F401" , # TODO nice to have, but annoying with NOT_HPI_MODULE thing ] diff --git a/setup.py b/setup.py index 385c810..42ffeaa 100644 --- a/setup.py +++ b/setup.py @@ -4,13 +4,12 @@ from setuptools import setup, find_namespace_packages # type: ignore INSTALL_REQUIRES = [ - 'pytz' , # even though it's not needed by the core, it's so common anyway... - 'typing-extensions' , # one of the most common pypi packages, ok to depend for core - 'appdirs' , # very common, and makes it portable - 'more-itertools' , # it's just too useful and very common anyway - 'decorator' , # less pain in writing correct decorators. very mature and stable, so worth keeping in core - 'click>=8.1' , # for the CLI, printing colors, decorator-based - may allow extensions to CLI - 'kompress>=0.2.20240918' , # for transparent access to compressed files via pathlib.Path + 'pytz', # even though it's not needed by the core, it's so common anyway... + 'appdirs', # very common, and makes it portable + 'more-itertools', # it's just too useful and very common anyway + 'decorator' , # less pain in writing correct decorators. very mature and stable, so worth keeping in core + 'click>=8.1' , # for the CLI, printing colors, decorator-based - may allow extensions to CLI + 'kompress' , # for transparent access to compressed files via pathlib.Path ] @@ -44,7 +43,7 @@ def main() -> None: author_email='karlicoss@gmail.com', description='A Python interface to my life', - python_requires='>=3.9', + python_requires='>=3.8', install_requires=INSTALL_REQUIRES, extras_require={ 'testing': [ @@ -55,25 +54,12 @@ def main() -> None: # used in some tests.. although shouldn't rely on it 'pandas', - - 'orjson', # for my.core.serialize and denylist - 'simplejson', # for my.core.serialize - - ## - # ideally we'd use --instal-types in mypy - # , but looks like it doesn't respect uv venv if it's running in it :( - 'types-pytz' , # for my.core - 'types-decorator' , # for my.core.compat - 'pandas-stubs' , # for my.core.pandas - 'types-dateparser', # for my.core.query_range - 'types-simplejson', # for my.core.serialize - ## ], 'optional': [ # todo document these? 'orjson', # for my.core.serialize 'pyfzf_iter', # for my.core.denylist - 'cachew>=0.15.20231019 ', + 'cachew>=0.8.0', 'mypy', # used for config checks 'colorlog', # for colored logs 'enlighten', # for CLI progress bars diff --git a/my/tests/bluemaestro.py b/tests/bluemaestro.py similarity index 78% rename from my/tests/bluemaestro.py rename to tests/bluemaestro.py index d139a8f..84d3eb0 100644 --- a/my/tests/bluemaestro.py +++ b/tests/bluemaestro.py @@ -1,15 +1,19 @@ -from collections.abc import Iterator +from pathlib import Path +from typing import TYPE_CHECKING, Iterator, Any -import pytest from more_itertools import one -from my.bluemaestro import Measurement, measurements -from my.core.cfg import tmp_config +import pytest -from .common import testdata + +if TYPE_CHECKING: + from my.bluemaestro import Measurement +else: + Measurement = Any def ok_measurements() -> Iterator[Measurement]: + from my.bluemaestro import measurements for m in measurements(): assert not isinstance(m, Exception) yield m @@ -26,7 +30,7 @@ def test() -> None: # check that timezone is set properly assert dts == '20200824 22' - assert len(tp) == 1 # should be unique + assert len(tp) == 1 # should be unique # 2.5 K + 4 K datapoints, somewhat overlapping assert len(res2020) < 6000 @@ -46,12 +50,14 @@ def test_old_db() -> None: @pytest.fixture(autouse=True) def prepare(): + from my.tests.common import testdata bmdata = testdata() / 'hpi-testdata' / 'bluemaestro' assert bmdata.exists(), bmdata class bluemaestro: export_path = bmdata + from my.core.cfg import tmp_config with tmp_config() as config: config.bluemaestro = bluemaestro yield diff --git a/tests/calendar.py b/tests/calendar.py new file mode 100644 index 0000000..3435da3 --- /dev/null +++ b/tests/calendar.py @@ -0,0 +1,19 @@ +from pathlib import Path + +import pytest + +from my.calendar.holidays import is_holiday + + +def test() -> None: + assert is_holiday('20190101') + assert not is_holiday('20180601') + assert is_holiday('20200906') # national holiday in Bulgaria + + +@pytest.fixture(autouse=True) +def prepare(tmp_path: Path): + from . import tz + # todo meh. fixtures can't be called directly? + orig = tz.prepare.__wrapped__ # type: ignore + yield from orig(tmp_path) diff --git a/tests/config.py b/tests/config.py new file mode 100644 index 0000000..101f7df --- /dev/null +++ b/tests/config.py @@ -0,0 +1,126 @@ +from pathlib import Path + + +def test_dynamic_configuration(notes: Path) -> None: + import pytz + from types import SimpleNamespace as NS + + from my.core.cfg import tmp_config + with tmp_config() as C: + C.orgmode = NS(paths=[notes]) + # TODO ugh. this belongs to tz provider or global config or something + C.weight = NS(default_timezone=pytz.timezone('Europe/London')) + + from my.body.weight import from_orgmode + weights = [0.0 if isinstance(x, Exception) else x.value for x in from_orgmode()] + + assert weights == [ + 0.0, + 62.0, + 0.0, + 61.0, + 62.0, + 0.0, + ] + +import pytest + + +def test_environment_variable(tmp_path: Path) -> None: + cfg_dir = tmp_path / 'my' + cfg_file = cfg_dir / 'config.py' + cfg_dir.mkdir() + cfg_file.write_text(''' +class feedly: + pass +class just_for_test: + pass +''') + + import os + oenv = dict(os.environ) + try: + os.environ['MY_CONFIG'] = str(tmp_path) + # should not raise at least + import my.rss.feedly + + import my.config as c + assert hasattr(c, 'just_for_test') + finally: + os.environ.clear() + os.environ.update(oenv) + + import sys + # TODO wtf??? doesn't work without unlink... is it caching something? + cfg_file.unlink() + del sys.modules['my.config'] # meh.. + + import my.config as c + assert not hasattr(c, 'just_for_test') + + +from dataclasses import dataclass + + +def test_user_config() -> None: + from my.core.common import classproperty + class user_config: + param1 = 'abacaba' + # TODO fuck. properties don't work here??? + @classproperty + def param2(cls) -> int: + return 456 + + extra = 'extra!' + + @dataclass + class test_config(user_config): + param1: str + param2: int # type: ignore[assignment] # TODO need to figure out how to trick mypy for @classproperty + param3: str = 'default' + + assert test_config.param1 == 'abacaba' + assert test_config.param2 == 456 + assert test_config.param3 == 'default' + assert test_config.extra == 'extra!' + + from my.core.cfg import make_config + c = make_config(test_config) + assert c.param1 == 'abacaba' + assert c.param2 == 456 + assert c.param3 == 'default' + assert c.extra == 'extra!' + + +@pytest.fixture +def notes(tmp_path: Path): + ndir = tmp_path / 'notes' + ndir.mkdir() + logs = ndir / 'logs.org' + logs.write_text(''' +#+TITLE: Stuff I'm logging + +* Weight (org-capture) :weight: +** [2020-05-01 Fri 09:00] 62 +** 63 + this should be ignored, got no timestamp +** [2020-05-03 Sun 08:00] 61 +** [2020-05-04 Mon 10:00] 62 + ''') + misc = ndir / 'misc.org' + misc.write_text(''' +Some misc stuff + +* unrelated note :weight:whatever: + ''') + try: + yield ndir + finally: + pass + + +@pytest.fixture(autouse=True) +def prepare(): + from my.tests.common import reset_modules + reset_modules() + yield diff --git a/tests/demo.py b/tests/demo.py new file mode 100644 index 0000000..73a6c65 --- /dev/null +++ b/tests/demo.py @@ -0,0 +1,118 @@ +import sys +from pathlib import Path +from more_itertools import ilen + +# TODO NOTE: this wouldn't work because of an early my.config.demo import +# from my.demo import items + +def test_dynamic_config_1(tmp_path: Path) -> None: + import my.config + + class user_config: + username = 'user' + data_path = f'{tmp_path}/*.json' + external = f'{tmp_path}/external' + my.config.demo = user_config # type: ignore[misc, assignment] + + from my.demo import items + [item1, item2] = items() + assert item1.username == 'user' + + +# exactly the same test, but using a different config, to test out the behaviour w.r.t. import order +def test_dynamic_config_2(tmp_path: Path) -> None: + # doesn't work without it! + # because the config from test_dybamic_config_1 is cached in my.demo.demo + del sys.modules['my.demo'] + + import my.config + + class user_config: + username = 'user2' + data_path = f'{tmp_path}/*.json' + external = f'{tmp_path}/external' + my.config.demo = user_config # type: ignore[misc, assignment] + + from my.demo import items + [item1, item2] = items() + assert item1.username == 'user2' + + +import pytest + +@pytest.mark.skip(reason="won't work at the moment because of inheritance") +def test_dynamic_config_simplenamespace(tmp_path: Path) -> None: + # doesn't work without it! + # because the config from test_dybamic_config_1 is cached in my.demo.demo + del sys.modules['my.demo'] + + import my.config + from types import SimpleNamespace + + user_config = SimpleNamespace( + username='user3', + data_path=f'{tmp_path}/*.json', + ) + my.config.demo = user_config # type: ignore[misc, assignment] + + from my.demo import config + assert config.username == 'user3' + + +# make sure our config handling pattern does it as expected +def test_attribute_handling(tmp_path: Path) -> None: + # doesn't work without it! + # because the config from test_dybamic_config_1 is cached in my.demo.demo + del sys.modules['my.demo'] + + import pytz + nytz = pytz.timezone('America/New_York') + + import my.config + class user_config: + # check that override is taken into the account + timezone = nytz + + irrelevant = 'hello' + + username = 'UUU' + data_path = f'{tmp_path}/*.json' + external = f'{tmp_path}/external' + + + my.config.demo = user_config # type: ignore[misc, assignment] + + from my.demo import config + + assert config.username == 'UUU' + + # mypy doesn't know about it, but the attribute is there + assert getattr(config, 'irrelevant') == 'hello' + + # check that overridden default attribute is actually getting overridden + assert config.timezone == nytz + + + +@pytest.fixture(autouse=True) +def prepare(tmp_path: Path): + (tmp_path / 'data.json').write_text(''' +[ + {"key1": 1}, + {"key2": 2} +] +''') + ext = tmp_path / 'external' + ext.mkdir() + (ext / '__init__.py').write_text(''' +def identity(x): + from .submodule import hello + hello(x) + return x + +''') + (ext / 'submodule.py').write_text('hello = lambda x: print("hello " + str(x))') + yield + ex = 'my.config.repos.external' + if ex in sys.modules: + del sys.modules[ex] diff --git a/tests/extra/polar.py b/tests/extra/polar.py index b5858b6..b2bc562 100644 --- a/tests/extra/polar.py +++ b/tests/extra/polar.py @@ -15,7 +15,7 @@ def test_hpi(prepare: str) -> None: assert len(list(get_entries())) > 1 def test_orger(prepare: str, tmp_path: Path) -> None: - from my.core.utils.imports import import_from, import_file + from my.core.common import import_from, import_file om = import_file(ROOT / 'orger/modules/polar.py') # reload(om) diff --git a/tests/github.py b/tests/github.py index ed89053..6b7df23 100644 --- a/tests/github.py +++ b/tests/github.py @@ -5,13 +5,11 @@ from more_itertools import ilen def test_gdpr() -> None: import my.github.gdpr as gdpr - assert ilen(gdpr.events()) > 100 def test() -> None: - from my.github.all import get_events - + from my.coding.github import get_events events = get_events() assert ilen(events) > 100 for e in events: diff --git a/tests/location.py b/tests/location.py new file mode 100644 index 0000000..2597d5e --- /dev/null +++ b/tests/location.py @@ -0,0 +1,28 @@ +from pathlib import Path + +import pytest + + +def test() -> None: + from my.location.google import locations + locs = list(locations()) + assert len(locs) == 3810 + + last = locs[-1] + assert last.dt.strftime('%Y%m%d %H:%M:%S') == '20170802 13:01:56' # should be utc + # todo approx + assert last.lat == 46.5515350 + assert last.lon == 16.4742742 + # todo check altitude + + +@pytest.fixture(autouse=True) +def prepare(tmp_path: Path): + from .shared_config import temp_config + user_config = temp_config(tmp_path) + + import my.core.cfg as C + with C.tmp_config() as config: + config.google = user_config.google + yield + diff --git a/my/tests/location/fallback.py b/tests/location_fallback.py similarity index 85% rename from my/tests/location/fallback.py rename to tests/location_fallback.py index c09b902..aad33ee 100644 --- a/my/tests/location/fallback.py +++ b/tests/location_fallback.py @@ -2,23 +2,32 @@ To test my.location.fallback_location.all """ -from collections.abc import Iterator -from datetime import datetime, timedelta, timezone +from typing import Iterator +from datetime import datetime, timezone, timedelta -import pytest from more_itertools import ilen -import my.ip.all as ip_module from my.ip.common import IP + +def data() -> Iterator[IP]: + # random IP addresses + yield IP(addr="67.98.113.0", dt=datetime(2020, 1, 1, 12, 0, 0, tzinfo=timezone.utc)) + yield IP(addr="67.98.112.0", dt=datetime(2020, 1, 15, 12, 0, 0, tzinfo=timezone.utc)) + yield IP(addr="59.40.113.87", dt=datetime(2020, 2, 1, 12, 0, 0, tzinfo=timezone.utc)) + yield IP(addr="59.40.139.87", dt=datetime(2020, 2, 1, 16, 0, 0, tzinfo=timezone.utc)) + yield IP(addr="161.235.192.228", dt=datetime(2020, 3, 1, 12, 0, 0, tzinfo=timezone.utc)) + +# redefine the my.ip.all function using data for testing +import my.ip.all as ip_module +ip_module.ips = data + from my.location.fallback import via_ip -from ..shared_tz_config import config # autoused fixture - - # these are all tests for the bisect algorithm defined in via_ip.py # to make sure we can correctly find IPs that are within the 'for_duration' of a given datetime + def test_ip_fallback() -> None: - # precondition, make sure that the data override works + # make sure that the data override works assert ilen(ip_module.ips()) == ilen(data()) assert ilen(ip_module.ips()) == ilen(via_ip.fallback_locations()) assert ilen(via_ip.fallback_locations()) == 5 @@ -38,9 +47,7 @@ def test_ip_fallback() -> None: assert len(est) == 1 # right after the 'for_duration' for an IP - est = list( - via_ip.estimate_location(datetime(2020, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + via_ip.config.for_duration + timedelta(seconds=1)) - ) + est = list(via_ip.estimate_location(datetime(2020, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + via_ip.config.for_duration + timedelta(seconds=1))) assert len(est) == 0 # on 2/1/2020, threes one IP if before 16:30 @@ -68,8 +75,8 @@ def test_ip_fallback() -> None: # # redefine fallback_estimators to prevent possible namespace packages the user # may have installed from having side effects testing this - from my.location.fallback import all, via_home - + from my.location.fallback import all + from my.location.fallback import via_home def _fe() -> Iterator[all.LocationEstimator]: yield via_ip.estimate_location yield via_home.estimate_location @@ -81,7 +88,6 @@ def test_ip_fallback() -> None: # # just passing via_ip should give one IP from my.location.fallback.common import _iter_estimate_from - raw_est = list(_iter_estimate_from(use_dt, (via_ip.estimate_location,))) assert len(raw_est) == 1 assert raw_est[0].datasource == "via_ip" @@ -104,7 +110,7 @@ def test_ip_fallback() -> None: # should have used the IP from via_ip since it was more accurate assert all_est.datasource == "via_ip" - # test that a home defined in shared_tz_config.py is used if no IP is found + # test that a home defined in shared_config.py is used if no IP is found loc = all.estimate_location(datetime(2021, 1, 1, 12, 30, 0, tzinfo=timezone.utc)) assert loc.datasource == "via_home" @@ -115,21 +121,5 @@ def test_ip_fallback() -> None: assert (loc.lat, loc.lon) != (bulgaria.lat, bulgaria.lon) -def data() -> Iterator[IP]: - # random IP addresses - yield IP(addr="67.98.113.0", dt=datetime(2020, 1, 1, 12, 0, 0, tzinfo=timezone.utc)) - yield IP(addr="67.98.112.0", dt=datetime(2020, 1, 15, 12, 0, 0, tzinfo=timezone.utc)) - yield IP(addr="59.40.113.87", dt=datetime(2020, 2, 1, 12, 0, 0, tzinfo=timezone.utc)) - yield IP(addr="59.40.139.87", dt=datetime(2020, 2, 1, 16, 0, 0, tzinfo=timezone.utc)) - yield IP(addr="161.235.192.228", dt=datetime(2020, 3, 1, 12, 0, 0, tzinfo=timezone.utc)) - - -@pytest.fixture(autouse=True) -def prepare(config): - before = ip_module.ips - # redefine the my.ip.all function using data for testing - ip_module.ips = data - try: - yield - finally: - ip_module.ips = before +# re-use prepare fixture for overriding config from shared_config.py +from .tz import prepare diff --git a/tests/misc.py b/tests/misc.py new file mode 100644 index 0000000..7e666d7 --- /dev/null +++ b/tests/misc.py @@ -0,0 +1,94 @@ +from typing import Iterable, List +import warnings +from my.core import warn_if_empty + + +def test_warn_if_empty() -> None: + @warn_if_empty + def nonempty() -> Iterable[str]: + yield 'a' + yield 'aba' + + @warn_if_empty + def empty() -> List[int]: + return [] + + # should be rejected by mypy! + # todo how to actually test it? + # @warn_if_empty + # def baad() -> float: + # return 0.00 + + # reveal_type(nonempty) + # reveal_type(empty) + + with warnings.catch_warnings(record=True) as w: + assert list(nonempty()) == ['a', 'aba'] + assert len(w) == 0 + + eee = empty() + assert eee == [] + assert len(w) == 1 + + +def test_warn_iterable() -> None: + from my.core.common import _warn_iterable + i1: List[str] = ['a', 'b'] + i2: Iterable[int] = iter([1, 2, 3]) + # reveal_type(i1) + # reveal_type(i2) + x1 = _warn_iterable(i1) + x2 = _warn_iterable(i2) + # vvvv this should be flagged by mypy + # _warn_iterable(123) + # reveal_type(x1) + # reveal_type(x2) + with warnings.catch_warnings(record=True) as w: + assert x1 is i1 # should be unchanged! + assert len(w) == 0 + + assert list(x2) == [1, 2, 3] + assert len(w) == 0 + + +def test_cachew() -> None: + from cachew import settings + settings.ENABLE = True # by default it's off in tests (see conftest.py) + + from my.core.cachew import cache_dir + from my.core.common import mcachew + + called = 0 + # FIXME ugh. need doublewrap or something + @mcachew() + def cf() -> List[int]: + nonlocal called + called += 1 + return [1, 2, 3] + + list(cf()) + cc = called + # todo ugh. how to clean cache? + # assert called == 1 # precondition, to avoid turdes from previous tests + + assert list(cf()) == [1, 2, 3] + assert called == cc + + +def test_cachew_dir_none() -> None: + from cachew import settings + settings.ENABLE = True # by default it's off in tests (see conftest.py) + + from my.core.cachew import cache_dir + from my.core.common import mcachew + from my.core.core_config import _reset_config as reset + with reset() as cc: + cc.cache_dir = None + called = 0 + @mcachew(cache_path=cache_dir() / 'ctest') + def cf() -> List[int]: + nonlocal called + called += 1 + return [called, called, called] + assert list(cf()) == [1, 1, 1] + assert list(cf()) == [2, 2, 2] diff --git a/tests/orgmode.py b/tests/orgmode.py index 9b5cc59..37d783e 100644 --- a/tests/orgmode.py +++ b/tests/orgmode.py @@ -1,9 +1,10 @@ from my.tests.common import skip_if_not_karlicoss as pytestmark -def test() -> None: - from my import orgmode - from my.core.orgmode import collect +from my import orgmode +from my.core.orgmode import collect + +def test() -> None: # meh results = list(orgmode.query().collect_all(lambda n: [n] if 'python' in n.tags else [])) assert len(results) > 5 diff --git a/my/tests/pdfs.py b/tests/pdfs.py similarity index 77% rename from my/tests/pdfs.py rename to tests/pdfs.py index 3702424..63b1319 100644 --- a/my/tests/pdfs.py +++ b/tests/pdfs.py @@ -1,15 +1,17 @@ -import inspect from pathlib import Path -import pytest from more_itertools import ilen -from my.core.cfg import tmp_config -from my.pdfs import annotated_pdfs, annotations, get_annots +import pytest + from my.tests.common import testdata def test_module(with_config) -> None: + # TODO crap. if module is imported too early (on the top level, it makes it super hard to override config) + # need to at least detect it... + from my.pdfs import annotations, annotated_pdfs + # todo check types etc as well assert ilen(annotations()) >= 3 assert ilen(annotated_pdfs()) >= 1 @@ -20,13 +22,12 @@ def test_with_error(with_config, tmp_path: Path) -> None: root = tmp_path g = root / 'garbage.pdf' g.write_text('garbage') - from my.config import pdfs - # meh. otherwise legacy config value 'wins' del pdfs.roots # type: ignore[attr-defined] pdfs.paths = (root,) + from my.pdfs import annotations annots = list(annotations()) [annot] = annots assert isinstance(annot, Exception) @@ -34,6 +35,9 @@ def test_with_error(with_config, tmp_path: Path) -> None: @pytest.fixture def with_config(): + from my.tests.common import reset_modules + reset_modules() # todo ugh.. getting boilerplaty.. need to make it a bit more automatic.. + # extra_data = Path(__file__).absolute().parent / 'extra/data/polar' # assert extra_data.exists(), extra_data # todo hmm, turned out no annotations in these ones.. whatever @@ -43,9 +47,13 @@ def with_config(): testdata(), ] - with tmp_config() as config: + import my.core.cfg as C + with C.tmp_config() as config: config.pdfs = user_config - yield + try: + yield + finally: + reset_modules() EXPECTED_HIGHLIGHTS = { @@ -60,9 +68,11 @@ def test_get_annots() -> None: Test get_annots, with a real PDF file get_annots should return a list of three Annotation objects """ + from my.pdfs import get_annots + annotations = get_annots(testdata() / 'pdfs' / 'Information Architecture for the World Wide Web.pdf') assert len(annotations) == 3 - assert {a.highlight for a in annotations} == EXPECTED_HIGHLIGHTS + assert set([a.highlight for a in annotations]) == EXPECTED_HIGHLIGHTS def test_annotated_pdfs_with_filelist() -> None: @@ -70,9 +80,12 @@ def test_annotated_pdfs_with_filelist() -> None: Test annotated_pdfs, with a real PDF file annotated_pdfs should return a list of one Pdf object, with three Annotations """ + from my.pdfs import annotated_pdfs + filelist = [testdata() / 'pdfs' / 'Information Architecture for the World Wide Web.pdf'] annotations_generator = annotated_pdfs(filelist=filelist) + import inspect assert inspect.isgeneratorfunction(annotated_pdfs) highlights_from_pdfs = [] diff --git a/tests/serialize.py b/tests/serialize.py new file mode 100644 index 0000000..d9ee9a3 --- /dev/null +++ b/tests/serialize.py @@ -0,0 +1 @@ +from my.core.serialize import * diff --git a/tests/serialize_simplejson.py b/tests/serialize_simplejson.py new file mode 100644 index 0000000..d421a15 --- /dev/null +++ b/tests/serialize_simplejson.py @@ -0,0 +1,23 @@ +''' +This file should only run when simplejson is installed, +but orjson is not installed to check compatibility +''' + +# none of these should fail + +import json +import simplejson +import pytest + +from my.core.serialize import dumps, _A + +def test_simplejson_fallback() -> None: + + # should fail to import + with pytest.raises(ModuleNotFoundError): + import orjson + + # simplejson should serialize namedtuple properly + res: str = dumps(_A(x=1, y=2.0)) + assert json.loads(res) == {"x": 1, "y": 2.0} + diff --git a/my/tests/shared_tz_config.py b/tests/shared_config.py similarity index 54% rename from my/tests/shared_tz_config.py rename to tests/shared_config.py index 810d989..c2f6973 100644 --- a/my/tests/shared_tz_config.py +++ b/tests/shared_config.py @@ -1,26 +1,47 @@ -""" -Helper to test various timezone/location dependent things -""" +# Defines some shared config for tests -from datetime import date, datetime, timezone +from datetime import datetime, date, timezone from pathlib import Path -import pytest +from typing import Any, NamedTuple +import my.time.tz.via_location as LTZ from more_itertools import one -from my.core.cfg import tmp_config + +class SharedConfig(NamedTuple): + google: Any + location: Any + time: Any -@pytest.fixture(autouse=True) -def config(tmp_path: Path): - # TODO could just pick a part of shared config? not sure - _takeout_path = _prepare_takeouts_dir(tmp_path) +def _prepare_google_config(tmp_path: Path): + from my.tests.common import testdata + try: + track = one(testdata().rglob('italy-slovenia-2017-07-29.json')) + except ValueError: + raise RuntimeError('testdata not found, setup git submodules?') - class google: - takeout_path = _takeout_path + + # todo ugh. unnecessary zipping, but at the moment takeout provider doesn't support plain dirs + import zipfile + with zipfile.ZipFile(tmp_path / 'takeout.zip', 'w') as zf: + zf.writestr('Takeout/Location History/Location History.json', track.read_bytes()) + + class google_config: + takeout_path = tmp_path + return google_config + + +# pass tmp_path from pytest to this helper function +# see tests/tz.py as an example +def temp_config(temp_path: Path) -> Any: + from my.tests.common import reset_modules + reset_modules() + + LTZ.config.fast = True class location: - # fmt: off + home_accuracy = 30_000 home = ( # supports ISO strings ('2005-12-04' , (42.697842, 23.325973)), # Bulgaria, Sofia @@ -29,32 +50,16 @@ def config(tmp_path: Path): # check tz handling.. (datetime.fromtimestamp(1600000000, tz=timezone.utc), (55.7558 , 37.6173 )), # Moscow, Russia ) - # fmt: on # note: order doesn't matter, will be sorted in the data provider + class via_ip: + accuracy = 15_000 + class gpslogger: + pass class time: class tz: class via_location: - fast = True # some tests rely on it - - with tmp_config() as cfg: - cfg.google = google - cfg.location = location - cfg.time = time - yield cfg + pass # just rely on the defaults... -def _prepare_takeouts_dir(tmp_path: Path) -> Path: - from .common import testdata - - try: - track = one(testdata().rglob('italy-slovenia-2017-07-29.json')) - except ValueError as e: - raise RuntimeError('testdata not found, setup git submodules?') from e - - # todo ugh. unnecessary zipping, but at the moment takeout provider doesn't support plain dirs - import zipfile - - with zipfile.ZipFile(tmp_path / 'takeout.zip', 'w') as zf: - zf.writestr('Takeout/Location History/Location History.json', track.read_bytes()) - return tmp_path + return SharedConfig(google=_prepare_google_config(temp_path), location=location, time=time) diff --git a/tests/smscalls.py b/tests/smscalls.py index ef78786..d063de1 100644 --- a/tests/smscalls.py +++ b/tests/smscalls.py @@ -4,7 +4,6 @@ from my.tests.common import skip_if_not_karlicoss as pytestmark # TODO implement via stat? def test() -> None: - from my.smscalls import calls, messages, mms + from my.smscalls import calls, messages assert len(list(calls())) > 10 assert len(list(messages())) > 10 - assert len(list(mms())) > 10 diff --git a/tests/takeout.py b/tests/takeout.py index 47d405b..cddc684 100644 --- a/tests/takeout.py +++ b/tests/takeout.py @@ -1,4 +1,4 @@ -from my.tests.common import skip_if_not_karlicoss as pytestmark +#!/usr/bin/env python3 from datetime import datetime, timezone from itertools import islice import pytz diff --git a/tests/tz.py b/tests/tz.py new file mode 100644 index 0000000..d86c5cb --- /dev/null +++ b/tests/tz.py @@ -0,0 +1,95 @@ +import sys +from datetime import datetime, timedelta +from pathlib import Path + +import pytest +import pytz + +from my.core.error import notnone + +import my.time.tz.main as TZ +import my.time.tz.via_location as LTZ + + +def test_iter_tzs() -> None: + ll = list(LTZ._iter_tzs()) + assert len(ll) > 3 + + +def test_past() -> None: + # should fallback to the home location provider + dt = D('20000101 12:34:45') + dt = TZ.localize(dt) + tz = dt.tzinfo + assert tz is not None + assert getattr(tz, 'zone') == 'America/New_York' + + +def test_future() -> None: + fut = datetime.now() + timedelta(days=100) + # shouldn't crash at least + assert TZ.localize(fut) is not None + + +def test_tz() -> None: + # todo hmm, the way it's implemented at the moment, never returns None? + + # not present in the test data + tz = LTZ._get_tz(D('20200101 10:00:00')) + assert notnone(tz).zone == 'Europe/Sofia' + + tz = LTZ._get_tz(D('20170801 11:00:00')) + assert notnone(tz).zone == 'Europe/Vienna' + + tz = LTZ._get_tz(D('20170730 10:00:00')) + assert notnone(tz).zone == 'Europe/Rome' + + tz = LTZ._get_tz(D('20201001 14:15:16')) + assert tz is not None + + on_windows = sys.platform == 'win32' + if not on_windows: + tz = LTZ._get_tz(datetime.min) + assert tz is not None + else: + # seems this fails because windows doesn't support same date ranges + # https://stackoverflow.com/a/41400321/ + with pytest.raises(OSError): + LTZ._get_tz(datetime.min) + + +def test_policies() -> None: + getzone = lambda dt: getattr(dt.tzinfo, 'zone') + + naive = D('20170730 10:00:00') + # actual timezone at the time + assert getzone(TZ.localize(naive)) == 'Europe/Rome' + + z = pytz.timezone('America/New_York') + aware = z.localize(naive) + + assert getzone(TZ.localize(aware)) == 'America/New_York' + + assert getzone(TZ.localize(aware, policy='convert')) == 'Europe/Rome' + + + with pytest.raises(RuntimeError): + assert TZ.localize(aware, policy='throw') + + +def D(dstr: str) -> datetime: + return datetime.strptime(dstr, '%Y%m%d %H:%M:%S') + + + +@pytest.fixture(autouse=True) +def prepare(tmp_path: Path): + from .shared_config import temp_config + conf = temp_config(tmp_path) + + import my.core.cfg as C + with C.tmp_config() as config: + config.google = conf.google + config.time = conf.time + config.location = conf.location + yield diff --git a/tox.ini b/tox.ini index d202bd2..ac0a68d 100644 --- a/tox.ini +++ b/tox.ini @@ -17,9 +17,6 @@ passenv = PYTHONPYCACHEPREFIX MYPY_CACHE_DIR RUFF_CACHE_DIR -setenv = - HPI_MODULE_INSTALL_USE_UV=true -uv_seed = true # seems necessary so uv creates separate venvs per tox env? # note: --use-pep517 below is necessary for tox --parallel flag to work properly @@ -27,96 +24,120 @@ uv_seed = true # seems necessary so uv creates separate venvs per tox env? [testenv:ruff] -deps = - -e .[testing] commands = - {envpython} -m ruff check my/ + {envpython} -m pip install --use-pep517 -e .[testing] + {envpython} -m ruff my/ # just the very core tests with minimal dependencies [testenv:tests-core] -deps = - -e .[testing] commands = + {envpython} -m pip install --use-pep517 -e .[testing] + + # seems that denylist tests rely on it? ideally we should get rid of this in tests-core + {envpython} -m pip install orjson + {envpython} -m pytest \ # importlib is the new suggested import-mode # without it test package names end up as core.tests.* instead of my.core.tests.* --import-mode=importlib \ - --pyargs {[testenv]package_name}.core \ + --pyargs my.core \ # ignore orgmode because it imports orgparse # tbh not sure if it even belongs to core, maybe move somewhere else.. # same with pandas? --ignore my/core/orgmode.py \ + # causes error during test collection on 3.8 + # dataset is deprecated anyway so whatever + --ignore my/core/dataset.py \ + # this test uses orjson which is an optional dependency + # it would be covered by tests-all + -k 'not test_nt_serialize' \ {posargs} # todo maybe also have core tests and misc tests? since ideally want them without dependencies [testenv:tests-all] -setenv = - # deliberately set to nonexistent path to check the fallback logic - # TODO not sure if need it? - MY_CONFIG=nonexistent - HPI_TESTS_USES_OPTIONAL_DEPS=true -deps = - -e .[testing] - uv # for hpi module install - cachew - ijson # optional dependency for various modules +# deliberately set to nonexistent path to check the fallback logic +# TODO not sure if need it? +setenv = MY_CONFIG = nonexistent commands = - {envpython} -m my.core module install \ - ## tz/location - my.location.google \ - my.time.tz.via_location \ - my.ip.all \ - my.location.gpslogger \ - my.location.fallback.via_ip \ - my.google.takeout.parser \ - ## - my.calendar.holidays \ - my.orgmode \ # my.body.weight dep - my.coding.commits \ - my.pdfs \ - my.reddit.rexport + {envpython} -m pip install --use-pep517 -e .[testing] + + # installed to test my.core.serialize while using simplejson and not orjson + {envpython} -m pip install simplejson + {envpython} -m pytest \ + tests/serialize_simplejson.py \ + {posargs} + + {envpython} -m pip install cachew + {envpython} -m pip install orjson + + {envpython} -m my.core module install my.location.google + {envpython} -m pip install ijson # optional dependency + + # tz/location + {envpython} -m my.core module install my.time.tz.via_location + {envpython} -m my.core module install my.ip.all + {envpython} -m my.core module install my.location.gpslogger + {envpython} -m my.core module install my.location.fallback.via_ip + {envpython} -m my.core module install my.google.takeout.parser + + {envpython} -m my.core module install my.calendar.holidays + + # my.body.weight dep + {envpython} -m my.core module install my.orgmode + + {envpython} -m my.core module install my.coding.commits + + {envpython} -m my.core module install my.pdfs + + {envpython} -m my.core module install my.reddit.rexport {envpython} -m pytest \ # importlib is the new suggested import-mode # without it test package names end up as core.tests.* instead of my.core.tests.* --import-mode=importlib \ - --pyargs {[testenv]package_name}.core {[testenv]package_name}.tests \ + --pyargs {[testenv]package_name}.tests \ + {posargs} + + {envpython} -m pytest tests \ + # ignore some tests which might take a while to run on ci.. + --ignore tests/takeout.py \ + --ignore tests/extra/polar.py \ + # dont run simplejson compatibility test since orjson is now installed + --ignore tests/serialize_simplejson.py \ {posargs} [testenv:demo] -deps = - git+https://github.com/karlicoss/hypexport commands = + {envpython} -m pip install git+https://github.com/karlicoss/hypexport {envpython} ./demo.py [testenv:mypy-core] -deps = - -e .[testing,optional] - orgparse # for core.orgmode - gpxpy # for hpi query --output gpx +allowlist_externals = cat commands = - {envpython} -m mypy --no-install-types \ + {envpython} -m pip install --use-pep517 -e .[testing,optional] + {envpython} -m pip install orgparse # used it core.orgmode? + {envpython} -m pip install gpxpy # for hpi query --output gpx + + {envpython} -m mypy --install-types --non-interactive \ -p {[testenv]package_name}.core \ --txt-report .coverage.mypy-core \ --html-report .coverage.mypy-core \ {posargs} + cat .coverage.mypy-core/index.txt # specific modules that are known to be mypy compliant (to avoid false negatives) # todo maybe split into separate jobs? need to add comment how to run [testenv:mypy-misc] -deps = - -e .[testing,optional] - uv # for hpi module install - lxml-stubs # for my.smscalls - types-protobuf # for my.google.maps.android - types-Pillow # for my.photos +allowlist_externals = cat commands = - {envpython} -m my.core module install \ + {envpython} -m pip install --use-pep517 -e .[testing,optional] + + {envpython} -m my.core module install --parallel \ my.arbtt \ my.browser.export \ my.coding.commits \ @@ -125,7 +146,6 @@ commands = my.fbmessenger.export \ my.github.ghexport \ my.goodreads \ - my.google.maps.android \ my.google.takeout.parser \ my.hackernews.harmonic \ my.hypothesis \ @@ -133,7 +153,6 @@ commands = my.ip.all \ my.kobo \ my.location.gpslogger \ - my.monzo.monzoexport \ my.orgmode \ my.pdfs \ my.pinboard \ @@ -147,13 +166,18 @@ commands = my.time.tz.via_location - {envpython} -m mypy --no-install-types \ + {envpython} -m mypy --install-types --non-interactive \ -p {[testenv]package_name} \ + --exclude 'my/coding/codeforces.py' \ + --exclude 'my/coding/topcoder.py' \ + --exclude 'my/jawbone/.*' \ --txt-report .coverage.mypy-misc \ --html-report .coverage.mypy-misc \ {posargs} + # txt report is a bit more convenient to view on CI + cat .coverage.mypy-misc/index.txt - {envpython} -m mypy --no-install-types \ + {envpython} -m mypy --install-types --non-interactive \ tests # note: this comment doesn't seem relevant anymore, but keeping it in case the issue happens again