From 05124882416efdebfa5a3241903cf2d7375f6d00 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 6 Oct 2023 02:02:43 +0100 Subject: [PATCH] ci: sync configs to pymplate - add python3.12 - add ruff --- {scripts => .ci}/release | 6 +- {scripts/ci => .ci}/run | 0 .github/workflows/main.yml | 31 ++++++---- my/core/core_config.py | 6 +- my/fbmessenger/all.py | 4 +- my/twitter/all.py | 4 +- ruff.toml | 25 ++++++++ setup.py | 1 + tox.ini | 114 ++++++++++++++++++++----------------- 9 files changed, 117 insertions(+), 74 deletions(-) rename {scripts => .ci}/release (92%) rename {scripts/ci => .ci}/run (100%) create mode 100644 ruff.toml diff --git a/scripts/release b/.ci/release similarity index 92% rename from scripts/release rename to .ci/release index 0ec687f..6cff663 100755 --- a/scripts/release +++ b/.ci/release @@ -21,7 +21,7 @@ import shutil is_ci = os.environ.get('CI') is not None -def main(): +def main() -> None: import argparse p = argparse.ArgumentParser() p.add_argument('--test', action='store_true', help='use test pypi') @@ -29,7 +29,7 @@ def main(): extra = [] if args.test: - extra.extend(['--repository-url', 'https://test.pypi.org/legacy/']) + extra.extend(['--repository', 'testpypi']) root = Path(__file__).absolute().parent.parent os.chdir(root) # just in case @@ -42,7 +42,7 @@ def main(): if dist.exists(): shutil.rmtree(dist) - check_call('python3 setup.py sdist bdist_wheel', shell=True) + check_call(['python3', '-m', 'build']) TP = 'TWINE_PASSWORD' password = os.environ.get(TP) diff --git a/scripts/ci/run b/.ci/run similarity index 100% rename from scripts/ci/run rename to .ci/run diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 47f84cb..cf85155 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,11 +5,17 @@ on: push: branches: '*' tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi - # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them". + # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug: pull_request: # needed to trigger on others' PRs # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them". workflow_dispatch: # needed to trigger workflows manually - # todo cron? + # todo cron? + inputs: + debug_enabled: + type: boolean + description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' + required: false + default: false jobs: @@ -17,15 +23,17 @@ jobs: strategy: matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] exclude: [ # windows runners are pretty scarce, so let's only run lowest and highest python version - {platform: windows-latest, python-version: '3.9'}, + {platform: windows-latest, python-version: '3.9' }, {platform: windows-latest, python-version: '3.10'}, + {platform: windows-latest, python-version: '3.11'}, # same, macos is a bit too slow and ubuntu covers python quirks well - {platform: macos-latest , python-version: '3.9' }, + {platform: macos-latest , python-version: '3.9' }, {platform: macos-latest , python-version: '3.10' }, + {platform: macos-latest , python-version: '3.11' }, ] runs-on: ${{ matrix.platform }} @@ -46,11 +54,11 @@ jobs: submodules: recursive fetch-depth: 0 # nicer to have all git history when debugging/for tests - # uncomment for SSH debugging - # - uses: mxschmitt/action-tmate@v3 + - uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd... - - run: bash scripts/ci/run + - run: bash .ci/run - if: matrix.platform == 'ubuntu-latest' # no need to compute coverage for other platforms uses: actions/upload-artifact@v3 @@ -71,7 +79,7 @@ jobs: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v4 with: python-version: '3.8' @@ -84,8 +92,7 @@ jobs: if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master' env: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }} - run: pip3 install --user wheel twine && scripts/release --test - # TODO run pip install just to test? + run: pip3 install --user --upgrade build twine && .ci/release --test - name: 'release to pypi' # always deploy tags to release pypi @@ -93,4 +100,4 @@ jobs: if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags') env: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - run: pip3 install --user wheel twine && scripts/release + run: pip3 install --user --upgrade build twine && .ci/release --test diff --git a/my/core/core_config.py b/my/core/core_config.py index 5c696ce..e70dc05 100644 --- a/my/core/core_config.py +++ b/my/core/core_config.py @@ -144,9 +144,9 @@ def test_active_modules() -> None: with reset() as cc: cc.enabled_modules = ['my.whatever'] cc.disabled_modules = ['my.body.*'] - assert cc._is_module_active('my.whatever' ) is True - assert cc._is_module_active('my.core' ) is None - assert not cc._is_module_active('my.body.exercise') is True + assert cc._is_module_active('my.whatever' ) is True + assert cc._is_module_active('my.core' ) is None + assert cc._is_module_active('my.body.exercise') is False with reset() as cc: # if both are set, enable all diff --git a/my/fbmessenger/all.py b/my/fbmessenger/all.py index f98b5f3..13689db 100644 --- a/my/fbmessenger/all.py +++ b/my/fbmessenger/all.py @@ -5,8 +5,8 @@ from my.core.source import import_source from .common import Message, _merge_messages -src_export = import_source(module_name=f'my.fbmessenger.export') -src_android = import_source(module_name=f'my.fbmessenger.android') +src_export = import_source(module_name='my.fbmessenger.export') +src_android = import_source(module_name='my.fbmessenger.android') @src_export diff --git a/my/twitter/all.py b/my/twitter/all.py index b203511..4714021 100644 --- a/my/twitter/all.py +++ b/my/twitter/all.py @@ -8,8 +8,8 @@ from .common import merge_tweets, Tweet # NOTE: you can comment out the sources you don't need -src_twint = import_source(module_name=f'my.twitter.twint') -src_archive = import_source(module_name=f'my.twitter.archive') +src_twint = import_source(module_name='my.twitter.twint') +src_archive = import_source(module_name='my.twitter.archive') @src_twint diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..0be93e0 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,25 @@ +ignore = [ +### too opinionated style checks + "E501", # too long lines + "E702", # Multiple statements on one line (semicolon) + "E731", # assigning lambda instead of using def + "E741", # Ambiguous variable name: `l` + "E742", # Ambiguous class name: `O + "E401", # Multiple imports on one line + "F403", # import *` used; unable to detect undefined names +### + +### + "E722", # Do not use bare `except` ## Sometimes it's useful for defensive imports and that sort of thing.. + "F811", # Redefinition of unused # this gets in the way of pytest fixtures (e.g. in cachew) + +## might be nice .. but later and I don't wanna make it strict + "E402", # Module level import not at top of file + +### maybe consider these soon +# sometimes it's useful to give a variable a name even if we don't use it as a documentation +# on the other hand, often is a sign of error + "F841", # Local variable `count` is assigned to but never used + "F401", # imported but unused +### +] diff --git a/setup.py b/setup.py index ee4c2f0..5fa988e 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ def main() -> None: extras_require={ 'testing': [ 'pytest', + 'ruff', 'mypy', 'lxml', # for mypy coverage diff --git a/tox.ini b/tox.ini index 860641f..ac0a68d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,31 +1,41 @@ [tox] -minversion = 3.5 -envlist = tests-core,tests-all,demo,mypy-core,mypy-misc +minversion = 3.21 +# relies on the correct version of Python installed +envlist = ruff,tests-core,tests-all,demo,mypy-core,mypy-misc # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 # hack to prevent .tox from crapping to the project directory -toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox +toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox [testenv] +# TODO how to get package name from setuptools? +package_name = "my" passenv = # useful for tests to know they are running under ci - CI - CI_* + CI + CI_* # respect user's cache dirs to prevent tox from crapping into project dir - MYPY_CACHE_DIR - PYTHONPYCACHEPREFIX + PYTHONPYCACHEPREFIX + MYPY_CACHE_DIR + RUFF_CACHE_DIR # note: --use-pep517 below is necessary for tox --parallel flag to work properly # otherwise it seems that it tries to modify .eggs dir in parallel and it fails +[testenv:ruff] +commands = + {envpython} -m pip install --use-pep517 -e .[testing] + {envpython} -m ruff my/ + + # just the very core tests with minimal dependencies [testenv:tests-core] commands = - pip install --use-pep517 -e .[testing] + {envpython} -m pip install --use-pep517 -e .[testing] # seems that denylist tests rely on it? ideally we should get rid of this in tests-core - pip install orjson + {envpython} -m pip install orjson {envpython} -m pytest \ # importlib is the new suggested import-mode @@ -51,72 +61,72 @@ commands = # TODO not sure if need it? setenv = MY_CONFIG = nonexistent commands = - pip install --use-pep517 -e .[testing] + {envpython} -m pip install --use-pep517 -e .[testing] # installed to test my.core.serialize while using simplejson and not orjson - pip install simplejson + {envpython} -m pip install simplejson {envpython} -m pytest \ - tests/serialize_simplejson.py \ - {posargs} + tests/serialize_simplejson.py \ + {posargs} - pip install cachew - pip install orjson + {envpython} -m pip install cachew + {envpython} -m pip install orjson - hpi module install my.location.google - pip install ijson # optional dependency + {envpython} -m my.core module install my.location.google + {envpython} -m pip install ijson # optional dependency # tz/location - hpi module install my.time.tz.via_location - hpi module install my.ip.all - hpi module install my.location.gpslogger - hpi module install my.location.fallback.via_ip - hpi module install my.google.takeout.parser + {envpython} -m my.core module install my.time.tz.via_location + {envpython} -m my.core module install my.ip.all + {envpython} -m my.core module install my.location.gpslogger + {envpython} -m my.core module install my.location.fallback.via_ip + {envpython} -m my.core module install my.google.takeout.parser - hpi module install my.calendar.holidays + {envpython} -m my.core module install my.calendar.holidays # my.body.weight dep - hpi module install my.orgmode + {envpython} -m my.core module install my.orgmode - hpi module install my.coding.commits + {envpython} -m my.core module install my.coding.commits - hpi module install my.pdfs + {envpython} -m my.core module install my.pdfs - hpi module install my.reddit.rexport + {envpython} -m my.core module install my.reddit.rexport {envpython} -m pytest \ # importlib is the new suggested import-mode # without it test package names end up as core.tests.* instead of my.core.tests.* --import-mode=importlib \ - --pyargs my.tests \ + --pyargs {[testenv]package_name}.tests \ {posargs} {envpython} -m pytest tests \ - # ignore some tests which might take a while to run on ci.. - --ignore tests/takeout.py \ - --ignore tests/extra/polar.py \ - # dont run simplejson compatibility test since orjson is now installed - --ignore tests/serialize_simplejson.py \ - {posargs} + # ignore some tests which might take a while to run on ci.. + --ignore tests/takeout.py \ + --ignore tests/extra/polar.py \ + # dont run simplejson compatibility test since orjson is now installed + --ignore tests/serialize_simplejson.py \ + {posargs} [testenv:demo] commands = - pip install git+https://github.com/karlicoss/hypexport + {envpython} -m pip install git+https://github.com/karlicoss/hypexport {envpython} ./demo.py [testenv:mypy-core] allowlist_externals = cat commands = - pip install --use-pep517 -e .[testing,optional] - pip install orgparse # used it core.orgmode? - pip install gpxpy # for hpi query --output gpx + {envpython} -m pip install --use-pep517 -e .[testing,optional] + {envpython} -m pip install orgparse # used it core.orgmode? + {envpython} -m pip install gpxpy # for hpi query --output gpx {envpython} -m mypy --install-types --non-interactive \ - -p my.core \ - --txt-report .coverage.mypy-core \ - --html-report .coverage.mypy-core \ - {posargs} + -p {[testenv]package_name}.core \ + --txt-report .coverage.mypy-core \ + --html-report .coverage.mypy-core \ + {posargs} cat .coverage.mypy-core/index.txt @@ -125,9 +135,9 @@ commands = [testenv:mypy-misc] allowlist_externals = cat commands = - pip install --use-pep517 -e .[testing,optional] + {envpython} -m pip install --use-pep517 -e .[testing,optional] - hpi module install --parallel \ + {envpython} -m my.core module install --parallel \ my.arbtt \ my.browser.export \ my.coding.commits \ @@ -157,18 +167,18 @@ commands = {envpython} -m mypy --install-types --non-interactive \ - -p my \ - --exclude 'my/coding/codeforces.py' \ - --exclude 'my/coding/topcoder.py' \ - --exclude 'my/jawbone/.*' \ - --txt-report .coverage.mypy-misc \ - --html-report .coverage.mypy-misc \ - {posargs} + -p {[testenv]package_name} \ + --exclude 'my/coding/codeforces.py' \ + --exclude 'my/coding/topcoder.py' \ + --exclude 'my/jawbone/.*' \ + --txt-report .coverage.mypy-misc \ + --html-report .coverage.mypy-misc \ + {posargs} # txt report is a bit more convenient to view on CI cat .coverage.mypy-misc/index.txt {envpython} -m mypy --install-types --non-interactive \ - tests + tests # note: this comment doesn't seem relevant anymore, but keeping it in case the issue happens again # > ugh ... need to reset HOME, otherwise user's site-packages are somehow leaking into mypy's path...