ci: sync configs to pymplate

- add python3.12
- add ruff

.ci/release

@@ -21,7 +21,7 @@ import shutil
 
 is_ci = os.environ.get('CI') is not None
 
-def main():
+def main() -> None:
     import argparse
     p = argparse.ArgumentParser()
     p.add_argument('--test', action='store_true', help='use test pypi')
@@ -29,7 +29,7 @@ def main():
 
     extra = []
     if args.test:
-        extra.extend(['--repository-url', 'https://test.pypi.org/legacy/'])
+        extra.extend(['--repository', 'testpypi'])
 
     root = Path(__file__).absolute().parent.parent
     os.chdir(root) # just in case
@@ -42,7 +42,7 @@ def main():
     if dist.exists():
         shutil.rmtree(dist)
 
-    check_call('python3 setup.py sdist bdist_wheel', shell=True)
+    check_call(['python3', '-m', 'build'])
 
     TP = 'TWINE_PASSWORD'
     password = os.environ.get(TP)

.github/workflows/main.yml

@@ -5,11 +5,17 @@ on:
   push:
     branches: '*'
     tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi
-  # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
+    # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug:
   pull_request: # needed to trigger on others' PRs
   # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
   workflow_dispatch: # needed to trigger workflows manually
     # todo cron?
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
 
 
 jobs:
@@ -17,15 +23,17 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest, macos-latest,  windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         exclude: [
             # windows runners are pretty scarce, so let's only run lowest and highest python version
             {platform: windows-latest, python-version: '3.9' },
             {platform: windows-latest, python-version: '3.10'},
+            {platform: windows-latest, python-version: '3.11'},
 
             # same, macos is a bit too slow and ubuntu covers python quirks well
             {platform: macos-latest  , python-version: '3.9'  },
             {platform: macos-latest  , python-version: '3.10' },
+            {platform: macos-latest  , python-version: '3.11' },
         ]
 
     runs-on: ${{ matrix.platform }}
@@ -46,11 +54,11 @@ jobs:
         submodules: recursive
         fetch-depth: 0  # nicer to have all git history when debugging/for tests
 
-    # uncomment for SSH debugging
+    - uses: mxschmitt/action-tmate@v3
-    # - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
     # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
-    - run: bash scripts/ci/run
+    - run: bash .ci/run
 
     - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
       uses: actions/upload-artifact@v3
@@ -71,7 +79,7 @@ jobs:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
     - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
 
-    - uses: actions/setup-python@v3
+    - uses: actions/setup-python@v4
       with:
         python-version: '3.8'
 
@@ -84,8 +92,7 @@ jobs:
       if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master'
       env:
         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }}
-      run: pip3 install --user wheel twine && scripts/release --test
+      run: pip3 install --user --upgrade build twine && .ci/release --test
-      # TODO run pip install just to test?
 
     - name: 'release to pypi'
       # always deploy tags to release pypi
@@ -93,4 +100,4 @@ jobs:
       if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags')
       env:
         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-      run: pip3 install --user wheel twine && scripts/release
+      run: pip3 install --user --upgrade build twine && .ci/release --test

my/core/core_config.py

@@ -146,7 +146,7 @@ def test_active_modules() -> None:
         cc.disabled_modules = ['my.body.*']
         assert cc._is_module_active('my.whatever'     ) is True
         assert cc._is_module_active('my.core'         ) is None
-        assert not cc._is_module_active('my.body.exercise') is True
+        assert cc._is_module_active('my.body.exercise') is False
 
     with reset() as cc:
         # if both are set, enable all

my/fbmessenger/all.py

@@ -5,8 +5,8 @@ from my.core.source import import_source
 from .common import Message, _merge_messages
 
 
-src_export  = import_source(module_name=f'my.fbmessenger.export')
+src_export  = import_source(module_name='my.fbmessenger.export')
-src_android = import_source(module_name=f'my.fbmessenger.android')
+src_android = import_source(module_name='my.fbmessenger.android')
 
 
 @src_export

my/twitter/all.py

@@ -8,8 +8,8 @@ from .common import merge_tweets, Tweet
 
 
 # NOTE: you can comment out the sources you don't need
-src_twint   = import_source(module_name=f'my.twitter.twint')
+src_twint   = import_source(module_name='my.twitter.twint')
-src_archive = import_source(module_name=f'my.twitter.archive')
+src_archive = import_source(module_name='my.twitter.archive')
 
 
 @src_twint

ruff.toml

@@ -0,0 +1,25 @@
+ignore = [
+### too opinionated style checks
+    "E501",  # too long lines
+    "E702",  # Multiple statements on one line (semicolon)
+    "E731",  # assigning lambda instead of using def
+    "E741",  # Ambiguous variable name: `l`
+    "E742",  # Ambiguous class name: `O
+    "E401",  # Multiple imports on one line
+    "F403",  # import *` used; unable to detect undefined names
+###
+
+###
+    "E722",  # Do not use bare `except` ## Sometimes it's useful for defensive imports and that sort of thing..
+    "F811",  # Redefinition of unused  # this gets in the way of pytest fixtures (e.g. in cachew)
+
+## might be nice .. but later and I don't wanna make it strict
+    "E402",  # Module level import not at top of file
+
+### maybe consider these soon
+# sometimes it's useful to give a variable a name even if we don't use it as a documentation
+# on the other hand, often is a sign of error
+    "F841",  # Local variable `count` is assigned to but never used
+    "F401",  # imported but unused
+###
+]

setup.py

@@ -47,6 +47,7 @@ def main() -> None:
         extras_require={
             'testing': [
                 'pytest',
+                'ruff',
                 'mypy',
                 'lxml', # for mypy coverage
 

tox.ini

@@ -1,31 +1,41 @@
 [tox]
-minversion = 3.5
+minversion = 3.21
-envlist = tests-core,tests-all,demo,mypy-core,mypy-misc
+# relies on the correct version of Python installed
+envlist = ruff,tests-core,tests-all,demo,mypy-core,mypy-misc
 # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333
 # hack to prevent .tox from crapping to the project directory
 toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox
 
 [testenv]
+# TODO how to get package name from setuptools?
+package_name = "my"
 passenv = 
 # useful for tests to know they are running under ci
     CI
     CI_*
 # respect user's cache dirs to prevent tox from crapping into project dir
-  MYPY_CACHE_DIR
     PYTHONPYCACHEPREFIX
+    MYPY_CACHE_DIR
+    RUFF_CACHE_DIR
 
 
 # note: --use-pep517 below is necessary for tox --parallel flag to work properly
 # otherwise it seems that it tries to modify .eggs dir in parallel and it fails
 
 
+[testenv:ruff]
+commands =
+    {envpython} -m pip install --use-pep517 -e .[testing]
+    {envpython} -m ruff my/
+
+
 # just the very core tests with minimal dependencies
 [testenv:tests-core]
 commands =
-    pip install --use-pep517 -e .[testing]
+    {envpython} -m pip install --use-pep517 -e .[testing]
     
     # seems that denylist tests rely on it? ideally we should get rid of this in tests-core
-    pip install orjson
+    {envpython} -m pip install orjson
 
     {envpython} -m pytest \
         # importlib is the new suggested import-mode
@@ -51,43 +61,43 @@ commands =
 # TODO not sure if need it?
 setenv = MY_CONFIG = nonexistent
 commands =
-    pip install --use-pep517 -e .[testing]
+    {envpython} -m pip install --use-pep517 -e .[testing]
 
     # installed to test my.core.serialize while using simplejson and not orjson
-    pip install simplejson
+    {envpython} -m pip install simplejson
     {envpython} -m pytest \
         tests/serialize_simplejson.py \
         {posargs}
 
-    pip install cachew
+    {envpython} -m pip install cachew
-    pip install orjson
+    {envpython} -m pip install orjson
 
-    hpi module install my.location.google
+    {envpython} -m my.core module install my.location.google
-    pip install ijson # optional dependency
+    {envpython} -m pip install ijson # optional dependency
 
     # tz/location
-    hpi module install my.time.tz.via_location
+    {envpython} -m my.core module install my.time.tz.via_location
-    hpi module install my.ip.all
+    {envpython} -m my.core module install my.ip.all
-    hpi module install my.location.gpslogger
+    {envpython} -m my.core module install my.location.gpslogger
-    hpi module install my.location.fallback.via_ip
+    {envpython} -m my.core module install my.location.fallback.via_ip
-    hpi module install my.google.takeout.parser
+    {envpython} -m my.core module install my.google.takeout.parser
 
-    hpi module install my.calendar.holidays
+    {envpython} -m my.core module install my.calendar.holidays
 
     # my.body.weight dep
-    hpi module install my.orgmode
+    {envpython} -m my.core module install my.orgmode
 
-    hpi module install my.coding.commits
+    {envpython} -m my.core module install my.coding.commits
 
-    hpi module install my.pdfs
+    {envpython} -m my.core module install my.pdfs
 
-    hpi module install my.reddit.rexport
+    {envpython} -m my.core module install my.reddit.rexport
 
     {envpython} -m pytest \
         # importlib is the new suggested import-mode
         # without it test package names end up as core.tests.* instead of my.core.tests.*
         --import-mode=importlib     \
-        --pyargs my.tests \
+        --pyargs {[testenv]package_name}.tests \
         {posargs}
 
     {envpython} -m pytest tests \
@@ -101,19 +111,19 @@ commands =
 
 [testenv:demo]
 commands =
-    pip install git+https://github.com/karlicoss/hypexport
+    {envpython} -m pip install git+https://github.com/karlicoss/hypexport
     {envpython} ./demo.py
 
 
 [testenv:mypy-core]
 allowlist_externals = cat
 commands =
-    pip install --use-pep517 -e .[testing,optional]
+    {envpython} -m pip install --use-pep517 -e .[testing,optional]
-    pip install orgparse # used it core.orgmode?
+    {envpython} -m pip install orgparse # used it core.orgmode?
-    pip install gpxpy # for hpi query --output gpx
+    {envpython} -m pip install gpxpy # for hpi query --output gpx
 
     {envpython} -m mypy --install-types --non-interactive \
-                    -p my.core \
+        -p {[testenv]package_name}.core \
         --txt-report  .coverage.mypy-core \
         --html-report .coverage.mypy-core \
         {posargs}
@@ -125,9 +135,9 @@ commands =
 [testenv:mypy-misc]
 allowlist_externals = cat
 commands =
-    pip install --use-pep517 -e .[testing,optional]
+    {envpython} -m pip install --use-pep517 -e .[testing,optional]
 
-    hpi module install --parallel                \
+    {envpython} -m my.core module install --parallel \
         my.arbtt                  \
         my.browser.export         \
         my.coding.commits         \
@@ -157,7 +167,7 @@ commands =
 
 
     {envpython} -m mypy --install-types --non-interactive \
-                    -p my \
+        -p {[testenv]package_name} \
         --exclude 'my/coding/codeforces.py' \
         --exclude 'my/coding/topcoder.py'   \
         --exclude 'my/jawbone/.*'           \