CI: clean up tox config a bit, get rid of custom lint script
This commit is contained in:
Dima Gerasimov
karlicoss
parent
f102101b39
commit
56d5587c20
6 changed files with 50 additions and 132 deletions
8
.github/workflows/main.yml
vendored
8
.github/workflows/main.yml
vendored
|
|
@ -40,8 +40,12 @@ jobs:
|
|||
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }}
|
||||
path: .coverage.mypy/
|
||||
name: .coverage.mypy-misc_${{ matrix.platform }}_${{ matrix.python-version }}
|
||||
path: .coverage.mypy-misc/
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: .coverage.mypy-core_${{ matrix.platform }}_${{ matrix.python-version }}
|
||||
path: .coverage.mypy-core/
|
||||
|
||||
pypi:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
- /.mypy_cache/
|
||||
|
|
@ -26,6 +26,11 @@ However, Pycharm/Emacs or whatever IDE you are using won't be able to figure tha
|
|||
i.e. create a new interpreter configuration (e.g. name it "Python 3.7 (for HPI)"), and add =~/.config/my=.
|
||||
|
||||
* Linting
|
||||
You should be able to use [[file:../lint]] script to run mypy checks.
|
||||
~tox~ should run all test, mypy, etc.
|
||||
|
||||
[[file:../mypy.ini]] points at =~/.config/my= by default.
|
||||
If you want to run some specific parts/tests, consult [[file:tox.ini]].
|
||||
|
||||
Some useful flags (look them up):
|
||||
|
||||
- ~-e~ flag for tox
|
||||
- ~-k~ flag for pytest
|
||||
|
|
|
|||
105
lint
105
lint
|
|
@ -1,105 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
from pathlib import Path
|
||||
from pprint import pprint
|
||||
from itertools import chain
|
||||
from subprocess import check_call, run, PIPE
|
||||
import sys
|
||||
import os
|
||||
from typing import List, Optional, Iterable
|
||||
|
||||
|
||||
def log(*args):
|
||||
print(*args, file=sys.stderr)
|
||||
|
||||
CI = 'CI' in os.environ
|
||||
|
||||
DIR = Path(__file__).absolute().parent
|
||||
|
||||
# hmm. I guess I need to check all subpackages separately
|
||||
# otherwise pylint doesn't work and mypy doesn't discover everything
|
||||
|
||||
# TODO could reuse in readme??
|
||||
# returns None if not a package
|
||||
def package_name(p: Path) -> str:
|
||||
def mname(p: Path):
|
||||
nosuf = p.with_suffix('')
|
||||
return str(nosuf).replace('/', '.')
|
||||
|
||||
has_init = (p.parent / '__init__.py').exists()
|
||||
if has_init:
|
||||
return mname(p.parent)
|
||||
else:
|
||||
return mname(p)
|
||||
|
||||
def subpackages(package: str) -> Iterable[str]:
|
||||
ppath = package.replace('.', '/')
|
||||
yield from sorted({
|
||||
package_name(p.relative_to(DIR)) for p in (DIR / ppath).rglob('*.py')
|
||||
})
|
||||
|
||||
|
||||
# TODO meh.. think how to check _everything_ on CI
|
||||
def core_modules() -> Iterable[str]:
|
||||
return [
|
||||
*subpackages('my.core'),
|
||||
'my.config',
|
||||
'my.cfg',
|
||||
'tests/misc.py',
|
||||
'tests/get_files.py',
|
||||
# 'tests/config.py', TODO hmm. unclear how to type check this module
|
||||
]
|
||||
|
||||
|
||||
|
||||
def all_modules() -> Iterable[str]:
|
||||
yield from subpackages('my')
|
||||
yield from sorted(
|
||||
str(f.relative_to(DIR)) for f in (DIR / 'tests').rglob('*.py')
|
||||
)
|
||||
|
||||
|
||||
def pylint():
|
||||
# TODO ugh. pylint still doesn't like checking my.config or my.books
|
||||
# only top level .py files seem ok??
|
||||
pass
|
||||
|
||||
|
||||
def mypy(thing: str):
|
||||
is_package = Path(thing).suffix != '.py'
|
||||
cmd = [
|
||||
'mypy',
|
||||
'--color-output', # TODO eh? doesn't work..
|
||||
*(['-p'] if is_package else []), thing,
|
||||
]
|
||||
print(' '.join(cmd), file=sys.stderr)
|
||||
return run(cmd, stdout=PIPE, stderr=PIPE)
|
||||
|
||||
|
||||
def mypy_all() -> Iterable[Exception]:
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
pkgs = list(core_modules() if CI else all_modules())
|
||||
log(f"Checking {pkgs}")
|
||||
with ThreadPoolExecutor() as pool:
|
||||
for p, res in zip(pkgs, pool.map(mypy, pkgs)):
|
||||
ret = res.returncode
|
||||
if ret > 0:
|
||||
log(f'FAILED: {p}')
|
||||
else:
|
||||
log(f'OK: {p}')
|
||||
print(res.stdout.decode('utf8'))
|
||||
print(res.stderr.decode('utf8'), file=sys.stderr)
|
||||
try:
|
||||
res.check_returncode()
|
||||
except Exception as e:
|
||||
yield e
|
||||
|
||||
|
||||
def main():
|
||||
errors = list(mypy_all())
|
||||
if len(errors) > 0:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
3
setup.py
3
setup.py
|
|
@ -46,11 +46,10 @@ def main():
|
|||
extras_require={
|
||||
'testing': [
|
||||
'pytest',
|
||||
'pylint',
|
||||
'mypy',
|
||||
'lxml', # for mypy coverage
|
||||
|
||||
# used in some tests
|
||||
# used in some tests.. although shouldn't rely on it
|
||||
'pandas',
|
||||
],
|
||||
'optional': [
|
||||
|
|
|
|||
56
tox.ini
56
tox.ini
|
|
@ -1,10 +1,11 @@
|
|||
[tox]
|
||||
minversion = 3.5
|
||||
envlist = py3,mypy,mypy-modules
|
||||
|
||||
# TODO ugh. unclear how to reuse setup.cfg deps in tox
|
||||
[testenv]
|
||||
passenv = CI CI_*
|
||||
|
||||
# TODO ugh. unclear how to reuse setup.cfg deps in tox
|
||||
[testenv:tests]
|
||||
# deliberately set to nonexistent path to check the fallback logic
|
||||
setenv = MY_CONFIG = nonexistent
|
||||
commands =
|
||||
|
|
@ -13,6 +14,7 @@ commands =
|
|||
pip install -e .[testing]
|
||||
# python -m pytest {posargs}
|
||||
|
||||
# TODO install via helper script..
|
||||
# my.location.google deps
|
||||
pip install geopy ijson
|
||||
|
||||
|
|
@ -34,12 +36,28 @@ commands =
|
|||
|
||||
|
||||
[testenv:demo]
|
||||
commands = ./demo.py
|
||||
commands =
|
||||
pip install git+https://github.com/karlicoss/hypexport
|
||||
./demo.py
|
||||
|
||||
|
||||
[testenv:mypy-core]
|
||||
whitelist_externals = cat
|
||||
commands =
|
||||
pip install -e .[testing,optional]
|
||||
pip install orgparse # used it core.orgparse?
|
||||
# todo add tests?
|
||||
python3 -m mypy -p my.core \
|
||||
--txt-report .coverage.mypy-core \
|
||||
--html-report .coverage.mypy-core \
|
||||
{posargs}
|
||||
cat .coverage.mypy-core/index.txt
|
||||
|
||||
|
||||
# specific modules that are known to be mypy compliant (to avoid false negatives)
|
||||
[testenv:mypy-modules]
|
||||
whitelist_externals = bash
|
||||
# todo maybe split into separate jobs? need to add comment how to run
|
||||
# and install dependencies via AST thing?
|
||||
[testenv:mypy-misc]
|
||||
commands =
|
||||
pip install -e .[testing,optional]
|
||||
pip install orgparse
|
||||
|
|
@ -51,13 +69,7 @@ commands =
|
|||
pip install git+https://github.com/karlicoss/rexport
|
||||
pip install git+https://github.com/karlicoss/stexport
|
||||
|
||||
# ugh fuck. soo... need to reset HOME, otherwise user's site-packages are somehow leaking into mypy's path...
|
||||
# see https://github.com/python/mypy/blob/f6fb60ef69738cbfe2dfe56c747eca8f03735d8e/mypy/modulefinder.py#L487
|
||||
# this is particularly annoying when user's config is leaking and mypy isn't running against the repository config
|
||||
# maybe this issue... https://github.com/tox-dev/tox/issues/838
|
||||
# and also since it's Tox, we can't just set an env variable for a single command, have to spawn a subshell. jeez.
|
||||
# TODO fuck. -p my.github isn't checking the subpackages?? wtf...
|
||||
bash -c 'HOME= \
|
||||
python3 -m mypy \
|
||||
-p my.endomondo \
|
||||
-p my.github.ghexport \
|
||||
|
|
@ -72,15 +84,19 @@ commands =
|
|||
-p my.location.google \
|
||||
-p my.time.tz.via_location \
|
||||
-p my.calendar.holidays \
|
||||
--txt-report .mypy-coverage \
|
||||
--html-report .mypy-coverage \
|
||||
{posargs}'
|
||||
--txt-report .coverage.mypy-misc \
|
||||
--html-report .coverage.mypy-misc \
|
||||
{posargs}
|
||||
# txt report is a bit more convenient to view on CI
|
||||
|
||||
# note: this comment doesn't seem relevant anymore, but keeping it in case the issue happens again
|
||||
# > ugh ... need to reset HOME, otherwise user's site-packages are somehow leaking into mypy's path...
|
||||
# > see https://github.com/python/mypy/blob/f6fb60ef69738cbfe2dfe56c747eca8f03735d8e/mypy/modulefinder.py#L487
|
||||
# > this is particularly annoying when user's config is leaking and mypy isn't running against the repository config
|
||||
|
||||
# ideally, keep core modules only here
|
||||
[testenv:mypy]
|
||||
skip_install = true
|
||||
commands =
|
||||
pip install -e .[testing,optional] orgparse
|
||||
./lint
|
||||
|
||||
# useful flags:
|
||||
# * sitepackages = true to inherit user/globally installed packages (default false)
|
||||
# * skip_install = true -- not sure when useful? (default false)
|
||||
# * -e to run specific subenvironment
|
||||
# * pass arguments with -- , e.g. `tox -e tests -- -k some_test_name` to only run one test with pytest
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue