From 56d5587c209dcbd27c7802d60c0bc8e8e2391672 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 17 Feb 2021 21:41:18 +0000 Subject: [PATCH] CI: clean up tox config a bit, get rid of custom lint script --- .github/workflows/main.yml | 8 ++- .projectile | 1 - doc/DEVELOPMENT.org | 9 +++- lint | 105 ------------------------------------- setup.py | 3 +- tox.ini | 56 +++++++++++++------- 6 files changed, 50 insertions(+), 132 deletions(-) delete mode 100644 .projectile delete mode 100755 lint diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f55feba..31d97c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,8 +40,12 @@ jobs: - uses: actions/upload-artifact@v2 with: - name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} - path: .coverage.mypy/ + name: .coverage.mypy-misc_${{ matrix.platform }}_${{ matrix.python-version }} + path: .coverage.mypy-misc/ + - uses: actions/upload-artifact@v2 + with: + name: .coverage.mypy-core_${{ matrix.platform }}_${{ matrix.python-version }} + path: .coverage.mypy-core/ pypi: runs-on: ubuntu-latest diff --git a/.projectile b/.projectile deleted file mode 100644 index 89335b7..0000000 --- a/.projectile +++ /dev/null @@ -1 +0,0 @@ -- /.mypy_cache/ diff --git a/doc/DEVELOPMENT.org b/doc/DEVELOPMENT.org index 3f5352b..d60f5c9 100644 --- a/doc/DEVELOPMENT.org +++ b/doc/DEVELOPMENT.org @@ -26,6 +26,11 @@ However, Pycharm/Emacs or whatever IDE you are using won't be able to figure tha i.e. create a new interpreter configuration (e.g. name it "Python 3.7 (for HPI)"), and add =~/.config/my=. * Linting -You should be able to use [[file:../lint]] script to run mypy checks. +~tox~ should run all test, mypy, etc. -[[file:../mypy.ini]] points at =~/.config/my= by default. +If you want to run some specific parts/tests, consult [[file:tox.ini]]. + +Some useful flags (look them up): + +- ~-e~ flag for tox +- ~-k~ flag for pytest diff --git a/lint b/lint deleted file mode 100755 index 535a9aa..0000000 --- a/lint +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -from pathlib import Path -from pprint import pprint -from itertools import chain -from subprocess import check_call, run, PIPE -import sys -import os -from typing import List, Optional, Iterable - - -def log(*args): - print(*args, file=sys.stderr) - -CI = 'CI' in os.environ - -DIR = Path(__file__).absolute().parent - -# hmm. I guess I need to check all subpackages separately -# otherwise pylint doesn't work and mypy doesn't discover everything - -# TODO could reuse in readme?? -# returns None if not a package -def package_name(p: Path) -> str: - def mname(p: Path): - nosuf = p.with_suffix('') - return str(nosuf).replace('/', '.') - - has_init = (p.parent / '__init__.py').exists() - if has_init: - return mname(p.parent) - else: - return mname(p) - -def subpackages(package: str) -> Iterable[str]: - ppath = package.replace('.', '/') - yield from sorted({ - package_name(p.relative_to(DIR)) for p in (DIR / ppath).rglob('*.py') - }) - - -# TODO meh.. think how to check _everything_ on CI -def core_modules() -> Iterable[str]: - return [ - *subpackages('my.core'), - 'my.config', - 'my.cfg', - 'tests/misc.py', - 'tests/get_files.py', - # 'tests/config.py', TODO hmm. unclear how to type check this module - ] - - - -def all_modules() -> Iterable[str]: - yield from subpackages('my') - yield from sorted( - str(f.relative_to(DIR)) for f in (DIR / 'tests').rglob('*.py') - ) - - -def pylint(): - # TODO ugh. pylint still doesn't like checking my.config or my.books - # only top level .py files seem ok?? - pass - - -def mypy(thing: str): - is_package = Path(thing).suffix != '.py' - cmd = [ - 'mypy', - '--color-output', # TODO eh? doesn't work.. - *(['-p'] if is_package else []), thing, - ] - print(' '.join(cmd), file=sys.stderr) - return run(cmd, stdout=PIPE, stderr=PIPE) - - -def mypy_all() -> Iterable[Exception]: - from concurrent.futures import ThreadPoolExecutor - - pkgs = list(core_modules() if CI else all_modules()) - log(f"Checking {pkgs}") - with ThreadPoolExecutor() as pool: - for p, res in zip(pkgs, pool.map(mypy, pkgs)): - ret = res.returncode - if ret > 0: - log(f'FAILED: {p}') - else: - log(f'OK: {p}') - print(res.stdout.decode('utf8')) - print(res.stderr.decode('utf8'), file=sys.stderr) - try: - res.check_returncode() - except Exception as e: - yield e - - -def main(): - errors = list(mypy_all()) - if len(errors) > 0: - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index 0c913a5..c06bdf0 100644 --- a/setup.py +++ b/setup.py @@ -46,11 +46,10 @@ def main(): extras_require={ 'testing': [ 'pytest', - 'pylint', 'mypy', 'lxml', # for mypy coverage - # used in some tests + # used in some tests.. although shouldn't rely on it 'pandas', ], 'optional': [ diff --git a/tox.ini b/tox.ini index 80e5382..4844f5b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,11 @@ [tox] minversion = 3.5 -envlist = py3,mypy,mypy-modules -# TODO ugh. unclear how to reuse setup.cfg deps in tox [testenv] passenv = CI CI_* + +# TODO ugh. unclear how to reuse setup.cfg deps in tox +[testenv:tests] # deliberately set to nonexistent path to check the fallback logic setenv = MY_CONFIG = nonexistent commands = @@ -13,6 +14,7 @@ commands = pip install -e .[testing] # python -m pytest {posargs} + # TODO install via helper script.. # my.location.google deps pip install geopy ijson @@ -34,12 +36,28 @@ commands = [testenv:demo] -commands = ./demo.py +commands = + pip install git+https://github.com/karlicoss/hypexport + ./demo.py + + +[testenv:mypy-core] +whitelist_externals = cat +commands = + pip install -e .[testing,optional] + pip install orgparse # used it core.orgparse? + # todo add tests? + python3 -m mypy -p my.core \ + --txt-report .coverage.mypy-core \ + --html-report .coverage.mypy-core \ + {posargs} + cat .coverage.mypy-core/index.txt # specific modules that are known to be mypy compliant (to avoid false negatives) -[testenv:mypy-modules] -whitelist_externals = bash +# todo maybe split into separate jobs? need to add comment how to run +# and install dependencies via AST thing? +[testenv:mypy-misc] commands = pip install -e .[testing,optional] pip install orgparse @@ -51,13 +69,7 @@ commands = pip install git+https://github.com/karlicoss/rexport pip install git+https://github.com/karlicoss/stexport - # ugh fuck. soo... need to reset HOME, otherwise user's site-packages are somehow leaking into mypy's path... - # see https://github.com/python/mypy/blob/f6fb60ef69738cbfe2dfe56c747eca8f03735d8e/mypy/modulefinder.py#L487 - # this is particularly annoying when user's config is leaking and mypy isn't running against the repository config - # maybe this issue... https://github.com/tox-dev/tox/issues/838 - # and also since it's Tox, we can't just set an env variable for a single command, have to spawn a subshell. jeez. # TODO fuck. -p my.github isn't checking the subpackages?? wtf... - bash -c 'HOME= \ python3 -m mypy \ -p my.endomondo \ -p my.github.ghexport \ @@ -72,15 +84,19 @@ commands = -p my.location.google \ -p my.time.tz.via_location \ -p my.calendar.holidays \ - --txt-report .mypy-coverage \ - --html-report .mypy-coverage \ - {posargs}' + --txt-report .coverage.mypy-misc \ + --html-report .coverage.mypy-misc \ + {posargs} # txt report is a bit more convenient to view on CI +# note: this comment doesn't seem relevant anymore, but keeping it in case the issue happens again +# > ugh ... need to reset HOME, otherwise user's site-packages are somehow leaking into mypy's path... +# > see https://github.com/python/mypy/blob/f6fb60ef69738cbfe2dfe56c747eca8f03735d8e/mypy/modulefinder.py#L487 +# > this is particularly annoying when user's config is leaking and mypy isn't running against the repository config -# ideally, keep core modules only here -[testenv:mypy] -skip_install = true -commands = - pip install -e .[testing,optional] orgparse - ./lint + +# useful flags: +# * sitepackages = true to inherit user/globally installed packages (default false) +# * skip_install = true -- not sure when useful? (default false) +# * -e to run specific subenvironment +# * pass arguments with -- , e.g. `tox -e tests -- -k some_test_name` to only run one test with pytest