CI: clean up tox config a bit, get rid of custom lint script

.github/workflows/main.yml

@@ -40,8 +40,12 @@ jobs:
 
     - uses: actions/upload-artifact@v2
       with:
-        name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }}
+        name: .coverage.mypy-misc_${{ matrix.platform }}_${{ matrix.python-version }}
-        path: .coverage.mypy/
+        path: .coverage.mypy-misc/
+    - uses: actions/upload-artifact@v2
+      with:
+        name: .coverage.mypy-core_${{ matrix.platform }}_${{ matrix.python-version }}
+        path: .coverage.mypy-core/
 
   pypi:
     runs-on: ubuntu-latest

.projectile

@@ -1 +0,0 @@
-- /.mypy_cache/

doc/DEVELOPMENT.org

@@ -26,6 +26,11 @@ However, Pycharm/Emacs or whatever IDE you are using won't be able to figure tha
   i.e. create a new interpreter configuration (e.g. name it "Python 3.7 (for HPI)"), and add =~/.config/my=.
 
 * Linting
-You should be able to use [[file:../lint]] script to run mypy checks.
+~tox~ should run all test, mypy, etc.
 
-[[file:../mypy.ini]] points at =~/.config/my= by default.
+If you want to run some specific parts/tests, consult [[file:tox.ini]].
+
+Some useful flags (look them up):
+
+- ~-e~ flag for tox
+- ~-k~ flag for pytest

lint

@@ -1,105 +0,0 @@
-#!/usr/bin/env python3
-from pathlib import Path
-from pprint import pprint
-from itertools import chain
-from subprocess import check_call, run, PIPE
-import sys
-import os
-from typing import List, Optional, Iterable
-
-
-def log(*args):
-    print(*args, file=sys.stderr)
-
-CI = 'CI' in os.environ
-
-DIR = Path(__file__).absolute().parent
-
-# hmm. I guess I need to check all subpackages separately
-# otherwise pylint doesn't work and mypy doesn't discover everything
-
-# TODO could reuse in readme??
-# returns None if not a package
-def package_name(p: Path) -> str:
-    def mname(p: Path):
-        nosuf = p.with_suffix('')
-        return str(nosuf).replace('/', '.')
-
-    has_init =  (p.parent / '__init__.py').exists()
-    if has_init:
-        return mname(p.parent)
-    else:
-        return mname(p)
-
-def subpackages(package: str) -> Iterable[str]:
-    ppath = package.replace('.', '/')
-    yield from sorted({
-        package_name(p.relative_to(DIR)) for p in (DIR / ppath).rglob('*.py')
-    })
-
-
-# TODO meh.. think how to check _everything_ on CI
-def core_modules() -> Iterable[str]:
-    return [
-        *subpackages('my.core'),
-        'my.config',
-        'my.cfg',
-        'tests/misc.py',
-        'tests/get_files.py',
-        # 'tests/config.py', TODO hmm. unclear how to type check this module
-    ]
-
-
-
-def all_modules() -> Iterable[str]:
-    yield from subpackages('my')
-    yield from sorted(
-        str(f.relative_to(DIR)) for f in (DIR / 'tests').rglob('*.py')
-    )
-
-
-def pylint():
-    # TODO ugh. pylint still doesn't like checking my.config or my.books
-    # only top level .py files seem ok??
-    pass
-
-
-def mypy(thing: str):
-    is_package = Path(thing).suffix != '.py'
-    cmd = [
-        'mypy',
-        '--color-output', # TODO eh? doesn't work..
-        *(['-p'] if is_package else []), thing,
-    ]
-    print(' '.join(cmd), file=sys.stderr)
-    return run(cmd, stdout=PIPE, stderr=PIPE)
-
-
-def mypy_all() -> Iterable[Exception]:
-    from concurrent.futures import ThreadPoolExecutor
-
-    pkgs = list(core_modules() if CI else all_modules())
-    log(f"Checking {pkgs}")
-    with ThreadPoolExecutor() as pool:
-        for p, res in zip(pkgs, pool.map(mypy, pkgs)):
-            ret = res.returncode
-            if ret > 0:
-                log(f'FAILED: {p}')
-            else:
-                log(f'OK: {p}')
-            print(res.stdout.decode('utf8'))
-            print(res.stderr.decode('utf8'), file=sys.stderr)
-            try:
-                res.check_returncode()
-            except Exception as e:
-                yield e
-
-
-def main():
-    errors = list(mypy_all())
-    if len(errors) > 0:
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    main()

setup.py

@@ -46,11 +46,10 @@ def main():
         extras_require={
             'testing': [
                 'pytest',
-                'pylint',
                 'mypy',
                 'lxml', # for mypy coverage
 
-                # used in some tests
+                # used in some tests.. although shouldn't rely on it
                 'pandas',
             ],
             'optional': [

tox.ini

@@ -1,10 +1,11 @@
 [tox]
 minversion = 3.5
-envlist = py3,mypy,mypy-modules
 
-# TODO ugh. unclear how to reuse setup.cfg deps in tox
 [testenv]
 passenv = CI CI_*
+
+# TODO ugh. unclear how to reuse setup.cfg deps in tox
+[testenv:tests]
 # deliberately set to nonexistent path to check the fallback logic
 setenv = MY_CONFIG = nonexistent
 commands =
@@ -13,6 +14,7 @@ commands =
     pip install -e .[testing]
     # python -m pytest {posargs}
 
+    # TODO install via helper script..
     # my.location.google deps
     pip install geopy ijson
 
@@ -34,12 +36,28 @@ commands =
 
 
 [testenv:demo]
-commands = ./demo.py
+commands =
+    pip install git+https://github.com/karlicoss/hypexport
+    ./demo.py
+
+
+[testenv:mypy-core]
+whitelist_externals = cat
+commands =
+    pip install -e .[testing,optional]
+    pip install orgparse # used it core.orgparse?
+    # todo add tests?
+    python3 -m mypy -p my.core \
+                    --txt-report  .coverage.mypy-core \
+                    --html-report .coverage.mypy-core \
+                    {posargs}
+    cat .coverage.mypy-core/index.txt
 
 
 # specific modules that are known to be mypy compliant (to avoid false negatives)
-[testenv:mypy-modules]
+# todo maybe split into separate jobs? need to add comment how to run
-whitelist_externals = bash
+# and install dependencies via AST thing?
+[testenv:mypy-misc]
 commands =
     pip install -e .[testing,optional]
     pip install orgparse
@@ -51,13 +69,7 @@ commands =
     pip install git+https://github.com/karlicoss/rexport
     pip install git+https://github.com/karlicoss/stexport
 
-    # ugh fuck. soo... need to reset HOME, otherwise user's site-packages are somehow leaking into mypy's path...
-    # see https://github.com/python/mypy/blob/f6fb60ef69738cbfe2dfe56c747eca8f03735d8e/mypy/modulefinder.py#L487
-    # this is particularly annoying when user's config is leaking and mypy isn't running against the repository config
-    # maybe this issue...  https://github.com/tox-dev/tox/issues/838
-    # and also since it's Tox, we can't just set an env variable for a single command, have to spawn a subshell. jeez.
     # TODO fuck. -p my.github isn't checking the subpackages?? wtf...
-    bash -c 'HOME= \
     python3 -m mypy \
                     -p my.endomondo                   \
                     -p my.github.ghexport             \
@@ -72,15 +84,19 @@ commands =
                     -p my.location.google             \
                     -p my.time.tz.via_location        \
                     -p my.calendar.holidays           \
-                    --txt-report  .mypy-coverage      \
+                    --txt-report  .coverage.mypy-misc \
-                    --html-report .mypy-coverage      \
+                    --html-report .coverage.mypy-misc \
-                    {posargs}'
+                    {posargs}
     # txt report is a bit more convenient to view on CI
 
+# note: this comment doesn't seem relevant anymore, but keeping it in case the issue happens again
+# > ugh ... need to reset HOME, otherwise user's site-packages are somehow leaking into mypy's path...
+# > see https://github.com/python/mypy/blob/f6fb60ef69738cbfe2dfe56c747eca8f03735d8e/mypy/modulefinder.py#L487
+# > this is particularly annoying when user's config is leaking and mypy isn't running against the repository config
 
-# ideally, keep core modules only here
+
-[testenv:mypy]
+# useful flags:
-skip_install = true
+# * sitepackages = true to inherit user/globally installed packages (default false)
-commands =
+# * skip_install = true -- not sure when useful? (default false)
-    pip install -e .[testing,optional] orgparse
+# * -e to run specific subenvironment
-    ./lint
+# * pass arguments with -- , e.g. `tox -e tests -- -k some_test_name` to only run one test with pytest