core/ci: fix windows-specific issues

- use portable separators
- paths should be prepended with r' (so backwards slash isn't treated as escaping)
- sqlite connections should be closed (otherwise windows fails to remove the underlying db file)
- workaround for emojis via PYTHONUTF8=1 test for now
- make ZipPath portable
- properly use tox python environment everywhere

  this was causing issues on Windows
  e.g.
      WARNING: test command found but not installed in testenv
        cmd: C:\hostedtoolcache\windows\Python\3.9.12\x64\python3.EXE

demo.py

@@ -3,6 +3,7 @@ from subprocess import check_call, DEVNULL
 from shutil import copy, copytree
 import os
 from os.path import abspath
+from sys import executable as python
 from pathlib import Path
 
 my_repo = Path(__file__).absolute().parent
@@ -18,12 +19,12 @@ def run():
     # 2. prepare repositories you'd be using. For this demo we only set up Hypothesis
     tox = 'TOX' in os.environ
     if tox: # tox doesn't like --user flag
-        check_call('pip3 install        git+https://github.com/karlicoss/hypexport.git'.split())
+        check_call(f'{python} -m pip install        git+https://github.com/karlicoss/hypexport.git'.split())
     else:
         try:
             import hypexport
         except ModuleNotFoundError:
-            check_call('pip3 install --user git+https://github.com/karlicoss/hypexport.git'.split())
+            check_call(f'{python} -m pip --user git+https://github.com/karlicoss/hypexport.git'.split())
 
 
     # 3. prepare some demo Hypothesis data
@@ -48,7 +49,7 @@ def run():
     # 4. now we can use it!
     os.chdir(my_repo)
 
-    check_call(['python3', '-c', '''
+    check_call([python, '-c', '''
 import my.hypothesis
 
 pages = my.hypothesis.pages()
@@ -106,12 +107,16 @@ def named_temp_dir(name: str):
     """
     Fixed name tmp dir
     """
-    td = (Path('/tmp') / name)
+    import tempfile
+    td = Path(tempfile.gettempdir()) / name
     try:
         td.mkdir(exist_ok=False)
         yield td
     finally:
-        import shutil
+        import os, shutil
+        skip_cleanup = 'CI' in os.environ and os.name == 'nt'
+        # TODO hmm for some reason cleanup on windows causes AccessError
+        if not skip_cleanup:
             shutil.rmtree(str(td))
 
 

my/config.py

@@ -19,7 +19,7 @@ from my.core import Paths, PathIsh
 class hypothesis:
     # expects outputs from https://github.com/karlicoss/hypexport
     # (it's just the standard Hypothes.is export format)
-    export_path: Paths = '/path/to/hypothesis/data'
+    export_path: Paths = r'/path/to/hypothesis/data'
 
 class instapaper:
     export_path: Paths = ''

my/core/discovery_pure.py

@@ -16,6 +16,7 @@ NOT_HPI_MODULE_VAR = '__NOT_HPI_MODULE__'
 ###
 
 import ast
+import os
 from typing import Optional, Sequence, List, NamedTuple, Iterable, cast, Any
 from pathlib import Path
 import re
@@ -151,7 +152,7 @@ def _modules_under_root(my_root: Path) -> Iterable[HPIModule]:
         mp = f.relative_to(my_root.parent)
         if mp.name == '__init__.py':
             mp = mp.parent
-        m = str(mp.with_suffix('')).replace('/', '.')
+        m = str(mp.with_suffix('')).replace(os.sep, '.')
         if ignored(m):
             continue
         a: ast.Module = ast.parse(f.read_text())
@@ -192,7 +193,7 @@ def test() -> None:
 def test_demo() -> None:
     demo = module_by_name('my.demo')
     assert demo.doc is not None
-    assert str(demo.file) == 'my/demo.py'
+    assert demo.file == Path('my', 'demo.py')
     assert demo.requires is None
 
 

my/core/kompress.py

@@ -203,7 +203,9 @@ class ZipPath(ZipPathBase):
     def stat(self) -> os.stat_result:
         # NOTE: zip datetimes have no notion of time zone, usually they just keep local time?
         # see https://en.wikipedia.org/wiki/ZIP_(file_format)#Structure
-        dt = datetime(*self.root.getinfo(str(self.subpath)).date_time)
+        # note: seems that zip always uses forward slash, regardless OS?
+        zip_subpath = '/'.join(self.subpath.parts)
+        dt = datetime(*self.root.getinfo(zip_subpath).date_time)
         ts = int(dt.timestamp())
         params = dict(
             st_mode=0,

my/core/sqlite.py

@@ -48,4 +48,5 @@ def sqlite_copy_and_open(db: PathIsh) -> sqlite3.Connection:
         with sqlite3.connect(str(tdir / dp.name)) as conn:
             from .compat import sqlite_backup
             sqlite_backup(source=conn, dest=dest)
+        conn.close()
     return dest

my/core/util.py

@@ -229,9 +229,9 @@ def test_bad_modules(tmp_path: Path) -> None:
 
     (par / 'malicious.py').write_text(f'''
 from pathlib import Path
-Path('{xx}').write_text('aaand your data is gone!')
+Path(r'{xx}').write_text('aaand your data is gone!')
 
-raise RuntimeError("FAIL ON IMPORT! naughy.")
+raise RuntimeError("FAIL ON IMPORT! naughty.")
 
 def stats():
     return [1, 2, 3]

tests/cli.py

@@ -1,4 +1,14 @@
+import os
 from subprocess import check_call
 
 def test_lists_modules() -> None:
-    check_call(['hpi', 'modules'])
+    # hack PYTHONUTF8 for windows
+    # see  https://github.com/karlicoss/promnesia/issues/274
+    # https://memex.zulipchat.com/#narrow/stream/279600-promnesia/topic/indexing.3A.20utf8.28emoji.29.20filenames.20in.20Windows
+    # necessary for this test cause emooji is causing trouble
+    # TODO need to fix it properly
+    env = {
+        **os.environ,
+        'PYTHONUTF8': '1',
+    }
+    check_call(['hpi', 'modules'], env=env)

tests/commits.py

@@ -1,9 +1,14 @@
-# TODO need fdfind on CI?
 from pathlib import Path
 
 from more_itertools import bucket
 import pytest
 
+import os
+pytestmark = pytest.mark.skipif(
+    os.name == 'nt',
+    reason='TODO figure out how to install fd-find on Windows',
+)
+
 
 def test() -> None:
     from my.coding.commits import commits

tests/core/test_kompress.py

@@ -76,24 +76,25 @@ def test_zippath() -> None:
     hash(zp)
 
     assert zp.exists()
-    assert (zp / 'gdpr_export/comments').exists()
+    assert (zp / 'gdpr_export' / 'comments').exists()
     # check str constructor just in case
-    assert (ZipPath(str(target)) / 'gdpr_export/comments').exists()
+    assert (ZipPath(str(target)) / 'gdpr_export' / 'comments').exists()
     assert not (ZipPath(str(target)) / 'whatever').exists()
 
     matched = list(zp.rglob('*'))
     assert len(matched) > 0
     assert all(p.filepath == target for p in matched), matched
 
-    rpaths = [str(p.relative_to(zp)) for p in matched]
+    rpaths = [p.relative_to(zp) for p in matched]
+    gdpr_export = Path('gdpr_export')
     assert rpaths == [
-        'gdpr_export',
+        gdpr_export,
-        'gdpr_export/comments',
+        gdpr_export / 'comments',
-        'gdpr_export/comments/comments.json',
+        gdpr_export / 'comments' / 'comments.json',
-        'gdpr_export/profile',
+        gdpr_export / 'profile',
-        'gdpr_export/profile/settings.json',
+        gdpr_export / 'profile' / 'settings.json',
-        'gdpr_export/messages',
+        gdpr_export / 'messages',
-        'gdpr_export/messages/index.csv',
+        gdpr_export / 'messages' / 'index.csv',
     ], rpaths
 
 
@@ -103,14 +104,15 @@ def test_zippath() -> None:
     # same for this one
     # assert ZipPath(Path('test'), 'whatever').absolute() == ZipPath(Path('test').absolute(), 'whatever')
 
-    assert (ZipPath(target) / 'gdpr_export/comments').exists()
+    assert (ZipPath(target) / 'gdpr_export' / 'comments').exists()
 
-    jsons = [str(p.relative_to(zp / 'gdpr_export')) for p in zp.rglob('*.json')]
+    jsons = [p.relative_to(zp / 'gdpr_export') for p in zp.rglob('*.json')]
     assert jsons == [
-        'comments/comments.json',
+        Path('comments','comments.json'),
-        'profile/settings.json',
+        Path('profile','settings.json'),
     ]
 
+    # NOTE: hmm interesting, seems that ZipPath is happy with forward slash regardless OS?
     assert list(zp.rglob('mes*')) == [ZipPath(target, 'gdpr_export/messages')]
 
     iterdir_res = list((zp / 'gdpr_export').iterdir())
@@ -118,7 +120,7 @@ def test_zippath() -> None:
     assert all(isinstance(p, Path) for p in iterdir_res)
 
     # date recorded in the zip archive
-    assert (zp / 'gdpr_export/comments/comments.json').stat().st_mtime > 1625000000
+    assert (zp / 'gdpr_export' / 'comments' / 'comments.json').stat().st_mtime > 1625000000
     # TODO ugh.
     # unzip -l shows the date  as 2021-07-01 09:43
     # however, python reads it as 2021-07-01 01:43 ??

tests/sqlite.py

@@ -43,20 +43,24 @@ def _test_do_copy(db: Path) -> None:
         shutil.copy(db, cdb)
         with sqlite3.connect(str(cdb)) as conn_copy:
             assert len(list(conn_copy.execute('SELECT * FROM testtable'))) == 5
+        conn_copy.close()
 
 
 def _test_do_immutable(db: Path) -> None:
     # in readonly mode doesn't touch
     with sqlite_connect_immutable(db) as conn_imm:
         assert len(list(conn_imm.execute('SELECT * FROM testtable'))) == 5
+    conn_imm.close()
 
 
 def _test_do_copy_and_open(db: Path) -> None:
     with sqlite_copy_and_open(db) as conn_mem:
         assert len(list(conn_mem.execute('SELECT * FROM testtable'))) == 10
+    conn_mem.close()
 
 
 def _test_open_asis(db: Path) -> None:
     # NOTE: this also works... but leaves some potential for DB corruption
     with sqlite3.connect(str(db)) as conn_db_2:
         assert len(list(conn_db_2.execute('SELECT * FROM testtable'))) == 10
+    conn_db_2.close()

tox.ini

@@ -12,7 +12,7 @@ passenv = CI CI_*
 [testenv:tests-core]
 commands =
     pip install -e .[testing]
-    python3 -m pytest \
+    {envpython} -m pytest \
                       tests/core.py      \
                       tests/sqlite.py    \
                       tests/get_files.py \
@@ -29,7 +29,7 @@ commands =
 
     # installed to test my.core.serialize while using simplejson and not orjson
     pip install simplejson
-    python3 -m pytest \
+    {envpython} -m pytest \
                       tests/serialize_simplejson.py \
                       {posargs}
 
@@ -52,28 +52,28 @@ commands =
 
     hpi module install my.reddit.rexport
 
-    python3 -m pytest tests \
+    {envpython} -m pytest tests \
             # ignore some tests which might take a while to run on ci..
             --ignore tests/takeout.py        \
             --ignore tests/extra/polar.py \
             # dont run simplejson compatibility test since orjson is now installed
-            --ignore tests/serialize_simplejson.py
+            --ignore tests/serialize_simplejson.py \
             {posargs}
 
 
 [testenv:demo]
 commands =
     pip install git+https://github.com/karlicoss/hypexport
-    ./demo.py
+    {envpython} ./demo.py
 
 
 [testenv:mypy-core]
-whitelist_externals = cat
+allowlist_externals = cat
 commands =
     pip install -e .[testing,optional]
     pip install orgparse # used it core.orgmode?
     # todo add tests?
-    python3 -m mypy --install-types --non-interactive \
+    {envpython} -m mypy --install-types --non-interactive \
                     -p my.core \
                     --txt-report  .coverage.mypy-core \
                     --html-report .coverage.mypy-core \
@@ -109,7 +109,7 @@ commands =
 
     # todo fuck. -p my.github isn't checking the subpackages?? wtf...
     # guess it wants .pyi file??
-    python3 -m mypy --install-types --non-interactive \
+    {envpython} -m mypy --install-types --non-interactive \
                     -p my.browser                     \
                     -p my.endomondo                   \
                     -p my.github.ghexport             \