diff --git a/demo.py b/demo.py index ae0ba06..3c08cce 100755 --- a/demo.py +++ b/demo.py @@ -3,6 +3,7 @@ from subprocess import check_call, DEVNULL from shutil import copy, copytree import os from os.path import abspath +from sys import executable as python from pathlib import Path my_repo = Path(__file__).absolute().parent @@ -18,12 +19,12 @@ def run(): # 2. prepare repositories you'd be using. For this demo we only set up Hypothesis tox = 'TOX' in os.environ if tox: # tox doesn't like --user flag - check_call('pip3 install git+https://github.com/karlicoss/hypexport.git'.split()) + check_call(f'{python} -m pip install git+https://github.com/karlicoss/hypexport.git'.split()) else: try: import hypexport except ModuleNotFoundError: - check_call('pip3 install --user git+https://github.com/karlicoss/hypexport.git'.split()) + check_call(f'{python} -m pip --user git+https://github.com/karlicoss/hypexport.git'.split()) # 3. prepare some demo Hypothesis data @@ -48,7 +49,7 @@ def run(): # 4. now we can use it! os.chdir(my_repo) - check_call(['python3', '-c', ''' + check_call([python, '-c', ''' import my.hypothesis pages = my.hypothesis.pages() @@ -106,13 +107,17 @@ def named_temp_dir(name: str): """ Fixed name tmp dir """ - td = (Path('/tmp') / name) + import tempfile + td = Path(tempfile.gettempdir()) / name try: td.mkdir(exist_ok=False) yield td finally: - import shutil - shutil.rmtree(str(td)) + import os, shutil + skip_cleanup = 'CI' in os.environ and os.name == 'nt' + # TODO hmm for some reason cleanup on windows causes AccessError + if not skip_cleanup: + shutil.rmtree(str(td)) def main(): diff --git a/my/config.py b/my/config.py index 7d31f5d..b841d31 100644 --- a/my/config.py +++ b/my/config.py @@ -19,7 +19,7 @@ from my.core import Paths, PathIsh class hypothesis: # expects outputs from https://github.com/karlicoss/hypexport # (it's just the standard Hypothes.is export format) - export_path: Paths = '/path/to/hypothesis/data' + export_path: Paths = r'/path/to/hypothesis/data' class instapaper: export_path: Paths = '' diff --git a/my/core/discovery_pure.py b/my/core/discovery_pure.py index 17a1976..dbd07b9 100644 --- a/my/core/discovery_pure.py +++ b/my/core/discovery_pure.py @@ -16,6 +16,7 @@ NOT_HPI_MODULE_VAR = '__NOT_HPI_MODULE__' ### import ast +import os from typing import Optional, Sequence, List, NamedTuple, Iterable, cast, Any from pathlib import Path import re @@ -151,7 +152,7 @@ def _modules_under_root(my_root: Path) -> Iterable[HPIModule]: mp = f.relative_to(my_root.parent) if mp.name == '__init__.py': mp = mp.parent - m = str(mp.with_suffix('')).replace('/', '.') + m = str(mp.with_suffix('')).replace(os.sep, '.') if ignored(m): continue a: ast.Module = ast.parse(f.read_text()) @@ -192,7 +193,7 @@ def test() -> None: def test_demo() -> None: demo = module_by_name('my.demo') assert demo.doc is not None - assert str(demo.file) == 'my/demo.py' + assert demo.file == Path('my', 'demo.py') assert demo.requires is None diff --git a/my/core/kompress.py b/my/core/kompress.py index 93a19a0..60b8b78 100644 --- a/my/core/kompress.py +++ b/my/core/kompress.py @@ -203,7 +203,9 @@ class ZipPath(ZipPathBase): def stat(self) -> os.stat_result: # NOTE: zip datetimes have no notion of time zone, usually they just keep local time? # see https://en.wikipedia.org/wiki/ZIP_(file_format)#Structure - dt = datetime(*self.root.getinfo(str(self.subpath)).date_time) + # note: seems that zip always uses forward slash, regardless OS? + zip_subpath = '/'.join(self.subpath.parts) + dt = datetime(*self.root.getinfo(zip_subpath).date_time) ts = int(dt.timestamp()) params = dict( st_mode=0, diff --git a/my/core/sqlite.py b/my/core/sqlite.py index 1b38869..5253607 100644 --- a/my/core/sqlite.py +++ b/my/core/sqlite.py @@ -48,4 +48,5 @@ def sqlite_copy_and_open(db: PathIsh) -> sqlite3.Connection: with sqlite3.connect(str(tdir / dp.name)) as conn: from .compat import sqlite_backup sqlite_backup(source=conn, dest=dest) + conn.close() return dest diff --git a/my/core/util.py b/my/core/util.py index a6204d9..0ffc3a7 100644 --- a/my/core/util.py +++ b/my/core/util.py @@ -229,9 +229,9 @@ def test_bad_modules(tmp_path: Path) -> None: (par / 'malicious.py').write_text(f''' from pathlib import Path -Path('{xx}').write_text('aaand your data is gone!') +Path(r'{xx}').write_text('aaand your data is gone!') -raise RuntimeError("FAIL ON IMPORT! naughy.") +raise RuntimeError("FAIL ON IMPORT! naughty.") def stats(): return [1, 2, 3] diff --git a/tests/cli.py b/tests/cli.py index 1e3c560..fce53b7 100644 --- a/tests/cli.py +++ b/tests/cli.py @@ -1,4 +1,14 @@ +import os from subprocess import check_call def test_lists_modules() -> None: - check_call(['hpi', 'modules']) + # hack PYTHONUTF8 for windows + # see https://github.com/karlicoss/promnesia/issues/274 + # https://memex.zulipchat.com/#narrow/stream/279600-promnesia/topic/indexing.3A.20utf8.28emoji.29.20filenames.20in.20Windows + # necessary for this test cause emooji is causing trouble + # TODO need to fix it properly + env = { + **os.environ, + 'PYTHONUTF8': '1', + } + check_call(['hpi', 'modules'], env=env) diff --git a/tests/commits.py b/tests/commits.py index ab4e2c7..1aa7aa0 100644 --- a/tests/commits.py +++ b/tests/commits.py @@ -1,9 +1,14 @@ -# TODO need fdfind on CI? from pathlib import Path from more_itertools import bucket import pytest +import os +pytestmark = pytest.mark.skipif( + os.name == 'nt', + reason='TODO figure out how to install fd-find on Windows', +) + def test() -> None: from my.coding.commits import commits diff --git a/tests/core/test_kompress.py b/tests/core/test_kompress.py index 481a025..97539cb 100644 --- a/tests/core/test_kompress.py +++ b/tests/core/test_kompress.py @@ -76,24 +76,25 @@ def test_zippath() -> None: hash(zp) assert zp.exists() - assert (zp / 'gdpr_export/comments').exists() + assert (zp / 'gdpr_export' / 'comments').exists() # check str constructor just in case - assert (ZipPath(str(target)) / 'gdpr_export/comments').exists() + assert (ZipPath(str(target)) / 'gdpr_export' / 'comments').exists() assert not (ZipPath(str(target)) / 'whatever').exists() matched = list(zp.rglob('*')) assert len(matched) > 0 assert all(p.filepath == target for p in matched), matched - rpaths = [str(p.relative_to(zp)) for p in matched] + rpaths = [p.relative_to(zp) for p in matched] + gdpr_export = Path('gdpr_export') assert rpaths == [ - 'gdpr_export', - 'gdpr_export/comments', - 'gdpr_export/comments/comments.json', - 'gdpr_export/profile', - 'gdpr_export/profile/settings.json', - 'gdpr_export/messages', - 'gdpr_export/messages/index.csv', + gdpr_export, + gdpr_export / 'comments', + gdpr_export / 'comments' / 'comments.json', + gdpr_export / 'profile', + gdpr_export / 'profile' / 'settings.json', + gdpr_export / 'messages', + gdpr_export / 'messages' / 'index.csv', ], rpaths @@ -103,14 +104,15 @@ def test_zippath() -> None: # same for this one # assert ZipPath(Path('test'), 'whatever').absolute() == ZipPath(Path('test').absolute(), 'whatever') - assert (ZipPath(target) / 'gdpr_export/comments').exists() + assert (ZipPath(target) / 'gdpr_export' / 'comments').exists() - jsons = [str(p.relative_to(zp / 'gdpr_export')) for p in zp.rglob('*.json')] + jsons = [p.relative_to(zp / 'gdpr_export') for p in zp.rglob('*.json')] assert jsons == [ - 'comments/comments.json', - 'profile/settings.json', + Path('comments','comments.json'), + Path('profile','settings.json'), ] + # NOTE: hmm interesting, seems that ZipPath is happy with forward slash regardless OS? assert list(zp.rglob('mes*')) == [ZipPath(target, 'gdpr_export/messages')] iterdir_res = list((zp / 'gdpr_export').iterdir()) @@ -118,7 +120,7 @@ def test_zippath() -> None: assert all(isinstance(p, Path) for p in iterdir_res) # date recorded in the zip archive - assert (zp / 'gdpr_export/comments/comments.json').stat().st_mtime > 1625000000 + assert (zp / 'gdpr_export' / 'comments' / 'comments.json').stat().st_mtime > 1625000000 # TODO ugh. # unzip -l shows the date as 2021-07-01 09:43 # however, python reads it as 2021-07-01 01:43 ?? diff --git a/tests/sqlite.py b/tests/sqlite.py index 1b423da..f80636e 100644 --- a/tests/sqlite.py +++ b/tests/sqlite.py @@ -43,20 +43,24 @@ def _test_do_copy(db: Path) -> None: shutil.copy(db, cdb) with sqlite3.connect(str(cdb)) as conn_copy: assert len(list(conn_copy.execute('SELECT * FROM testtable'))) == 5 + conn_copy.close() def _test_do_immutable(db: Path) -> None: # in readonly mode doesn't touch with sqlite_connect_immutable(db) as conn_imm: assert len(list(conn_imm.execute('SELECT * FROM testtable'))) == 5 + conn_imm.close() def _test_do_copy_and_open(db: Path) -> None: with sqlite_copy_and_open(db) as conn_mem: assert len(list(conn_mem.execute('SELECT * FROM testtable'))) == 10 + conn_mem.close() def _test_open_asis(db: Path) -> None: # NOTE: this also works... but leaves some potential for DB corruption with sqlite3.connect(str(db)) as conn_db_2: assert len(list(conn_db_2.execute('SELECT * FROM testtable'))) == 10 + conn_db_2.close() diff --git a/tox.ini b/tox.ini index 8c22eb3..33c2c71 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ passenv = CI CI_* [testenv:tests-core] commands = pip install -e .[testing] - python3 -m pytest \ + {envpython} -m pytest \ tests/core.py \ tests/sqlite.py \ tests/get_files.py \ @@ -29,7 +29,7 @@ commands = # installed to test my.core.serialize while using simplejson and not orjson pip install simplejson - python3 -m pytest \ + {envpython} -m pytest \ tests/serialize_simplejson.py \ {posargs} @@ -52,28 +52,28 @@ commands = hpi module install my.reddit.rexport - python3 -m pytest tests \ + {envpython} -m pytest tests \ # ignore some tests which might take a while to run on ci.. --ignore tests/takeout.py \ --ignore tests/extra/polar.py \ # dont run simplejson compatibility test since orjson is now installed - --ignore tests/serialize_simplejson.py + --ignore tests/serialize_simplejson.py \ {posargs} [testenv:demo] commands = pip install git+https://github.com/karlicoss/hypexport - ./demo.py + {envpython} ./demo.py [testenv:mypy-core] -whitelist_externals = cat +allowlist_externals = cat commands = pip install -e .[testing,optional] pip install orgparse # used it core.orgmode? # todo add tests? - python3 -m mypy --install-types --non-interactive \ + {envpython} -m mypy --install-types --non-interactive \ -p my.core \ --txt-report .coverage.mypy-core \ --html-report .coverage.mypy-core \ @@ -109,7 +109,7 @@ commands = # todo fuck. -p my.github isn't checking the subpackages?? wtf... # guess it wants .pyi file?? - python3 -m mypy --install-types --non-interactive \ + {envpython} -m mypy --install-types --non-interactive \ -p my.browser \ -p my.endomondo \ -p my.github.ghexport \