general: move reddit tests into my/tests + tweak my.core.cfg to be more reliable
This commit is contained in:
parent
fcfc423a75
commit
f8cd31044e
21 changed files with 77 additions and 54 deletions
|
@ -1,3 +1,5 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import TypeVar, Type, Callable, Dict, Any
|
from typing import TypeVar, Type, Callable, Dict, Any
|
||||||
|
|
||||||
Attrs = Dict[str, Any]
|
Attrs = Dict[str, Any]
|
||||||
|
@ -46,24 +48,29 @@ def _override_config(config: F) -> Iterator[F]:
|
||||||
|
|
||||||
import importlib
|
import importlib
|
||||||
import sys
|
import sys
|
||||||
from typing import Optional, Set
|
from typing import Optional
|
||||||
ModuleRegex = str
|
ModuleRegex = str
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _reload_modules(modules: ModuleRegex) -> Iterator[None]:
|
def _reload_modules(modules: ModuleRegex) -> Iterator[None]:
|
||||||
def loaded_modules() -> Set[str]:
|
# need to use list here, otherwise reordering with set might mess things up
|
||||||
return {name for name in sys.modules if re.fullmatch(modules, name)}
|
def loaded_modules() -> list[str]:
|
||||||
|
return [name for name in sys.modules if re.fullmatch(modules, name)]
|
||||||
|
|
||||||
modules_before = loaded_modules()
|
modules_before = loaded_modules()
|
||||||
|
|
||||||
for m in modules_before:
|
# uhh... seems that reversed might make more sense -- not 100% sure why, but this works for tests/reddit.py
|
||||||
|
for m in reversed(modules_before):
|
||||||
|
# ugh... seems that reload works whereas pop doesn't work in some cases (e.g. on tests/reddit.py)
|
||||||
|
# sys.modules.pop(m, None)
|
||||||
importlib.reload(sys.modules[m])
|
importlib.reload(sys.modules[m])
|
||||||
|
|
||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
modules_after = loaded_modules()
|
modules_after = loaded_modules()
|
||||||
|
modules_before_set = set(modules_before)
|
||||||
for m in modules_after:
|
for m in modules_after:
|
||||||
if m in modules_before:
|
if m in modules_before_set:
|
||||||
# was previously loaded, so need to reload to pick up old config
|
# was previously loaded, so need to reload to pick up old config
|
||||||
importlib.reload(sys.modules[m])
|
importlib.reload(sys.modules[m])
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -1,27 +1,29 @@
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
V = 'HPI_TESTS_KARLICOSS'
|
V = 'HPI_TESTS_KARLICOSS'
|
||||||
|
|
||||||
skip_if_not_karlicoss = pytest.mark.skipif(
|
skip_if_not_karlicoss = pytest.mark.skipif(
|
||||||
V not in os.environ, reason=f'test only works on @karlicoss data for now. Set evn variable {V}=true to override.',
|
V not in os.environ,
|
||||||
|
reason=f'test only works on @karlicoss data for now. Set evn variable {V}=true to override.',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def reset_modules() -> None:
|
def reset_modules() -> None:
|
||||||
'''
|
'''
|
||||||
A hack to 'unload' HPI modules, otherwise some modules might cache the config
|
A hack to 'unload' HPI modules, otherwise some modules might cache the config
|
||||||
TODO: a bit crap, need a better way..
|
TODO: a bit crap, need a better way..
|
||||||
'''
|
'''
|
||||||
import sys
|
|
||||||
import re
|
|
||||||
to_unload = [m for m in sys.modules if re.match(r'my[.]?', m)]
|
to_unload = [m for m in sys.modules if re.match(r'my[.]?', m)]
|
||||||
for m in to_unload:
|
for m in to_unload:
|
||||||
del sys.modules[m]
|
del sys.modules[m]
|
||||||
|
|
||||||
|
|
||||||
def testdata() -> Path:
|
def testdata() -> Path:
|
||||||
d = Path(__file__).absolute().parent.parent / 'testdata'
|
d = Path(__file__).absolute().parent.parent.parent / 'testdata'
|
||||||
assert d.exists(), d
|
assert d.exists(), d
|
||||||
return d
|
return d
|
|
@ -1,5 +1,16 @@
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
import pytz
|
|
||||||
|
from my.core.cfg import tmp_config
|
||||||
|
from my.core.common import make_dict
|
||||||
|
|
||||||
|
# todo ugh, it's discovered as a test???
|
||||||
|
from .common import testdata
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
# deliberately use mixed style imports on the top level and inside the methods to test tmp_config stuff
|
||||||
|
import my.reddit.rexport as my_reddit_rexport
|
||||||
|
import my.reddit.all as my_reddit_all
|
||||||
|
|
||||||
|
|
||||||
def test_basic() -> None:
|
def test_basic() -> None:
|
||||||
|
@ -7,17 +18,18 @@ def test_basic() -> None:
|
||||||
# would ensure reasonable stat implementation as well and less duplication
|
# would ensure reasonable stat implementation as well and less duplication
|
||||||
# note: deliberately use old module (instead of my.reddit.all) to test bwd compatibility
|
# note: deliberately use old module (instead of my.reddit.all) to test bwd compatibility
|
||||||
from my.reddit import saved, events
|
from my.reddit import saved, events
|
||||||
|
|
||||||
assert len(list(events())) > 0
|
assert len(list(events())) > 0
|
||||||
assert len(list(saved())) > 0
|
assert len(list(saved())) > 0
|
||||||
|
|
||||||
|
|
||||||
def test_comments() -> None:
|
def test_comments() -> None:
|
||||||
from my.reddit.all import comments
|
assert len(list(my_reddit_all.comments())) > 0
|
||||||
assert len(list(comments())) > 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_unfav() -> None:
|
def test_unfav() -> None:
|
||||||
from my.reddit import events, saved
|
from my.reddit import events
|
||||||
|
|
||||||
ev = events()
|
ev = events()
|
||||||
url = 'https://reddit.com/r/QuantifiedSelf/comments/acxy1v/personal_dashboard/'
|
url = 'https://reddit.com/r/QuantifiedSelf/comments/acxy1v/personal_dashboard/'
|
||||||
uev = [e for e in ev if e.url == url]
|
uev = [e for e in ev if e.url == url]
|
||||||
|
@ -31,52 +43,48 @@ def test_unfav() -> None:
|
||||||
|
|
||||||
def test_saves() -> None:
|
def test_saves() -> None:
|
||||||
from my.reddit.all import saved
|
from my.reddit.all import saved
|
||||||
|
|
||||||
saves = list(saved())
|
saves = list(saved())
|
||||||
assert len(saves) > 0
|
assert len(saves) > 0
|
||||||
|
|
||||||
# just check that they are unique (makedict will throw)
|
# just check that they are unique (makedict will throw)
|
||||||
from my.core.common import make_dict
|
|
||||||
make_dict(saves, key=lambda s: s.sid)
|
make_dict(saves, key=lambda s: s.sid)
|
||||||
|
|
||||||
|
|
||||||
def test_disappearing() -> None:
|
def test_disappearing() -> None:
|
||||||
from my.reddit.rexport import events
|
|
||||||
# eh. so for instance, 'metro line colors' is missing from reddit-20190402005024.json for no reason
|
# eh. so for instance, 'metro line colors' is missing from reddit-20190402005024.json for no reason
|
||||||
# but I guess it was just a short glitch... so whatever
|
# but I guess it was just a short glitch... so whatever
|
||||||
saves = events()
|
evs = my_reddit_rexport.events()
|
||||||
favs = [s.kind for s in saves if s.text == 'favorited']
|
favs = [s.kind for s in evs if s.text == 'favorited']
|
||||||
[deal_with_it] = [f for f in favs if f.title == '"Deal with it!"']
|
[deal_with_it] = [f for f in favs if f.title == '"Deal with it!"']
|
||||||
assert deal_with_it.backup_dt == datetime(2019, 4, 1, 23, 10, 25, tzinfo=pytz.utc)
|
assert deal_with_it.backup_dt == datetime(2019, 4, 1, 23, 10, 25, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
def test_unfavorite() -> None:
|
def test_unfavorite() -> None:
|
||||||
from my.reddit.rexport import events
|
evs = my_reddit_rexport.events()
|
||||||
evs = events()
|
|
||||||
unfavs = [s for s in evs if s.text == 'unfavorited']
|
unfavs = [s for s in evs if s.text == 'unfavorited']
|
||||||
[xxx] = [u for u in unfavs if u.eid == 'unf-19ifop']
|
[xxx] = [u for u in unfavs if u.eid == 'unf-19ifop']
|
||||||
assert xxx.dt == datetime(2019, 1, 29, 10, 10, 20, tzinfo=pytz.utc)
|
assert xxx.dt == datetime(2019, 1, 29, 10, 10, 20, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
def test_preserves_extra_attr() -> None:
|
def test_preserves_extra_attr() -> None:
|
||||||
# doesn't strictly belong here (not specific to reddit)
|
# doesn't strictly belong here (not specific to reddit)
|
||||||
# but my.reddit does a fair bit of dyunamic hacking, so perhaps a good place to check nothing is lost
|
# but my.reddit does a fair bit of dynamic hacking, so perhaps a good place to check nothing is lost
|
||||||
from my.reddit import config
|
from my.reddit import config
|
||||||
|
|
||||||
assert isinstance(getattr(config, 'please_keep_me'), str)
|
assert isinstance(getattr(config, 'please_keep_me'), str)
|
||||||
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
@pytest.fixture(autouse=True, scope='module')
|
@pytest.fixture(autouse=True, scope='module')
|
||||||
def prepare():
|
def prepare():
|
||||||
from .common import testdata
|
|
||||||
data = testdata() / 'hpi-testdata' / 'reddit'
|
data = testdata() / 'hpi-testdata' / 'reddit'
|
||||||
assert data.exists(), data
|
assert data.exists(), data
|
||||||
|
|
||||||
# note: deliberately using old config schema so we can test migrations
|
# note: deliberately using old config schema so we can test migrations
|
||||||
class test_config:
|
class config:
|
||||||
|
class reddit:
|
||||||
export_dir = data
|
export_dir = data
|
||||||
please_keep_me = 'whatever'
|
please_keep_me = 'whatever'
|
||||||
|
|
||||||
from my.core.cfg import tmp_config
|
with tmp_config(modules='my.reddit.*', config=config):
|
||||||
with tmp_config() as config:
|
|
||||||
config.reddit = test_config
|
|
||||||
yield
|
yield
|
|
@ -50,7 +50,7 @@ def test_old_db() -> None:
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def prepare():
|
def prepare():
|
||||||
from .common import testdata
|
from my.tests.common import testdata
|
||||||
bmdata = testdata() / 'hpi-testdata' / 'bluemaestro'
|
bmdata = testdata() / 'hpi-testdata' / 'bluemaestro'
|
||||||
assert bmdata.exists(), bmdata
|
assert bmdata.exists(), bmdata
|
||||||
|
|
||||||
|
|
|
@ -121,6 +121,6 @@ Some misc stuff
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def prepare():
|
def prepare():
|
||||||
from .common import reset_modules
|
from my.tests.common import reset_modules
|
||||||
reset_modules()
|
reset_modules()
|
||||||
yield
|
yield
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
|
|
||||||
def test() -> None:
|
def test() -> None:
|
||||||
|
@ -13,8 +13,6 @@ def test() -> None:
|
||||||
assert d.sleep_end.tzinfo is not None
|
assert d.sleep_end.tzinfo is not None
|
||||||
|
|
||||||
|
|
||||||
from .common import skip_if_not_karlicoss
|
|
||||||
@skip_if_not_karlicoss
|
|
||||||
def test_tz() -> None:
|
def test_tz() -> None:
|
||||||
from my.emfit import datas
|
from my.emfit import datas
|
||||||
# TODO check errors too?
|
# TODO check errors too?
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
def test_checkins() -> None:
|
def test_checkins() -> None:
|
||||||
from my.foursquare import get_checkins
|
from my.foursquare import get_checkins
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
from more_itertools import ilen
|
from more_itertools import ilen
|
||||||
# todo test against stats? not sure.. maybe both
|
# todo test against stats? not sure.. maybe both
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
from more_itertools import ilen
|
from more_itertools import ilen
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
def test() -> None:
|
def test() -> None:
|
||||||
from my.hypothesis import pages, highlights
|
from my.hypothesis import pages, highlights
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
|
|
||||||
def test_pages() -> None:
|
def test_pages() -> None:
|
||||||
# TODO ugh. need lazy import to simplify testing?
|
# TODO ugh. need lazy import to simplify testing?
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
from datetime import date, time
|
from datetime import date, time
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
# todo maybe belongs to common
|
# todo maybe belongs to common
|
||||||
from more_itertools import ilen
|
from more_itertools import ilen
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
from my import orgmode
|
from my import orgmode
|
||||||
from my.core.orgmode import collect
|
from my.core.orgmode import collect
|
||||||
|
|
||||||
from .common import skip_if_not_karlicoss
|
|
||||||
|
|
||||||
@skip_if_not_karlicoss
|
|
||||||
def test() -> None:
|
def test() -> None:
|
||||||
# meh
|
# meh
|
||||||
results = list(orgmode.query().collect_all(lambda n: [n] if 'python' in n.tags else []))
|
results = list(orgmode.query().collect_all(lambda n: [n] if 'python' in n.tags else []))
|
||||||
|
|
|
@ -4,7 +4,7 @@ from more_itertools import ilen
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from .common import testdata
|
from my.tests.common import testdata
|
||||||
|
|
||||||
|
|
||||||
def test_module(with_config) -> None:
|
def test_module(with_config) -> None:
|
||||||
|
@ -35,7 +35,7 @@ def test_with_error(with_config, tmp_path: Path) -> None:
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def with_config():
|
def with_config():
|
||||||
from .common import reset_modules
|
from my.tests.common import reset_modules
|
||||||
reset_modules() # todo ugh.. getting boilerplaty.. need to make it a bit more automatic..
|
reset_modules() # todo ugh.. getting boilerplaty.. need to make it a bit more automatic..
|
||||||
|
|
||||||
# extra_data = Path(__file__).absolute().parent / 'extra/data/polar'
|
# extra_data = Path(__file__).absolute().parent / 'extra/data/polar'
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
|
|
||||||
def test() -> None:
|
def test() -> None:
|
||||||
|
|
|
@ -15,7 +15,7 @@ class SharedConfig(NamedTuple):
|
||||||
|
|
||||||
|
|
||||||
def _prepare_google_config(tmp_path: Path):
|
def _prepare_google_config(tmp_path: Path):
|
||||||
from .common import testdata
|
from my.tests.common import testdata
|
||||||
try:
|
try:
|
||||||
track = one(testdata().rglob('italy-slovenia-2017-07-29.json'))
|
track = one(testdata().rglob('italy-slovenia-2017-07-29.json'))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
|
@ -35,7 +35,7 @@ def _prepare_google_config(tmp_path: Path):
|
||||||
# pass tmp_path from pytest to this helper function
|
# pass tmp_path from pytest to this helper function
|
||||||
# see tests/tz.py as an example
|
# see tests/tz.py as an example
|
||||||
def temp_config(temp_path: Path) -> Any:
|
def temp_config(temp_path: Path) -> Any:
|
||||||
from .common import reset_modules
|
from my.tests.common import reset_modules
|
||||||
reset_modules()
|
reset_modules()
|
||||||
|
|
||||||
LTZ.config.fast = True
|
LTZ.config.fast = True
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
# TODO maybe instead detect if it has any data at all
|
# TODO maybe instead detect if it has any data at all
|
||||||
# if none, then skip the test, say that user doesn't have any data?
|
# if none, then skip the test, say that user doesn't have any data?
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
# todo current test doesn't depend on data, in principle...
|
# todo current test doesn't depend on data, in principle...
|
||||||
# should make lazy loading the default..
|
# should make lazy loading the default..
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
from my.tests.common import skip_if_not_karlicoss as pytestmark
|
||||||
|
|
||||||
# TODO move elsewhere?
|
# TODO move elsewhere?
|
||||||
# these tests would only make sense with some existing data? although some of them would work for everyone..
|
# these tests would only make sense with some existing data? although some of them would work for everyone..
|
||||||
# not sure what's a good way of handling this..
|
# not sure what's a good way of handling this..
|
||||||
|
@ -6,8 +8,6 @@ import pytz
|
||||||
from more_itertools import bucket
|
from more_itertools import bucket
|
||||||
|
|
||||||
|
|
||||||
from .common import skip_if_not_karlicoss as pytestmark
|
|
||||||
|
|
||||||
# TODO ugh. if i uncomment this here (on top level), then this test vvv fails
|
# TODO ugh. if i uncomment this here (on top level), then this test vvv fails
|
||||||
# from my.media.youtube import get_watched, Watched
|
# from my.media.youtube import get_watched, Watched
|
||||||
# HPI_TESTS_KARLICOSS=true pytest -raps tests/tz.py tests/youtube.py
|
# HPI_TESTS_KARLICOSS=true pytest -raps tests/tz.py tests/youtube.py
|
||||||
|
|
7
tox.ini
7
tox.ini
|
@ -79,6 +79,13 @@ commands =
|
||||||
|
|
||||||
hpi module install my.reddit.rexport
|
hpi module install my.reddit.rexport
|
||||||
|
|
||||||
|
{envpython} -m pytest \
|
||||||
|
# importlib is the new suggested import-mode
|
||||||
|
# without it test package names end up as core.tests.* instead of my.core.tests.*
|
||||||
|
--import-mode=importlib \
|
||||||
|
--pyargs my.tests \
|
||||||
|
{posargs}
|
||||||
|
|
||||||
{envpython} -m pytest tests \
|
{envpython} -m pytest tests \
|
||||||
# ignore some tests which might take a while to run on ci..
|
# ignore some tests which might take a while to run on ci..
|
||||||
--ignore tests/takeout.py \
|
--ignore tests/takeout.py \
|
||||||
|
|
Loading…
Add table
Reference in a new issue