tests: move remaining tests from tests/ to my.tests, cleanup corresponding modules

my/body/weight.py

@@ -2,21 +2,29 @@
 Weight data (manually logged)
 '''
 
+from dataclasses import dataclass
 from datetime import datetime
-from typing import NamedTuple, Iterator
+from typing import Any, Iterator
 
-from ..core import LazyLogger
-from ..core.error import Res, set_error_datetime, extract_error_datetime
+from my.core import make_logger
+from my.core.error import Res, extract_error_datetime, set_error_datetime
 
-from .. import orgmode
+from my import orgmode
 
-from my.config import weight as config  # type: ignore[attr-defined]
+config = Any
 
 
-log = LazyLogger('my.body.weight')
+def make_config() -> config:
+    from my.config import weight as user_config  # type: ignore[attr-defined]
+
+    return user_config()
 
 
-class Entry(NamedTuple):
+log = make_logger(__name__)
+
+
+@dataclass
+class Entry:
     dt: datetime
     value: float
     # TODO comment??
@@ -26,6 +34,8 @@ Result = Res[Entry]
 
 
 def from_orgmode() -> Iterator[Result]:
+    cfg = make_config()
+
     orgs = orgmode.query()
     for o in orgmode.query().all():
         if 'weight' not in o.tags:
@@ -46,8 +56,8 @@ def from_orgmode() -> Iterator[Result]:
             yield e
             continue
         # FIXME use timezone provider
-        created = config.default_timezone.localize(created)
-        assert created is not None #??? somehow mypy wasn't happy?
+        created = cfg.default_timezone.localize(created)
+        assert created is not None  # ??? somehow mypy wasn't happy?
         yield Entry(
             dt=created,
             value=w,
@@ -57,19 +67,21 @@ def from_orgmode() -> Iterator[Result]:
 
 def make_dataframe(data: Iterator[Result]):
     import pandas as pd
+
     def it():
         for e in data:
             if isinstance(e, Exception):
                 dt = extract_error_datetime(e)
                 yield {
-                    'dt'    : dt,
+                    'dt': dt,
                     'error': str(e),
                 }
             else:
                 yield {
-                    'dt'    : e.dt,
+                    'dt': e.dt,
                     'weight': e.value,
                 }
+
     df = pd.DataFrame(it())
     df.set_index('dt', inplace=True)
     # TODO not sure about UTC??
@@ -81,6 +93,7 @@ def dataframe():
     entries = from_orgmode()
     return make_dataframe(entries)
 
+
 # TODO move to a submodule? e.g. my.body.weight.orgmode?
 # so there could be more sources
 # not sure about my.body thing though

my/orgmode.py

@@ -6,18 +6,28 @@ REQUIRES = [
     'orgparse',
 ]
 
+import re
 from datetime import datetime
 from pathlib import Path
-import re
-from typing import List, Sequence, Iterable, NamedTuple, Optional, Tuple
+from typing import Iterable, List, NamedTuple, Optional, Sequence, Tuple
 
-from my.core import get_files
+import orgparse
+
+from my.core import Paths, Stats, get_files, stat
 from my.core.cachew import cache_dir, mcachew
 from my.core.orgmode import collect
 
-from my.config import orgmode as user_config
 
-import orgparse
+class config:
+    paths: Paths
+
+
+def make_config() -> config:
+    from my.config import orgmode as user_config
+
+    class combined_config(user_config, config): ...
+
+    return combined_config()
 
 
 # temporary? hack to cache org-mode notes
@@ -28,10 +38,13 @@ class OrgNote(NamedTuple):
 
 
 def inputs() -> Sequence[Path]:
-    return get_files(user_config.paths)
+    cfg = make_config()
+    return get_files(cfg.paths)
 
 
 _rgx = re.compile(orgparse.date.gene_timestamp_regex(brtype='inactive'), re.VERBOSE)
+
+
 def _created(n: orgparse.OrgNode) -> Tuple[Optional[datetime], str]:
     heading = n.heading
     # meh.. support in orgparse?
@@ -41,7 +54,7 @@ def _created(n: orgparse.OrgNode) -> Tuple[Optional[datetime], str]:
         # try to guess from heading
         m = _rgx.search(heading)
         if m is not None:
-            createds = m.group(0) # could be None
+            createds = m.group(0)  # could be None
     if createds is None:
         return (None, heading)
     assert isinstance(createds, str)
@@ -67,7 +80,7 @@ def to_note(x: orgparse.OrgNode) -> OrgNote:
         created = None
     return OrgNote(
         created=created,
-        heading=heading, # todo include the body?
+        heading=heading,  # todo include the body?
         tags=list(x.tags),
     )
 
@@ -84,14 +97,15 @@ def _cachew_cache_path(_self, f: Path) -> Path:
 def _cachew_depends_on(_self, f: Path):
     return (f, f.stat().st_mtime)
 
- 
+
 class Query:
     def __init__(self, files: Sequence[Path]) -> None:
         self.files = files
 
     # TODO yield errors?
     @mcachew(
-        cache_path=_cachew_cache_path, force_file=True,
+        cache_path=_cachew_cache_path,
+        force_file=True,
         depends_on=_cachew_depends_on,
     )
     def _iterate(self, f: Path) -> Iterable[OrgNote]:
@@ -114,8 +128,8 @@ def query() -> Query:
     return Query(files=inputs())
 
 
-from my.core import Stats, stat
 def stats() -> Stats:
     def outlines():
         return query().all()
+
     return stat(outlines)

my/pdfs.py

@@ -10,7 +10,7 @@ REQUIRES = [
 import time
 from datetime import datetime
 from pathlib import Path
-from typing import Iterator, List, NamedTuple, Optional, Protocol, Sequence
+from typing import Iterator, List, NamedTuple, Optional, Protocol, Sequence, TYPE_CHECKING
 
 import pdfannots
 from more_itertools import bucket
@@ -185,8 +185,6 @@ def stats() -> Stats:
 
 
 ### legacy/misc stuff
-iter_annotations = annotations  # for backwards compatibility
+if not TYPE_CHECKING:
+    iter_annotations = annotations
 ###
-
-# can use 'hpi query my.pdfs.annotations -o pprint' to test
-#

my/tests/bluemaestro.py

@@ -0,0 +1,57 @@
+from typing import Iterator
+
+import pytest
+from more_itertools import one
+
+from my.bluemaestro import Measurement, measurements
+from my.core.cfg import tmp_config
+
+from .common import testdata
+
+
+def ok_measurements() -> Iterator[Measurement]:
+    for m in measurements():
+        assert not isinstance(m, Exception)
+        yield m
+
+
+def test() -> None:
+    res2020 = [m for m in ok_measurements() if '2020' in str(m.dt)]
+
+    tp = [x for x in res2020 if x.temp == 2.1]
+    assert len(tp) > 0
+    for p in tp:
+        print(p)
+        dts = p.dt.strftime('%Y%m%d %H')
+        # check that timezone is set properly
+        assert dts == '20200824 22'
+
+    assert len(tp) == 1  # should be unique
+
+    # 2.5 K + 4 K datapoints, somewhat overlapping
+    assert len(res2020) < 6000
+
+
+def test_old_db() -> None:
+    res = list(ok_measurements())
+
+    r1 = one(x for x in res if x.dt.strftime('%Y%m%d %H:%M:%S') == '20181003 09:07:00')
+    r2 = one(x for x in res if x.dt.strftime('%Y%m%d %H:%M:%S') == '20181003 09:19:00')
+
+    assert r1.temp == 16.8
+    assert r2.temp == 18.5
+    assert r1.pressure == 1024.5
+    assert r2.pressure == 1009.8
+
+
+@pytest.fixture(autouse=True)
+def prepare():
+    bmdata = testdata() / 'hpi-testdata' / 'bluemaestro'
+    assert bmdata.exists(), bmdata
+
+    class bluemaestro:
+        export_path = bmdata
+
+    with tmp_config() as config:
+        config.bluemaestro = bluemaestro
+        yield

my/tests/body/weight.py

@@ -0,0 +1,57 @@
+from pathlib import Path
+import pytz
+from my.core.cfg import tmp_config
+import pytest
+from my.body.weight import from_orgmode
+
+
+def test_body_weight() -> None:
+    weights = [0.0 if isinstance(x, Exception) else x.value for x in from_orgmode()]
+
+    assert weights == [
+        0.0,
+        62.0,
+        0.0,
+        61.0,
+        62.0,
+        0.0,
+    ]
+
+
+@pytest.fixture(autouse=True)
+def prepare(tmp_path: Path):
+    ndir = tmp_path / 'notes'
+    ndir.mkdir()
+    logs = ndir / 'logs.org'
+    logs.write_text(
+        '''
+#+TITLE: Stuff I'm logging
+
+* Weight (org-capture) :weight:
+** [2020-05-01 Fri 09:00] 62
+** 63
+    this should be ignored, got no timestamp
+** [2020-05-03 Sun 08:00] 61
+** [2020-05-04 Mon 10:00] 62
+'''
+    )
+    misc = ndir / 'misc.org'
+    misc.write_text(
+        '''
+Some misc stuff
+
+* unrelated note :weight:whatever:
+'''
+    )
+
+    class orgmode:
+        paths = [ndir]
+
+    class weight:
+        # TODO ugh. this belongs to tz provider or global config or something
+        default_timezone = pytz.timezone('Europe/London')
+
+    with tmp_config() as cfg:
+        cfg.orgmode = orgmode
+        cfg.weight = weight
+        yield

my/tests/pdfs.py

@@ -0,0 +1,90 @@
+import inspect
+from pathlib import Path
+
+import pytest
+from more_itertools import ilen
+
+from my.core.cfg import tmp_config
+from my.pdfs import annotated_pdfs, annotations, get_annots
+from my.tests.common import testdata
+
+
+def test_module(with_config) -> None:
+    # todo check types etc as well
+    assert ilen(annotations()) >= 3
+    assert ilen(annotated_pdfs()) >= 1
+
+
+def test_with_error(with_config, tmp_path: Path) -> None:
+    """should handle crappy files gracefully"""
+    root = tmp_path
+    g = root / 'garbage.pdf'
+    g.write_text('garbage')
+
+    from my.config import pdfs
+
+    # meh. otherwise legacy config value 'wins'
+    del pdfs.roots  # type: ignore[attr-defined]
+    pdfs.paths = (root,)
+
+    annots = list(annotations())
+    [annot] = annots
+    assert isinstance(annot, Exception)
+
+
+@pytest.fixture
+def with_config():
+    # extra_data = Path(__file__).absolute().parent / 'extra/data/polar'
+    # assert extra_data.exists(), extra_data
+    # todo hmm, turned out no annotations in these ones.. whatever
+
+    class user_config:
+        roots = [
+            testdata(),
+        ]
+
+    with tmp_config() as config:
+        config.pdfs = user_config
+        yield
+
+
+EXPECTED_HIGHLIGHTS = {
+    'Since 1994, when we first began organizing web sites, we have enjoyed a rare oppor-tunity to participate in the birth of a new discipline.',
+    'And yet, unlearn we must,',
+    '',
+}
+
+
+def test_get_annots() -> None:
+    """
+    Test get_annots, with a real PDF file
+    get_annots should return a list of three Annotation objects
+    """
+    annotations = get_annots(testdata() / 'pdfs' / 'Information Architecture for the World Wide Web.pdf')
+    assert len(annotations) == 3
+    assert set([a.highlight for a in annotations]) == EXPECTED_HIGHLIGHTS
+
+
+def test_annotated_pdfs_with_filelist() -> None:
+    """
+    Test annotated_pdfs, with a real PDF file
+    annotated_pdfs should return a list of one Pdf object, with three Annotations
+    """
+    filelist = [testdata() / 'pdfs' / 'Information Architecture for the World Wide Web.pdf']
+    annotations_generator = annotated_pdfs(filelist=filelist)
+
+    assert inspect.isgeneratorfunction(annotated_pdfs)
+
+    highlights_from_pdfs = []
+
+    for pdf_object in list(annotations_generator):
+        assert not isinstance(pdf_object, Exception)
+        highlights_from_pdfs.extend([a.highlight for a in pdf_object.annotations])
+
+    assert len(highlights_from_pdfs) == 3
+    assert set(highlights_from_pdfs) == EXPECTED_HIGHLIGHTS
+
+
+# todo old test on my(karlicoss) computer:
+# - mature-optimization_wtf.pdf: >3 annotations?
+# - nonlinear2.pdf

my/tests/reddit.py

@@ -1,15 +1,16 @@
+import pytest
+from more_itertools import consume
+
 from my.core.cfg import tmp_config
 from my.core.utils.itertools import ensure_unique
 
-# todo ugh, it's discovered as a test???
 from .common import testdata
 
-from more_itertools import consume
-import pytest
 
 # deliberately use mixed style imports on the top level and inside the methods to test tmp_config stuff
-import my.reddit.rexport as my_reddit_rexport
+# todo won't really be necessary once we migrate to lazy user config
 import my.reddit.all as my_reddit_all
+import my.reddit.rexport as my_reddit_rexport
 
 
 def test_basic_1() -> None: