core: experiments with attaching datetime information to errors

my.body.weigth: put datetime information in the error rows

my/body/weight.py

@@ -5,8 +5,9 @@ Weight data (manually logged)
 from datetime import datetime
 from typing import NamedTuple, Iterator
 
-from ..common import LazyLogger
-from ..error import Res
+from ..core import LazyLogger
+from ..core.error import Res, set_error_datetime, extract_error_datetime
+
 from ..notes import orgmode
 
 from my.config import weight as config
@@ -24,7 +25,7 @@ class Entry(NamedTuple):
 Result = Res[Entry]
 
 
-# TODO cachew?
+# TODO cachew? but in order for that to work, would need timestamps for input org-mode files..
 def from_orgmode() -> Iterator[Result]:
     orgs = orgmode.query()
     for o in orgs.query_all(lambda o: o.with_tag('weight')):
@@ -39,10 +40,11 @@ def from_orgmode() -> Iterator[Result]:
         try:
             w = float(o.heading)
         except Exception as e:
+            set_error_datetime(e, dt=created)
             log.exception(e)
             yield e
             continue
-        # TODO not sure if it's really necessary..
+        # todo perhaps, better to use timezone provider
         created = config.default_timezone.localize(created)
         yield Entry(
             dt=created,
@@ -57,7 +59,9 @@ def dataframe():
     def it():
         for e in from_orgmode():
             if isinstance(e, Exception):
+                dt = extract_error_datetime(e)
                 yield {
+                    'dt'    : dt,
                     'error': str(e),
                 }
             else:
@@ -67,6 +71,7 @@ def dataframe():
                 }
     df = pd.DataFrame(it())
     df.set_index('dt', inplace=True)
+    # TODO not sure about UTC??
     df.index = pd.to_datetime(df.index, utc=True)
     return df
 

my/core/error.py

@@ -4,7 +4,7 @@ See https://beepb00p.xyz/mypy-error-handling.html#kiss for more detail
 """
 
 from itertools import tee
-from typing import Union, TypeVar, Iterable, List, Tuple, Type
+from typing import Union, TypeVar, Iterable, List, Tuple, Type, Optional
 
 
 T = TypeVar('T')
@@ -97,3 +97,47 @@ def test_sort_res_by() -> None:
     results2 = sort_res_by(ress + [0], lambda x: x) # type: ignore
     assert results2 == [Exc('last'), 0] + results[:-1]
 
+
+
+# helpers to associate timestamps with the errors (so something meaningful could be displayed on the plots, for example)
+# todo document it under 'patterns' somewhere...
+
+# todo proper typevar?
+from datetime import datetime
+def set_error_datetime(e: Exception, dt: datetime) -> None:
+    # at the moment, we're using isoformat() instead of datetime directly to make it cachew-friendly
+    # once cachew preserves exception argument types, we can remove these hacks
+    e.args = e.args + (dt.isoformat(), )
+    # todo not sure if should return new exception?
+
+
+def extract_error_datetime(e: Exception) -> Optional[datetime]:
+    from .common import fromisoformat
+    import re
+    # TODO FIXME meh. definitely need to preserve exception args types in cachew if possible..
+    for x in reversed(e.args):
+        m = re.search(r'\d{4}.*T.*:..(\.\d{6})?(\+.....)?', x)
+        if m is None:
+            continue
+        ss = m.group(0)
+        # todo not sure if should be defensive??
+        return fromisoformat(ss)
+    return None
+
+
+def test_datetime_errors():
+    import pytz
+    dt_notz = datetime.now()
+    dt_tz   = datetime.now(tz=pytz.timezone('Europe/Amsterdam'))
+    for dt in [dt_tz, dt_notz]:
+        e1 = RuntimeError('whatever')
+        assert extract_error_datetime(e1) is None
+        set_error_datetime(e1, dt=dt)
+        assert extract_error_datetime(e1) == dt
+        # test that cachew can handle it...
+        e2 = RuntimeError(str(e1.args))
+        assert extract_error_datetime(e2) == dt
+
+
+    e3 = RuntimeError(str(['one', '2019-11-27T08:56:00', 'three']))
+    assert extract_error_datetime(e3) is not None

my/hypothesis.py

@@ -69,11 +69,10 @@ def pages() -> List[Res[Page]]:
 
 # todo not public api yet
 def stats():
-    # todo add 'last date' checks et
+    from .core import stat
     return {
-        # todo ilen
-        'highlights': len(highlights()),
-        'pages'     : len(pages()),
+        **stat(highlights),
+        **stat(pages),
     }
 
 

pytest.ini

@@ -2,6 +2,7 @@
 # for now, running with with_my pytest tests/reddit.py
 # discover __init__,py as well
 # TODO also importing too much with it...
+# TODO FIXME wtf is this??
 python_files = reddit.py
 addopts =
   --verbose

tests/misc.py

@@ -46,8 +46,8 @@ def prepare(tmp_path: Path):
         pass
 
 
-# meh
-from my.core.error import test_sort_res_by
+# meh. otherwise was struggling to run directly against my.core.error...
+from my.core.error import *
 
 
 from typing import Iterable, List