diff --git a/my/body/weight.py b/my/body/weight.py index d64dc42..9a2ff00 100644 --- a/my/body/weight.py +++ b/my/body/weight.py @@ -53,11 +53,10 @@ def from_orgmode() -> Iterator[Result]: ) -def dataframe(): +def make_dataframe(data: Iterator[Result]): import pandas as pd # type: ignore - entries = from_orgmode() def it(): - for e in from_orgmode(): + for e in data: if isinstance(e, Exception): dt = extract_error_datetime(e) yield { @@ -75,6 +74,11 @@ def dataframe(): df.index = pd.to_datetime(df.index, utc=True) return df + +def dataframe(): + entries = from_orgmode() + return make_dataframe(entries) + # TODO move to a submodule? e.g. my.body.weight.orgmode? # so there could be more sources # not sure about my.body thing though diff --git a/my/core/error.py b/my/core/error.py index 4303261..b60ab5a 100644 --- a/my/core/error.py +++ b/my/core/error.py @@ -116,7 +116,7 @@ def extract_error_datetime(e: Exception) -> Optional[datetime]: import re # TODO FIXME meh. definitely need to preserve exception args types in cachew if possible.. for x in reversed(e.args): - m = re.search(r'\d{4}.*T.*:..(\.\d{6})?(\+.....)?', x) + m = re.search(r'\d{4}-\d\d-\d\d(T..:..:..)?(\.\d{6})?(\+.....)?', x) if m is None: continue ss = m.group(0) @@ -141,3 +141,7 @@ def test_datetime_errors(): e3 = RuntimeError(str(['one', '2019-11-27T08:56:00', 'three'])) assert extract_error_datetime(e3) is not None + + # date only + e4 = RuntimeError(str(['one', '2019-11-27', 'three'])) + assert extract_error_datetime(e4) is not None diff --git a/my/emfit/__init__.py b/my/emfit/__init__.py index 0a1e1b3..7184e78 100755 --- a/my/emfit/__init__.py +++ b/my/emfit/__init__.py @@ -11,7 +11,7 @@ from typing import Dict, List, Iterable from ..core import get_files from ..core.common import mcachew from ..core.cachew import cache_dir -from ..core.error import Res +from ..core.error import Res, set_error_datetime, extract_error_datetime from ..core.types import DataFrameT from my.config import emfit as config @@ -70,6 +70,7 @@ def pre_dataframe() -> Iterable[Res[Emfit]]: yield r else: err = RuntimeError(f'Multiple sleeps per night, not supported yet: {g}') + set_error_datetime(err, dt=g[0].date) g.clear() yield err @@ -90,8 +91,11 @@ def dataframe() -> DataFrameT: last = None for s in pre_dataframe(): if isinstance(s, Exception): - # todo date would be nice too? - d = {'error': str(s)} + edt = extract_error_datetime(s) + d = { + 'date' : edt, + 'error': str(s), + } else: dd = s.date pday = dd - timedelta(days=1) diff --git a/my/emfit/plot.py b/my/emfit/plot.py index 87ec6e8..b483801 100755 --- a/my/emfit/plot.py +++ b/my/emfit/plot.py @@ -1,3 +1,4 @@ +# TODO this should be integrated into dashboard # def stats(): # for jj in iter_datas(): # # TODO fimezone?? diff --git a/my/jawbone/__init__.py b/my/jawbone/__init__.py index 0879b20..7ea9ad6 100755 --- a/my/jawbone/__init__.py +++ b/my/jawbone/__init__.py @@ -1,12 +1,16 @@ #!/usr/bin/env python3 -from typing import Dict, Any, List +from typing import Dict, Any, List, Iterable import json from functools import lru_cache from datetime import datetime, date, time, timedelta from pathlib import Path -import logging + import pytz +from ..core.common import LazyLogger + +logger = LazyLogger(__name__) + from my.config import jawbone as config @@ -16,9 +20,6 @@ SLEEPS_FILE = BDIR / 'sleeps.json' GRAPHS_DIR = BDIR / 'graphs' -def get_logger(): - return logging.getLogger('jawbone-provider') - XID = str # TODO how to shared with backup thing? @@ -32,13 +33,13 @@ class SleepEntry: def __init__(self, js) -> None: self.js = js - # TODO @memoize decorator? @property def date_(self) -> date: return self.sleep_end.date() def _fromts(self, ts: int) -> datetime: - return pytz.utc.localize(datetime.utcfromtimestamp(ts)).astimezone(self._tz).astimezone(self._tz) + return datetime.fromtimestamp(ts, tz=self._tz) + @property def _tz(self): return pytz.timezone(self._details['tz']) @@ -103,15 +104,63 @@ def load_sleeps() -> List[SleepEntry]: return [SleepEntry(js) for js in sleeps] +from ..core.error import Res, set_error_datetime, extract_error_datetime + +def pre_dataframe() -> Iterable[Res[SleepEntry]]: + sleeps = load_sleeps() + # todo emit error if graph doesn't exist?? + sleeps = [s for s in sleeps if s.graph.exists()] # TODO careful.. + from ..common import group_by_key + for dd, group in group_by_key(sleeps, key=lambda s: s.date_).items(): + if len(group) == 1: + yield group[0] + else: + err = RuntimeError(f'Multiple sleeps per night, not supported yet: {group}') + set_error_datetime(err, dt=dd) + logger.exception(err) + yield err + + +def dataframe(): + dicts: List[Dict] = [] + for s in pre_dataframe(): + if isinstance(s, Exception): + dt = extract_error_datetime(s) + d = { + 'date' : dt, + 'error': str(s), + } + else: + d = { + # TODO make sure sleep start/end are consistent with emfit? add to test as well.. + # I think it makes sense to be end date as 99% of time + # or maybe I shouldn't care about this at all? + 'date' : s.date_, + 'sleep_start': s.sleep_start, + 'sleep_end' : s.sleep_end, + 'bed_time' : s.bed_time, + } + dicts.append(d) + + import pandas as pd # type: ignore + return pd.DataFrame(dicts) + # TODO tz is in sleeps json + + +# TODO add dataframe support to stat() +def stats(): + from ..core import stat + return stat(pre_dataframe) + + +#### NOTE: most of the stuff below is deprecated and remnants of my old code! +#### sorry for it, feel free to remove if you don't need it + import numpy as np # type: ignore import matplotlib.pyplot as plt # type: ignore from matplotlib.figure import Figure # type: ignore from matplotlib.axes import Axes # type: ignore -# pip install imageio -from imageio import imread # type: ignore - - def hhmm(time: datetime): return time.strftime("%H:%M") @@ -128,6 +177,9 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None, showtext=Tr span = sleep.completed - sleep.created print(f"{sleep.xid} span: {span}") + # pip install imageio + from imageio import imread # type: ignore + img = imread(sleep.graph) # all of them are 300x300 images apparently # span for image @@ -187,28 +239,6 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None, showtext=Tr axes.text(xlims[1] - timedelta(hours=1.5), 20, str(sleep),) # plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) -from ..common import group_by_key - -def sleeps_by_date() -> Dict[date, SleepEntry]: - logger = get_logger() - - sleeps = load_sleeps() - sleeps = [s for s in sleeps if s.graph.exists()] # TODO careful.. - res = {} - for dd, group in group_by_key(sleeps, key=lambda s: s.date_).items(): - if len(group) == 1: - res[dd] = group[0] - else: - # TODO short ones I can ignore I guess. but won't bother now - logger.error('multiple sleeps on %s: %s', dd, group) - return res - -# sleeps_count = 35 # len(sleeps) # apparently MPL fails at 298 with outofmemory or something -# start = 40 -# 65 is arount 1 july -# sleeps = sleeps[start: start + sleeps_count] -# sleeps = sleeps[:sleeps_count] -# dt = {k: v for k, v in dt.items() if v is not None} # TODO not really sure it belongs here... # import melatonin @@ -225,6 +255,7 @@ def predicate(sleep: SleepEntry): return False +# TODO move to dashboard def plot(): # TODO FIXME melatonin data melatonin_data = {} # type: ignore[var-annotated] @@ -264,48 +295,3 @@ def plot(): # plt.savefig('res.png', asp) plt.show() -import pandas as pd # type: ignore -def get_dataframe(): - sleeps = sleeps_by_date() - items = [] - for dd, s in sleeps.items(): - items.append({ - 'date' : dd, # TODO not sure... # TODO would also be great to sync column names... - 'sleep_start': s.sleep_start, - 'sleep_end' : s.sleep_end, - 'bed_time' : s.bed_time, - }) - # TODO tz is in sleeps json - res = pd.DataFrame(items) - return res - - -def test_tz(): - sleeps = sleeps_by_date() - for s in sleeps.values(): - assert s.sleep_start.tzinfo is not None - assert s.sleep_end.tzinfo is not None - - for dd, exp in [ - (date(year=2015, month=8 , day=28), time(hour=7, minute=20)), - (date(year=2015, month=9 , day=15), time(hour=6, minute=10)), - ]: - sleep = sleeps[dd] - end = sleep.sleep_end - - assert end.time() == exp - - # TODO fuck. on 0909 I woke up at around 6 according to google timeline - # but according to jawbone, it was on 0910?? eh. I guess it's jus shitty tracking. - - -def main(): - # TODO eh. vendorize klogging already? - from kython.klogging import setup_logzero - setup_logzero(get_logger()) - test_tz() - # print(get_dataframe()) - - -if __name__ == '__main__': - main() diff --git a/my/jawbone/plots.py b/my/jawbone/plots.py index ae44a8c..eb04200 100755 --- a/my/jawbone/plots.py +++ b/my/jawbone/plots.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# TODO +# TODO this should be in dashboard from pathlib import Path # from kython.plotting import * from csv import DictReader diff --git a/tests/jawbone.py b/tests/jawbone.py new file mode 100644 index 0000000..b6630a4 --- /dev/null +++ b/tests/jawbone.py @@ -0,0 +1,23 @@ +# TODO depends on my private data... move somewhere, and exclude from CI somehow? + +from datetime import date, time + +from my.jawbone import sleeps_by_date + +# todo private test.. move away +def test_tz(): + sleeps = sleeps_by_date() + for s in sleeps.values(): + assert s.sleep_start.tzinfo is not None + assert s.sleep_end.tzinfo is not None + + for dd, exp in [ + (date(year=2015, month=8 , day=28), time(hour=7, minute=20)), + (date(year=2015, month=9 , day=15), time(hour=6, minute=10)), + ]: + sleep = sleeps[dd] + end = sleep.sleep_end + + assert end.time() == exp + # TODO fuck. on 0909 I woke up at around 6 according to google timeline + # but according to jawbone, it was on 0910?? eh. I guess it's just shitty tracking.