diff --git a/emfit/__init__.py b/emfit/__init__.py new file mode 100755 index 0000000..370f926 --- /dev/null +++ b/emfit/__init__.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +import json +import logging +from collections import OrderedDict as odict +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta +from pathlib import Path +from typing import Dict, Iterator, List, NamedTuple, Any, cast + +import kython +import pytz +from kython import cproperty, group_by_key + +from cachew import cachew + + +def get_logger(): + return logging.getLogger('emfit-provider') + +timed = lambda f: kython.timed(f, logger=get_logger()) + +def hhmm(minutes): + return '{:02d}:{:02d}'.format(*divmod(minutes, 60)) + +PATH = Path("/L/backups/emfit") + +EXCLUDED = [ + '***REMOVED***', # pretty weird, detected sleep and HR (!) during the day when I was at work + '***REMOVED***', + + '***REMOVED***', # some weird sleep during the day? +] + +AWAKE = 4 + +Sid = str + +# TODO FIXME use tz provider for that? although emfit is always in london... +_TZ = pytz.timezone('Europe/London') + +def fromts(ts) -> datetime: + dt = datetime.fromtimestamp(ts) + return _TZ.localize(dt) + + +class Mixin: + # TODO ugh. tricking mypy... + sleep_minutes_emfit: int + + @property + # ok, I guess that's reasonable way of defining sleep date + def date(self): + return self.end.date() # type: ignore[attr-defined] + + @cproperty + def time_in_bed(self): + return int((self.sleep_end - self.sleep_start).total_seconds()) // 60 # type: ignore[attr-defined] + + @property + def recovery(self): + return self.hrv_morning - self.hrv_evening # type: ignore[attr-defined] + + @property + def summary(self): + return f"""in bed for {hhmm(self.time_in_bed)} +emfit time: {hhmm(self.sleep_minutes_emfit)}; covered: {self.sleep_hr_coverage:.0f} +hrv morning: {self.hrv_morning:.0f} +hrv evening: {self.hrv_evening:.0f} +avg hr: {self.measured_hr_avg:.0f} +recovery: {self.recovery:3.0f} +{self.hrv_lf}/{self.hrv_hf}""" # type: ignore[attr-defined] + + +# TODO def use multiple threads for that.. +class EmfitOld(Mixin): + def __init__(self, sid: str, jj): + self.sid = sid + self.jj = jj + + def __hash__(self): + return hash(self.sid) + + @property + def hrv_morning(self): + return self.jj['hrv_rmssd_morning'] + + @property + def hrv_evening(self): + return self.jj['hrv_rmssd_evening'] + + """ + Bed time, not necessarily sleep + """ + @property + def start(self): + return fromts(self.jj['time_start']) + + """ + Bed time, not necessarily sleep + """ + @property + def end(self): + return fromts(self.jj['time_end']) + + @property + def epochs(self): + return self.jj['sleep_epoch_datapoints'] + + @property + def epoch_series(self): + tss = [] + eps = [] + for [ts, e] in self.epochs: + tss.append(ts) + eps.append(e) + return tss, eps + + @cproperty + def sleep_start(self) -> datetime: + for [ts, e] in self.epochs: + if e == AWAKE: + continue + return fromts(ts) + raise RuntimeError + + @cproperty + def sleep_end(self) -> datetime: + for [ts, e] in reversed(self.epochs): + if e == AWAKE: + continue + return fromts(ts) + raise RuntimeError +# 'sleep_epoch_datapoints' +# [[timestamp, number]] + + # so it's actual sleep, without awake + # ok, so I need time_asleep + @property + def sleep_minutes_emfit(self): + return self.jj['sleep_duration'] // 60 + + @property + def hrv_lf(self): + return self.jj['hrv_lf'] + + @property + def hrv_hf(self): + return self.jj['hrv_hf'] + + @property + def strip_awakes(self): + ff = None + ll = None + for i, [ts, e] in enumerate(self.epochs): + if e != AWAKE: + ff = i + break + for i in range(len(self.epochs) - 1, -1, -1): + [ts, e] = self.epochs[i] + if e != AWAKE: + ll = i + break + return self.epochs[ff: ll] + + + # # TODO epochs with implicit sleeps? not sure... e.g. night wakeups. + # # I guess I could input intervals/correct data/exclude days manually? + # @property + # def pulse_percentage(self): + + # # TODO pulse intervals are 4 seconds? + # # TODO ok, how to compute that?... + # # TODO cut ff start and end? + # # TODO remove awakes from both sides + # sp = self.strip_awakes + # present = {ep[0] for ep in sp} + # start = min(present) + # end = max(present) + # # TODO get start and end in one go? + + # for p in self.iter_points(): + # p.ts + + + # INT = 30 + + # missing = 0 + # total = 0 + # for tt in range(start, end + INT, INT): + # total += 1 + # if tt not in present: + # missing += 1 + # # TODO get hr instead! + # import ipdb; ipdb.set_trace() + # return missing + + + # st = st[0][0] + # INT = 30 + # for [ts, e] in sp: + # if e == AWAKE: + # continue + # return fromts(ts) + # raise RuntimeError + # pass + + def __str__(self) -> str: + return f"from {self.sleep_start} to {self.sleep_end}" + +# measured_datapoints +# [[timestamp, pulse, breath?, ??? hrv?]] # every 4 seconds? + + def iter_points(self): + for ll in self.jj['measured_datapoints']: + [ts, pulse, br, activity] = ll + # TODO what the fuck is whaat?? It can't be HRV, it's about 500 ms on average + # act in csv.. so it must be activity? wonder how is it measured. + # but I guess makes sense. yeah, "measured_activity_avg": 595, about that + # makes even more sense given tossturn datapoints only have timestamp + yield ts, pulse + + @property + def sleep_hr(self): + tss = [] + res = [] + for ts, pulse in self.iter_points(): + if self.sleep_start < fromts(ts) < self.sleep_end: + tss.append(ts) + res.append(pulse) + return tss, res + + @property + def sleep_hr_series(self): + return self.sleep_hr + + @property + def hrv(self): + tss = [] + res = [] + for ll in self.jj['hrv_rmssd_datapoints']: + [ts, rmssd, _, _, almost_always_zero, _] = ll + # timestamp,rmssd,tp,lfn,hfn,r_hrv + # TP is total_power?? + # erm. looks like there is a discrepancy between csv and json data. + # right, so web is using api v 1. what if i use v1?? + # definitely a discrepancy between v1 and v4. have no idea how to resolve it :( + # also if one of them is indeed tp value, it must have been rounded. + # TODO what is the meaning of the rest??? + # they don't look like HR data. + tss.append(ts) + res.append(rmssd) + return tss, res + + @property + def measured_hr_avg(self): + return self.jj["measured_hr_avg"] + + @cproperty + def sleep_hr_coverage(self): + tss, hrs = self.sleep_hr + covered = len([h for h in hrs if h is not None]) + expected = len(hrs) + return covered / expected * 100 + + +# right, so dataclass is better because you can use mixins +@dataclass(eq=True, frozen=True) +class Emfit(Mixin): + sid: Sid + hrv_morning: float + hrv_evening: float + start: datetime + end : datetime + sleep_start: datetime + sleep_end : datetime + sleep_hr_coverage: float + measured_hr_avg: float + sleep_minutes_emfit: int + hrv_lf: float + hrv_hf: float + + @classmethod + def make(cls, em) -> Iterator['Emfit']: + # TODO FIXME res? + logger = get_logger() + if em.epochs is None: + logger.error('%s (on %s) got None in epochs! ignoring', em.sid, em.date) + return + + yield cls(**{ + # pylint: disable=no-member + k: getattr(em, k) for k in Emfit.__annotations__ + }) + + +# TODO move to common? +def dir_hash(path: Path): + mtimes = tuple(p.stat().st_mtime for p in sorted(path.glob('*.json'))) + return mtimes + + +@cachew(cache_path=Path('/L/data/.cache/emfit.cache'), hashf=dir_hash) +def iter_datas_cached(path: Path) -> Iterator[Emfit]: + for f in sorted(path.glob('*.json')): + sid = f.stem + if sid in EXCLUDED: + continue + + em = EmfitOld(sid=sid, jj=json.loads(f.read_text())) + yield from Emfit.make(em) + + +def iter_datas(path=PATH) -> Iterator[Emfit]: + yield from iter_datas_cached(path) + + +def get_datas() -> List[Emfit]: + return list(sorted(iter_datas(), key=lambda e: e.start)) +# TODO move away old entries if there is a diff?? + + +@timed +def by_night() -> Dict[date, Emfit]: + logger = get_logger() + res: Dict[date, Emfit] = odict() + # TODO shit. I need some sort of interrupted sleep detection? + for dd, sleeps in group_by_key(get_datas(), key=lambda s: s.date).items(): + if len(sleeps) > 1: + logger.warning("multiple sleeps per night, not handled yet: %s", sleeps) + continue + [s] = sleeps + res[s.date] = s + return res + + + +def test(): + datas = get_datas() + for d in datas: + pass + + +def test_tz(): + datas = get_datas() + for d in datas: + assert d.start.tzinfo is not None + assert d.end.tzinfo is not None + assert d.sleep_start.tzinfo is not None + assert d.sleep_end.tzinfo is not None + + # https://qs.emfit.com/#/device/presence/***REMOVED*** + # this was winter time, so GMT, UTC+0 + sid_20190109 = '***REMOVED***' + [s0109] = [s for s in datas if s.sid == sid_20190109] + assert s0109.end.time() == time(hour=6, minute=42) + + # summer time, so UTC+1 + sid_20190411 = '***REMOVED***' + [s0411] = [s for s in datas if s.sid == sid_20190411] + assert s0411.end.time() == time(hour=9, minute=30) + + +def main(): + from kython.klogging import setup_logzero + setup_logzero(get_logger(), level=logging.DEBUG) + for k, v in by_night().items(): + print(k, v.start, v.end) + +if __name__ == '__main__': + main() diff --git a/plot.py b/plot.py new file mode 100755 index 0000000..87ec6e8 --- /dev/null +++ b/plot.py @@ -0,0 +1,26 @@ +# def stats(): +# for jj in iter_datas(): +# # TODO fimezone?? +# # TODOgetinterval on 16 aug -- err it's pretty stupid. I shouldn't count bed exit interval... +# st = fromts(jj['time_start']) +# en = fromts(jj['time_end']) +# tfmt = "%Y-%m-%d %a" +# tot_mins = 0 +# res = [] +# res.append(f"{st.strftime(tfmt)} -- {en.strftime(tfmt)}") +# for cls in ['rem', 'light', 'deep']: +# mins = jj[f'sleep_class_{cls}_duration'] // 60 +# res += [cls, hhmm(mins)] +# tot_mins += mins +# res += ["total", hhmm(tot_mins)] +# print(*res) + + +# then I could enable emfit feed and slog feed (pulled from all org notes) and see the correlation? also could pull workouts provider (and wlog) -- actually wlog processing could be moved to timeline too + +# TODO maybe rmssd should only be computed if we have a reasonable chunk of datas + +# TODO X axes: show hours and only two dates +# TODO 4 is awake, 3 REM, 2 light, 1 deep + +# deviartion beyond 25-75 or 75-25 is bad??