From 35924b82b0e0c69e59edb7caefe5eb237384cf10 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 13 Aug 2018 22:25:07 +0100 Subject: [PATCH 01/19] initial --- .gitignore | 172 ++++++++++++++++++++++++++++++++++++++++++++++ emfit/__init__.py | 0 2 files changed, 172 insertions(+) create mode 100644 .gitignore create mode 100644 emfit/__init__.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b539013 --- /dev/null +++ b/.gitignore @@ -0,0 +1,172 @@ + +# Created by https://www.gitignore.io/api/python,emacs + +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +### Python Patch ### +.venv/ + +### Python.VirtualEnv Stack ### +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +pip-selfcheck.json + + +# End of https://www.gitignore.io/api/python,emacs diff --git a/emfit/__init__.py b/emfit/__init__.py new file mode 100644 index 0000000..e69de29 From d531cba77a9c512e770aedb097d51366118d110b Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 18 Aug 2018 16:47:33 +0100 Subject: [PATCH 02/19] split into data and plotting parts --- emfit/__init__.py | 138 ++++++++++++++++++++++++++++++++++++++++++++++ plot.py | 128 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 plot.py diff --git a/emfit/__init__.py b/emfit/__init__.py index e69de29..a7e6077 100644 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -0,0 +1,138 @@ +from kython import json_load +from datetime import datetime +from os.path import join +from functools import lru_cache + +from datetime import timedelta, datetime +from typing import List, Dict, Iterator + +fromts = datetime.fromtimestamp + +def hhmm(minutes): + return '{:02d}:{:02d}'.format(*divmod(minutes, 60)) + +PATH = "/L/backups/emfit/" + +AWAKE = 4 + +class Emfit: + def __init__(self, jj): + self.jj = jj + + @property + def hrv_morning(self): + return self.jj['hrv_rmssd_morning'] + + @property + def hrv_evening(self): + return self.jj['hrv_rmssd_evening'] + + @property + def date(self): + return self.end.date() + + @property + def start(self): + return fromts(self.jj['time_start']) + + @property + def end(self): + return fromts(self.jj['time_end']) + + @property + def epochs(self): + return self.jj['sleep_epoch_datapoints'] + + @property + @lru_cache() + def sleep_start(self) -> datetime: + for [ts, e] in self.epochs: + if e == AWAKE: + continue + return fromts(ts) + raise RuntimeError + + @property + @lru_cache() + def sleep_end(self) -> datetime: + for [ts, e] in reversed(self.epochs): + if e == AWAKE: + continue + return fromts(ts) + raise RuntimeError +# 'sleep_epoch_datapoints' +# [[timestamp, number]] + + # so it's actual sleep, without awake + # ok, so I need time_asleep + @property + def sleep_minutes(self): + return self.jj['sleep_duration'] // 60 + + @property + def hrv_lf(self): + return self.jj['hrv_lf'] + + @property + def hrv_hf(self): + return self.jj['hrv_hf'] + + @property + def summary(self): + return f"for {hhmm(self.sleep_minutes)} hrv: [{self.hrv_morning:.0f} {self.hrv_evening:.0f} {self.hrv_morning - self.hrv_evening:3.0f} {self.hrv_lf}/{self.hrv_hf}]" + + +# measured_datapoints +# [[timestamp, pulse, breath?, ??? hrv?]] # every 4 seconds? + @property + def sleep_hr(self): + tss = [] + res = [] + for ll in self.jj['measured_datapoints']: + [ts, pulse, br, activity] = ll + # TODO what the fuck is whaat?? It can't be HRV, it's about 500 ms on average + # act in csv.. so it must be activity? wonder how is it measured. + # but I guess makes sense. yeah, "measured_activity_avg": 595, about that + # makes even more sense given tossturn datapoints only have timestamp + if self.sleep_start < fromts(ts) < self.sleep_end: + tss.append(ts) + res.append(pulse) + return tss, res + + @property + def hrv(self): + tss = [] + res = [] + for ll in self.jj['hrv_rmssd_datapoints']: + [ts, rmssd, _, _, almost_always_zero, _] = ll + # timestamp,rmssd,tp,lfn,hfn,r_hrv + # TP is total_power?? + # erm. looks like there is a discrepancy between csv and json data. + # right, so web is using api v 1. what if i use v1?? + # definitely a discrepancy between v1 and v4. have no idea how to resolve it :( + # also if one of them is indeed tp value, it must have been rounded. + # TODO what is the meaning of the rest??? + # they don't look like HR data. + tss.append(ts) + res.append(rmssd) + return tss, res + + @property + def sleep_hr_coverage(self): + tss, hrs = self.sleep_hr + covered_sec = len([h for h in hrs if h is not None]) + expected_sec = self.sleep_minutes * 60 / 4 + return covered_sec / expected_sec * 100 + +def iter_datas() -> Iterator[Emfit]: + import os + for f in sorted(os.listdir(PATH)): + if not f.endswith('.json'): + continue + + with open(join(PATH, f), 'r') as fo: + ef = Emfit(json_load(fo)) + yield ef + +def get_datas() -> List[Emfit]: + return list(sorted(list(iter_datas()), key=lambda e: e.start)) diff --git a/plot.py b/plot.py new file mode 100644 index 0000000..f6e3139 --- /dev/null +++ b/plot.py @@ -0,0 +1,128 @@ +import matplotlib.dates as md # type: ignore +import numpy as np # type: ignore +import seaborn as sns # type: ignore +import matplotlib.pyplot as plt + + +def plot_file(jj: str): + pts = jj['sleep_epoch_datapoints'] + + + tss = [datetime.fromtimestamp(p[0]) for p in pts] + vals = [p[1] for p in pts] + + plt.figure(figsize=(20,10)) + plt.plot(tss, vals) + + + xformatter = md.DateFormatter('%H:%M') + xlocator = md.MinuteLocator(interval = 15) + + ## Set xtick labels to appear every 15 minutes + plt.gcf().axes[0].xaxis.set_major_locator(xlocator) + + ## Format xtick labels as HH:MM + plt.gcf().axes[0].xaxis.set_major_formatter(xformatter) + + + + plt.xlabel('time') + plt.ylabel('phase') + plt.title('Sleep phases') + plt.grid(True) + plt.savefig(f"{f}.png") + plt.close() # TODO + # plt.show() + + pass + + +def plot_all(): + for jj in iter_datas(): + plot_file(jj) + + +# def stats(): +# for jj in iter_datas(): +# # TODO fimezone?? +# # TODOgetinterval on 16 aug -- err it's pretty stupid. I shouldn't count bed exit interval... +# st = fromts(jj['time_start']) +# en = fromts(jj['time_end']) +# tfmt = "%Y-%m-%d %a" +# tot_mins = 0 +# res = [] +# res.append(f"{st.strftime(tfmt)} -- {en.strftime(tfmt)}") +# for cls in ['rem', 'light', 'deep']: +# mins = jj[f'sleep_class_{cls}_duration'] // 60 +# res += [cls, hhmm(mins)] +# tot_mins += mins +# res += ["total", hhmm(tot_mins)] +# print(*res) + + +def stats(): + datas = get_datas() + cur = datas[0].date + for jj in datas: + # import ipdb; ipdb.set_trace() + while cur < jj.date: + cur += timedelta(days=1) + if cur.weekday() == 0: + print("---") + if cur != jj.date: + print(" ") + # cur = jj.date + print(f"{jj.date.strftime('%m.%d %a')} {jj.hrv_morning:.0f} {jj.hrv_evening:.0f} {jj.hrv_morning - jj.hrv_evening:3.0f} {hhmm(jj.sleep_minutes)} {jj.hrv_lf}/{jj.hrv_hf} {jj.sleep_hr_coverage:3.0f}") + + +def plot_recovery_vs_hr_percentage(): + sns.set(color_codes=True) + xs = [] + ys = [] + for jj in get_datas(): + xs.append(jj.hrv_morning - jj.hrv_evening) + ys.append(jj.sleep_hr_coverage) + ax = sns.regplot(x=xs, y=ys) # "recovery", y="percentage", data=pdata) + ax.set(xlabel='recovery', ylabel='percentage') + plt.show() + + +def plot_hr(): + jj = get_datas()[-1] + tss, uu = jj.sleep_hr + tss = tss[::10] + uu = uu[::10] + plt.figure(figsize=(15,4)) + ax = sns.pointplot(tss, uu, markers=" ") + ax.set(ylim=(None, 1000)) + + plt.show() + +# TODO ok, would be nice to have that every morning in timeline +# also timeline should have dynamic filters? maybe by tags +# then I could enable emfit feed and slog feed (pulled from all org notes) and see the correlation? also could pull workouts provider (and wlog) -- actually wlog processing could be moved to timeline too + +# TODO could plot 'recovery' thing and see if it is impacted by workouts + + +# TODO time_start, time_end + +# plot_hrv() +# stats() +# plot_recovery_vs_hr_percentage() +# stats() +# import matplotlib +# matplotlib.use('Agg') + + +# TODO maybe rmssd should only be computed if we have a reasonable chunk of datas +# also, trust it only if it's stable + +# plot_timestamped([p[0] for p in pts], [p[1] for p in pts], mavgs=[]).savefig('res.png') +# TODO X axes: show hours and only two dates +# TODO 4 is awake, 3 REM, 2 light, 1 deep + + + + +# deviartion beyond 25-75 or 75-25 is bad?? From 3484a5a39bdd1ef74213c28cf001125968f23b0f Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 16 Oct 2018 23:19:09 +0100 Subject: [PATCH 03/19] some attempts ti analyse data --- emfit/__init__.py | 18 +++++++++++++++--- emfit/__main__.py | 23 +++++++++++++++++++++++ plot.py | 19 ++++++++++++++++++- run | 2 ++ test.py | 5 +++++ 5 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 emfit/__main__.py mode change 100644 => 100755 plot.py create mode 100755 run create mode 100755 test.py diff --git a/emfit/__init__.py b/emfit/__init__.py index a7e6077..bc34b26 100644 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -43,7 +43,7 @@ class Emfit: def epochs(self): return self.jj['sleep_epoch_datapoints'] - @property + @property # type: ignore @lru_cache() def sleep_start(self) -> datetime: for [ts, e] in self.epochs: @@ -52,7 +52,7 @@ class Emfit: return fromts(ts) raise RuntimeError - @property + @property # type: ignore @lru_cache() def sleep_end(self) -> datetime: for [ts, e] in reversed(self.epochs): @@ -79,9 +79,17 @@ class Emfit: @property def summary(self): - return f"for {hhmm(self.sleep_minutes)} hrv: [{self.hrv_morning:.0f} {self.hrv_evening:.0f} {self.hrv_morning - self.hrv_evening:3.0f} {self.hrv_lf}/{self.hrv_hf}]" + return f""" +slept for {hhmm(self.sleep_minutes)} +hrv morning: {self.hrv_morning:.0f} +hrv evening: {self.hrv_evening:.0f} +recovery: {self.hrv_morning - self.hrv_evening:3.0f} +{self.hrv_lf}/{self.hrv_hf}""".replace('\n', ' ') + def __str__(self) -> str: + return f"from {self.sleep_start} to {self.sleep_end}" + # measured_datapoints # [[timestamp, pulse, breath?, ??? hrv?]] # every 4 seconds? @property @@ -117,6 +125,10 @@ class Emfit: res.append(rmssd) return tss, res + @property + def measured_hr_avg(self): + return self.jj["measured_hr_avg"] + @property def sleep_hr_coverage(self): tss, hrs = self.sleep_hr diff --git a/emfit/__main__.py b/emfit/__main__.py new file mode 100644 index 0000000..1e6012f --- /dev/null +++ b/emfit/__main__.py @@ -0,0 +1,23 @@ +from emfit import get_datas + +for e in get_datas(): + # print("-------") + print(f"{e.end} {e.measured_hr_avg} {e.summary}") + + +# TODO get average HR +# TODO get 'quality', that is amount of time it actually had signal + +from kython.plotting import plot_timestamped +everything = get_datas() +tss = [e.end for e in everything] +hrs = [e.measured_hr_avg for e in everything] + +plot_timestamped( + tss, + hrs, + ratio=(15, 3), + mavgs=[(5, 'blue'), (10, 'green')], + marker='.', + ylimits=[40, 70], + ).savefig('hrs.png') diff --git a/plot.py b/plot.py old mode 100644 new mode 100755 index f6e3139..e160e7b --- a/plot.py +++ b/plot.py @@ -1,8 +1,11 @@ +#!/usr/bin/env python3 import matplotlib.dates as md # type: ignore import numpy as np # type: ignore import seaborn as sns # type: ignore import matplotlib.pyplot as plt +from emfit import get_datas + def plot_file(jj: str): pts = jj['sleep_epoch_datapoints'] @@ -87,6 +90,7 @@ def plot_recovery_vs_hr_percentage(): plt.show() +# TODO ah. it's only last segment? def plot_hr(): jj = get_datas()[-1] tss, uu = jj.sleep_hr @@ -94,7 +98,19 @@ def plot_hr(): uu = uu[::10] plt.figure(figsize=(15,4)) ax = sns.pointplot(tss, uu, markers=" ") - ax.set(ylim=(None, 1000)) + # TODO wtf is that/?? + ax.set(ylim=(None, 200)) + + plt.show() + +def plot_hr_trend(): + everything = get_datas() + tss = [e.end for e in everything] + hrs = [e.measured_hr_avg for e in everything] + plt.figure(figsize=(15,4)) + ax = sns.pointplot(tss, hrs) # , markers=" ") + # TODO wtf is that/?? + ax.set(ylim=(None, 70)) plt.show() @@ -111,6 +127,7 @@ def plot_hr(): # stats() # plot_recovery_vs_hr_percentage() # stats() +plot_hr_trend() # import matplotlib # matplotlib.use('Agg') diff --git a/run b/run new file mode 100755 index 0000000..fe846cd --- /dev/null +++ b/run @@ -0,0 +1,2 @@ +#!/bin/bash +python3 -m emfit diff --git a/test.py b/test.py new file mode 100755 index 0000000..aac0c8f --- /dev/null +++ b/test.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +from emfit import get_datas + +for d in get_datas(): + print(d) From 5658327883d3d8dc9a5463f32646a7eb65e7278b Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 13 Nov 2018 21:41:21 +0000 Subject: [PATCH 04/19] some updates --- emfit/__init__.py | 123 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 106 insertions(+), 17 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index bc34b26..f9ae52d 100644 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -4,7 +4,7 @@ from os.path import join from functools import lru_cache from datetime import timedelta, datetime -from typing import List, Dict, Iterator +from typing import List, Dict, Iterator, NamedTuple fromts = datetime.fromtimestamp @@ -13,10 +13,20 @@ def hhmm(minutes): PATH = "/L/backups/emfit/" +EXCLUDED = [ + '***REMOVED***', # pretty weird, detected sleep and HR (!) during the day when I was at work + '***REMOVED***', +] + AWAKE = 4 +class Point(NamedTuple): + ts: int + pulse: float + class Emfit: - def __init__(self, jj): + def __init__(self, sid: str, jj): + self.sid = sid self.jj = jj @property @@ -31,10 +41,16 @@ class Emfit: def date(self): return self.end.date() + """ + Bed time, not necessarily sleep + """ @property def start(self): return fromts(self.jj['time_start']) + """ + Bed time, not necessarily sleep + """ @property def end(self): return fromts(self.jj['time_end']) @@ -63,10 +79,15 @@ class Emfit: # 'sleep_epoch_datapoints' # [[timestamp, number]] + @property # type: ignore + @lru_cache() + def time_in_bed(self): + return int((self.sleep_end - self.sleep_start).total_seconds()) // 60 + # so it's actual sleep, without awake # ok, so I need time_asleep @property - def sleep_minutes(self): + def sleep_minutes_emfit(self): return self.jj['sleep_duration'] // 60 @property @@ -79,32 +100,94 @@ class Emfit: @property def summary(self): - return f""" -slept for {hhmm(self.sleep_minutes)} + return f"""in bed for {hhmm(self.time_in_bed)} +emfit time: {hhmm(self.sleep_minutes_emfit)}; covered: {self.sleep_hr_coverage:.0f} hrv morning: {self.hrv_morning:.0f} hrv evening: {self.hrv_evening:.0f} +avg hr: {self.measured_hr_avg:.0f} recovery: {self.hrv_morning - self.hrv_evening:3.0f} -{self.hrv_lf}/{self.hrv_hf}""".replace('\n', ' ') +{self.hrv_lf}/{self.hrv_hf}""" + @property + def strip_awakes(self): + ff = None + ll = None + for i, [ts, e] in enumerate(self.epochs): + if e != AWAKE: + ff = i + break + for i in range(len(self.epochs) - 1, -1, -1): + [ts, e] = self.epochs[i] + if e != AWAKE: + ll = i + break + return self.epochs[ff: ll] + + + # # TODO epochs with implicit sleeps? not sure... e.g. night wakeups. + # # I guess I could input intervals/correct data/exclude days manually? + # @property + # def pulse_percentage(self): + + # # TODO pulse intervals are 4 seconds? + # # TODO ok, how to compute that?... + # # TODO cut ff start and end? + # # TODO remove awakes from both sides + # sp = self.strip_awakes + # present = {ep[0] for ep in sp} + # start = min(present) + # end = max(present) + # # TODO get start and end in one go? + + # for p in self.iter_points(): + # p.ts + + + # INT = 30 + + # missing = 0 + # total = 0 + # for tt in range(start, end + INT, INT): + # total += 1 + # if tt not in present: + # missing += 1 + # # TODO get hr instead! + # import ipdb; ipdb.set_trace() + # return missing + + + # st = st[0][0] + # INT = 30 + # for [ts, e] in sp: + # if e == AWAKE: + # continue + # return fromts(ts) + # raise RuntimeError + # pass def __str__(self) -> str: return f"from {self.sleep_start} to {self.sleep_end}" # measured_datapoints # [[timestamp, pulse, breath?, ??? hrv?]] # every 4 seconds? - @property - def sleep_hr(self): - tss = [] - res = [] + + def iter_points(self): for ll in self.jj['measured_datapoints']: [ts, pulse, br, activity] = ll # TODO what the fuck is whaat?? It can't be HRV, it's about 500 ms on average # act in csv.. so it must be activity? wonder how is it measured. # but I guess makes sense. yeah, "measured_activity_avg": 595, about that # makes even more sense given tossturn datapoints only have timestamp - if self.sleep_start < fromts(ts) < self.sleep_end: - tss.append(ts) - res.append(pulse) + yield Point(ts=ts, pulse=pulse) + + @property + def sleep_hr(self): + tss = [] + res = [] + for p in self.iter_points(): + if self.sleep_start < fromts(p.ts) < self.sleep_end: + tss.append(p.ts) + res.append(p.pulse) return tss, res @property @@ -132,19 +215,25 @@ recovery: {self.hrv_morning - self.hrv_evening:3.0f} @property def sleep_hr_coverage(self): tss, hrs = self.sleep_hr - covered_sec = len([h for h in hrs if h is not None]) - expected_sec = self.sleep_minutes * 60 / 4 - return covered_sec / expected_sec * 100 + covered = len([h for h in hrs if h is not None]) + expected = len(hrs) + return covered / expected * 100 def iter_datas() -> Iterator[Emfit]: import os for f in sorted(os.listdir(PATH)): if not f.endswith('.json'): continue + sid = f[:-len('.json')] + if sid in EXCLUDED: + continue with open(join(PATH, f), 'r') as fo: - ef = Emfit(json_load(fo)) + ef = Emfit(sid, json_load(fo)) yield ef def get_datas() -> List[Emfit]: return list(sorted(list(iter_datas()), key=lambda e: e.start)) + + +# TODO move away old entries if there is a diff?? From e11cef6f91ef4b0850b352170f515bfe308ab7d1 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 29 Mar 2019 22:16:12 +0000 Subject: [PATCH 05/19] get sleep for each night --- emfit/__init__.py | 61 ++++++++++++++++++++++++++++++++++++----------- plot.py | 1 + 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index f9ae52d..911810e 100644 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -2,12 +2,17 @@ from kython import json_load from datetime import datetime from os.path import join from functools import lru_cache - -from datetime import timedelta, datetime +import logging +from datetime import timedelta, datetime, date from typing import List, Dict, Iterator, NamedTuple +from collections import OrderedDict as odict + fromts = datetime.fromtimestamp +def get_logger(): + return logging.getLogger('emfit-provider') + def hhmm(minutes): return '{:02d}:{:02d}'.format(*divmod(minutes, 60)) @@ -16,14 +21,12 @@ PATH = "/L/backups/emfit/" EXCLUDED = [ '***REMOVED***', # pretty weird, detected sleep and HR (!) during the day when I was at work '***REMOVED***', + + '***REMOVED***', # some weird sleep during the day? ] AWAKE = 4 -class Point(NamedTuple): - ts: int - pulse: float - class Emfit: def __init__(self, sid: str, jj): self.sid = sid @@ -37,6 +40,7 @@ class Emfit: def hrv_evening(self): return self.jj['hrv_rmssd_evening'] + # ok, I guess that's reasonable way of defining sleep date @property def date(self): return self.end.date() @@ -59,6 +63,15 @@ class Emfit: def epochs(self): return self.jj['sleep_epoch_datapoints'] + @property + def epoch_series(self): + tss = [] + eps = [] + for [ts, e] in self.epochs: + tss.append(ts) + eps.append(e) + return tss, eps + @property # type: ignore @lru_cache() def sleep_start(self) -> datetime: @@ -98,6 +111,10 @@ class Emfit: def hrv_hf(self): return self.jj['hrv_hf'] + @property + def recovery(self): + return self.hrv_morning - self.hrv_evening + @property def summary(self): return f"""in bed for {hhmm(self.time_in_bed)} @@ -105,7 +122,7 @@ emfit time: {hhmm(self.sleep_minutes_emfit)}; covered: {self.sleep_hr_coverage:. hrv morning: {self.hrv_morning:.0f} hrv evening: {self.hrv_evening:.0f} avg hr: {self.measured_hr_avg:.0f} -recovery: {self.hrv_morning - self.hrv_evening:3.0f} +recovery: {self.recovery:3.0f} {self.hrv_lf}/{self.hrv_hf}""" @property @@ -178,18 +195,22 @@ recovery: {self.hrv_morning - self.hrv_evening:3.0f} # act in csv.. so it must be activity? wonder how is it measured. # but I guess makes sense. yeah, "measured_activity_avg": 595, about that # makes even more sense given tossturn datapoints only have timestamp - yield Point(ts=ts, pulse=pulse) + yield ts, pulse @property def sleep_hr(self): tss = [] res = [] - for p in self.iter_points(): - if self.sleep_start < fromts(p.ts) < self.sleep_end: - tss.append(p.ts) - res.append(p.pulse) + for ts, pulse in self.iter_points(): + if self.sleep_start < fromts(ts) < self.sleep_end: + tss.append(ts) + res.append(pulse) return tss, res + @property + def sleep_hr_series(self): + return self.sleep_hr + @property def hrv(self): tss = [] @@ -234,6 +255,18 @@ def iter_datas() -> Iterator[Emfit]: def get_datas() -> List[Emfit]: return list(sorted(list(iter_datas()), key=lambda e: e.start)) - - # TODO move away old entries if there is a diff?? + +from kython import group_by_key +def by_night() -> Dict[date, Emfit]: + logger = get_logger() + res: Dict[date, Emfit] = odict() + # TODO shit. I need some sort of interrupted sleep detection? + for dd, sleeps in group_by_key(get_datas(), key=lambda s: s.date).items(): + if len(sleeps) > 1: + logger.warning("multiple sleeps per night, not handled yet: %s", sleeps) + continue + [s] = sleeps + res[s.date] = s + return res + diff --git a/plot.py b/plot.py index e160e7b..0f0f4fb 100755 --- a/plot.py +++ b/plot.py @@ -91,6 +91,7 @@ def plot_recovery_vs_hr_percentage(): # TODO ah. it's only last segment? +# ok, handled in dashboard now def plot_hr(): jj = get_datas()[-1] tss, uu = jj.sleep_hr From 1fa1ead94da87bd29eddf51f86f7072c82f4146f Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 7 Apr 2019 16:13:29 +0100 Subject: [PATCH 06/19] add caching --- emfit/__init__.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index 911810e..a055aad 100644 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -1,13 +1,16 @@ -from kython import json_load from datetime import datetime -from os.path import join +from pathlib import Path from functools import lru_cache import logging from datetime import timedelta, datetime, date from typing import List, Dict, Iterator, NamedTuple +import json from collections import OrderedDict as odict +from kython import cproperty + + fromts = datetime.fromtimestamp def get_logger(): @@ -16,7 +19,7 @@ def get_logger(): def hhmm(minutes): return '{:02d}:{:02d}'.format(*divmod(minutes, 60)) -PATH = "/L/backups/emfit/" +PATH = Path("/L/backups/emfit") EXCLUDED = [ '***REMOVED***', # pretty weird, detected sleep and HR (!) during the day when I was at work @@ -32,6 +35,9 @@ class Emfit: self.sid = sid self.jj = jj + def __hash__(self): + return hash(self.sid) + @property def hrv_morning(self): return self.jj['hrv_rmssd_morning'] @@ -41,14 +47,14 @@ class Emfit: return self.jj['hrv_rmssd_evening'] # ok, I guess that's reasonable way of defining sleep date - @property + @cproperty def date(self): return self.end.date() """ Bed time, not necessarily sleep """ - @property + @cproperty def start(self): return fromts(self.jj['time_start']) @@ -240,23 +246,29 @@ recovery: {self.recovery:3.0f} expected = len(hrs) return covered / expected * 100 + +import functools + +@functools.lru_cache(1000) # TODO hmm. should I configure it dynamically??? +def get_emfit(sid: str, f: Path) -> Emfit: + return Emfit(sid=sid, jj=json.loads(f.read_text())) + + def iter_datas() -> Iterator[Emfit]: - import os - for f in sorted(os.listdir(PATH)): - if not f.endswith('.json'): - continue - sid = f[:-len('.json')] + for f in PATH.glob('*.json'): + sid = f.stem if sid in EXCLUDED: continue - with open(join(PATH, f), 'r') as fo: - ef = Emfit(sid, json_load(fo)) - yield ef + yield get_emfit(sid, f) + +# @functools.lru_cache() def get_datas() -> List[Emfit]: - return list(sorted(list(iter_datas()), key=lambda e: e.start)) + return list(sorted(iter_datas(), key=lambda e: e.start)) # TODO move away old entries if there is a diff?? + from kython import group_by_key def by_night() -> Dict[date, Emfit]: logger = get_logger() From 57be4a1d6328a4a052786d9a8909fb1d2e091744 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 7 Apr 2019 20:27:55 +0100 Subject: [PATCH 07/19] cleanup old stuff, move test to __init__ --- emfit/__init__.py | 5 +++ plot.py | 97 +---------------------------------------------- test.py | 5 --- 3 files changed, 6 insertions(+), 101 deletions(-) delete mode 100755 test.py diff --git a/emfit/__init__.py b/emfit/__init__.py index a055aad..ddb3389 100644 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -282,3 +282,8 @@ def by_night() -> Dict[date, Emfit]: res[s.date] = s return res + + +def test(): + for d in get_datas(): + assert len(d.epochs) > 0 diff --git a/plot.py b/plot.py index 0f0f4fb..ade18be 100755 --- a/plot.py +++ b/plot.py @@ -2,49 +2,10 @@ import matplotlib.dates as md # type: ignore import numpy as np # type: ignore import seaborn as sns # type: ignore -import matplotlib.pyplot as plt +import matplotlib.pyplot as plt # type: ignore from emfit import get_datas - -def plot_file(jj: str): - pts = jj['sleep_epoch_datapoints'] - - - tss = [datetime.fromtimestamp(p[0]) for p in pts] - vals = [p[1] for p in pts] - - plt.figure(figsize=(20,10)) - plt.plot(tss, vals) - - - xformatter = md.DateFormatter('%H:%M') - xlocator = md.MinuteLocator(interval = 15) - - ## Set xtick labels to appear every 15 minutes - plt.gcf().axes[0].xaxis.set_major_locator(xlocator) - - ## Format xtick labels as HH:MM - plt.gcf().axes[0].xaxis.set_major_formatter(xformatter) - - - - plt.xlabel('time') - plt.ylabel('phase') - plt.title('Sleep phases') - plt.grid(True) - plt.savefig(f"{f}.png") - plt.close() # TODO - # plt.show() - - pass - - -def plot_all(): - for jj in iter_datas(): - plot_file(jj) - - # def stats(): # for jj in iter_datas(): # # TODO fimezone?? @@ -63,47 +24,6 @@ def plot_all(): # print(*res) -def stats(): - datas = get_datas() - cur = datas[0].date - for jj in datas: - # import ipdb; ipdb.set_trace() - while cur < jj.date: - cur += timedelta(days=1) - if cur.weekday() == 0: - print("---") - if cur != jj.date: - print(" ") - # cur = jj.date - print(f"{jj.date.strftime('%m.%d %a')} {jj.hrv_morning:.0f} {jj.hrv_evening:.0f} {jj.hrv_morning - jj.hrv_evening:3.0f} {hhmm(jj.sleep_minutes)} {jj.hrv_lf}/{jj.hrv_hf} {jj.sleep_hr_coverage:3.0f}") - - -def plot_recovery_vs_hr_percentage(): - sns.set(color_codes=True) - xs = [] - ys = [] - for jj in get_datas(): - xs.append(jj.hrv_morning - jj.hrv_evening) - ys.append(jj.sleep_hr_coverage) - ax = sns.regplot(x=xs, y=ys) # "recovery", y="percentage", data=pdata) - ax.set(xlabel='recovery', ylabel='percentage') - plt.show() - - -# TODO ah. it's only last segment? -# ok, handled in dashboard now -def plot_hr(): - jj = get_datas()[-1] - tss, uu = jj.sleep_hr - tss = tss[::10] - uu = uu[::10] - plt.figure(figsize=(15,4)) - ax = sns.pointplot(tss, uu, markers=" ") - # TODO wtf is that/?? - ax.set(ylim=(None, 200)) - - plt.show() - def plot_hr_trend(): everything = get_datas() tss = [e.end for e in everything] @@ -119,28 +39,13 @@ def plot_hr_trend(): # also timeline should have dynamic filters? maybe by tags # then I could enable emfit feed and slog feed (pulled from all org notes) and see the correlation? also could pull workouts provider (and wlog) -- actually wlog processing could be moved to timeline too -# TODO could plot 'recovery' thing and see if it is impacted by workouts - - -# TODO time_start, time_end - -# plot_hrv() -# stats() -# plot_recovery_vs_hr_percentage() -# stats() plot_hr_trend() -# import matplotlib -# matplotlib.use('Agg') # TODO maybe rmssd should only be computed if we have a reasonable chunk of datas -# also, trust it only if it's stable # plot_timestamped([p[0] for p in pts], [p[1] for p in pts], mavgs=[]).savefig('res.png') # TODO X axes: show hours and only two dates # TODO 4 is awake, 3 REM, 2 light, 1 deep - - - # deviartion beyond 25-75 or 75-25 is bad?? diff --git a/test.py b/test.py deleted file mode 100755 index aac0c8f..0000000 --- a/test.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -from emfit import get_datas - -for d in get_datas(): - print(d) From 397c50ffcc381b51e46c75163866301834ca4487 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 7 Apr 2019 20:58:45 +0100 Subject: [PATCH 08/19] cleanup from old shit --- emfit/__main__.py | 23 ----------------------- plot.py | 25 ------------------------- 2 files changed, 48 deletions(-) delete mode 100644 emfit/__main__.py diff --git a/emfit/__main__.py b/emfit/__main__.py deleted file mode 100644 index 1e6012f..0000000 --- a/emfit/__main__.py +++ /dev/null @@ -1,23 +0,0 @@ -from emfit import get_datas - -for e in get_datas(): - # print("-------") - print(f"{e.end} {e.measured_hr_avg} {e.summary}") - - -# TODO get average HR -# TODO get 'quality', that is amount of time it actually had signal - -from kython.plotting import plot_timestamped -everything = get_datas() -tss = [e.end for e in everything] -hrs = [e.measured_hr_avg for e in everything] - -plot_timestamped( - tss, - hrs, - ratio=(15, 3), - mavgs=[(5, 'blue'), (10, 'green')], - marker='.', - ylimits=[40, 70], - ).savefig('hrs.png') diff --git a/plot.py b/plot.py index ade18be..87ec6e8 100755 --- a/plot.py +++ b/plot.py @@ -1,11 +1,3 @@ -#!/usr/bin/env python3 -import matplotlib.dates as md # type: ignore -import numpy as np # type: ignore -import seaborn as sns # type: ignore -import matplotlib.pyplot as plt # type: ignore - -from emfit import get_datas - # def stats(): # for jj in iter_datas(): # # TODO fimezone?? @@ -24,27 +16,10 @@ from emfit import get_datas # print(*res) -def plot_hr_trend(): - everything = get_datas() - tss = [e.end for e in everything] - hrs = [e.measured_hr_avg for e in everything] - plt.figure(figsize=(15,4)) - ax = sns.pointplot(tss, hrs) # , markers=" ") - # TODO wtf is that/?? - ax.set(ylim=(None, 70)) - - plt.show() - -# TODO ok, would be nice to have that every morning in timeline -# also timeline should have dynamic filters? maybe by tags # then I could enable emfit feed and slog feed (pulled from all org notes) and see the correlation? also could pull workouts provider (and wlog) -- actually wlog processing could be moved to timeline too -plot_hr_trend() - - # TODO maybe rmssd should only be computed if we have a reasonable chunk of datas -# plot_timestamped([p[0] for p in pts], [p[1] for p in pts], mavgs=[]).savefig('res.png') # TODO X axes: show hours and only two dates # TODO 4 is awake, 3 REM, 2 light, 1 deep From 8b5a88f36f575eead0fd38d6b6ed35f1dc396fbf Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 7 Apr 2019 21:04:15 +0100 Subject: [PATCH 09/19] use cproperty --- emfit/__init__.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index ddb3389..004459b 100644 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -78,8 +78,7 @@ class Emfit: eps.append(e) return tss, eps - @property # type: ignore - @lru_cache() + @cproperty def sleep_start(self) -> datetime: for [ts, e] in self.epochs: if e == AWAKE: @@ -87,8 +86,7 @@ class Emfit: return fromts(ts) raise RuntimeError - @property # type: ignore - @lru_cache() + @cproperty def sleep_end(self) -> datetime: for [ts, e] in reversed(self.epochs): if e == AWAKE: @@ -98,8 +96,7 @@ class Emfit: # 'sleep_epoch_datapoints' # [[timestamp, number]] - @property # type: ignore - @lru_cache() + @cproperty def time_in_bed(self): return int((self.sleep_end - self.sleep_start).total_seconds()) // 60 From 4f49c2c4acd62b31e3d61905e59b54a2ec7942c9 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 8 Apr 2019 19:44:23 +0100 Subject: [PATCH 10/19] cache more things --- emfit/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index 004459b..a629df8 100644 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -47,14 +47,14 @@ class Emfit: return self.jj['hrv_rmssd_evening'] # ok, I guess that's reasonable way of defining sleep date - @cproperty + @property def date(self): return self.end.date() """ Bed time, not necessarily sleep """ - @cproperty + @property def start(self): return fromts(self.jj['time_start']) @@ -78,6 +78,7 @@ class Emfit: eps.append(e) return tss, eps + # TODO are these utc?? should be visible on big plot @cproperty def sleep_start(self) -> datetime: for [ts, e] in self.epochs: @@ -236,7 +237,7 @@ recovery: {self.recovery:3.0f} def measured_hr_avg(self): return self.jj["measured_hr_avg"] - @property + @cproperty def sleep_hr_coverage(self): tss, hrs = self.sleep_hr covered = len([h for h in hrs if h is not None]) @@ -260,13 +261,14 @@ def iter_datas() -> Iterator[Emfit]: yield get_emfit(sid, f) -# @functools.lru_cache() def get_datas() -> List[Emfit]: return list(sorted(iter_datas(), key=lambda e: e.start)) # TODO move away old entries if there is a diff?? - +from kython import timed from kython import group_by_key + +@timed def by_night() -> Dict[date, Emfit]: logger = get_logger() res: Dict[date, Emfit] = odict() From 04586fa3bc1ce8838d61d150036d4536bd7f2cd0 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 11 Apr 2019 11:11:00 +0100 Subject: [PATCH 11/19] fix timezone --- emfit/__init__.py | 50 ++++++++++++++++++++++++++++++++++++++++------- run | 2 -- 2 files changed, 43 insertions(+), 9 deletions(-) mode change 100644 => 100755 emfit/__init__.py delete mode 100755 run diff --git a/emfit/__init__.py b/emfit/__init__.py old mode 100644 new mode 100755 index a629df8..09a724e --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -1,18 +1,17 @@ -from datetime import datetime +#!/usr/bin/env python3 +from datetime import datetime, time from pathlib import Path from functools import lru_cache import logging +from collections import OrderedDict as odict from datetime import timedelta, datetime, date from typing import List, Dict, Iterator, NamedTuple import json - -from collections import OrderedDict as odict +import pytz from kython import cproperty -fromts = datetime.fromtimestamp - def get_logger(): return logging.getLogger('emfit-provider') @@ -30,6 +29,15 @@ EXCLUDED = [ AWAKE = 4 +# TODO use tz provider for that? although emfit is always in london... + +_TZ = pytz.timezone('Europe/London') + +def fromts(ts) -> datetime: + dt = datetime.fromtimestamp(ts) + return _TZ.localize(dt) + + class Emfit: def __init__(self, sid: str, jj): self.sid = sid @@ -78,7 +86,6 @@ class Emfit: eps.append(e) return tss, eps - # TODO are these utc?? should be visible on big plot @cproperty def sleep_start(self) -> datetime: for [ts, e] in self.epochs: @@ -284,5 +291,34 @@ def by_night() -> Dict[date, Emfit]: def test(): - for d in get_datas(): + datas = get_datas() + for d in datas: assert len(d.epochs) > 0 + + +def test_tz(): + datas = get_datas() + for d in datas: + assert d.start.tzinfo is not None + assert d.end.tzinfo is not None + assert d.sleep_start.tzinfo is not None + assert d.sleep_end.tzinfo is not None + + # https://qs.emfit.com/#/device/presence/***REMOVED*** + # this was winter time, so GMT, UTC+0 + sid_20190109 = '***REMOVED***' + [s0109] = [s for s in datas if s.sid == sid_20190109] + assert s0109.end.time() == time(hour=6, minute=42) + + # summer time, so UTC+1 + sid_20190411 = '***REMOVED***' + [s0411] = [s for s in datas if s.sid == sid_20190411] + assert s0411.end.time() == time(hour=9, minute=30) + + +def main(): + for k, v in by_night().items(): + print(k, v.start, v.end) + +if __name__ == '__main__': + main() diff --git a/run b/run deleted file mode 100755 index fe846cd..0000000 --- a/run +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -python3 -m emfit From 2ba0ad6f2b6e5a6572e1dc159f75b53da5c4971c Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 13 Apr 2019 14:14:33 +0100 Subject: [PATCH 12/19] handle missing epochs defensively --- emfit/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index 09a724e..01aa482 100755 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -9,7 +9,7 @@ from typing import List, Dict, Iterator, NamedTuple import json import pytz -from kython import cproperty +from kython import cproperty, timed, group_by_key def get_logger(): @@ -252,9 +252,8 @@ recovery: {self.recovery:3.0f} return covered / expected * 100 -import functools -@functools.lru_cache(1000) # TODO hmm. should I configure it dynamically??? +@lru_cache(1000) # TODO hmm. should I configure it dynamically??? def get_emfit(sid: str, f: Path) -> Emfit: return Emfit(sid=sid, jj=json.loads(f.read_text())) @@ -272,8 +271,6 @@ def get_datas() -> List[Emfit]: return list(sorted(iter_datas(), key=lambda e: e.start)) # TODO move away old entries if there is a diff?? -from kython import timed -from kython import group_by_key @timed def by_night() -> Dict[date, Emfit]: @@ -285,6 +282,11 @@ def by_night() -> Dict[date, Emfit]: logger.warning("multiple sleeps per night, not handled yet: %s", sleeps) continue [s] = sleeps + + if s.epochs is None: + logger.error('%s (on %s) got None in epochs! ignoring', s.sid, dd) + continue + res[s.date] = s return res @@ -321,4 +323,6 @@ def main(): print(k, v.start, v.end) if __name__ == '__main__': + from kython.klogging import setup_logzero + setup_logzero(get_logger()) main() From 8c5bdf060383d8875256809b86739a22a25ac0da Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 4 Aug 2019 12:55:12 +0100 Subject: [PATCH 13/19] use cachew --- emfit/__init__.py | 98 ++++++++++++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 35 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index 01aa482..4de056a 100755 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -1,20 +1,25 @@ #!/usr/bin/env python3 -from datetime import datetime, time -from pathlib import Path -from functools import lru_cache +import json import logging from collections import OrderedDict as odict -from datetime import timedelta, datetime, date -from typing import List, Dict, Iterator, NamedTuple -import json -import pytz +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta +from functools import lru_cache +from pathlib import Path +from typing import Dict, Iterator, List, NamedTuple -from kython import cproperty, timed, group_by_key +import kython +import pytz +from kython import cproperty, group_by_key + +from cachew import cachew def get_logger(): return logging.getLogger('emfit-provider') +timed = lambda f: kython.timed(f, logger=get_logger()) + def hhmm(minutes): return '{:02d}:{:02d}'.format(*divmod(minutes, 60)) @@ -29,8 +34,9 @@ EXCLUDED = [ AWAKE = 4 -# TODO use tz provider for that? although emfit is always in london... +Sid = str +# TODO FIXME use tz provider for that? although emfit is always in london... _TZ = pytz.timezone('Europe/London') def fromts(ts) -> datetime: @@ -38,7 +44,23 @@ def fromts(ts) -> datetime: return _TZ.localize(dt) -class Emfit: +class Mixin: + @property + # ok, I guess that's reasonable way of defining sleep date + def date(self): + return self.end.date() + + @cproperty + def time_in_bed(self): + return int((self.sleep_end - self.sleep_start).total_seconds()) // 60 + + @property + def recovery(self): + return self.hrv_morning - self.hrv_evening + + +# TODO def use multiple threads for that.. +class EmfitOld(Mixin): def __init__(self, sid: str, jj): self.sid = sid self.jj = jj @@ -54,11 +76,6 @@ class Emfit: def hrv_evening(self): return self.jj['hrv_rmssd_evening'] - # ok, I guess that's reasonable way of defining sleep date - @property - def date(self): - return self.end.date() - """ Bed time, not necessarily sleep """ @@ -104,10 +121,6 @@ class Emfit: # 'sleep_epoch_datapoints' # [[timestamp, number]] - @cproperty - def time_in_bed(self): - return int((self.sleep_end - self.sleep_start).total_seconds()) // 60 - # so it's actual sleep, without awake # ok, so I need time_asleep @property @@ -122,10 +135,6 @@ class Emfit: def hrv_hf(self): return self.jj['hrv_hf'] - @property - def recovery(self): - return self.hrv_morning - self.hrv_evening - @property def summary(self): return f"""in bed for {hhmm(self.time_in_bed)} @@ -252,19 +261,43 @@ recovery: {self.recovery:3.0f} return covered / expected * 100 +# right, so dataclass is better because you can use mixins +@dataclass(eq=True, frozen=True) +class Emfit(Mixin): + sid: Sid + hrv_morning: float + hrv_evening: float + start: datetime + end : datetime + sleep_start: datetime + sleep_end : datetime + sleep_hr_coverage: float + measured_hr_avg: float -@lru_cache(1000) # TODO hmm. should I configure it dynamically??? -def get_emfit(sid: str, f: Path) -> Emfit: - return Emfit(sid=sid, jj=json.loads(f.read_text())) + @classmethod + def make(cls, em) -> Iterator['Emfit']: + # TODO FIXME res? + logger = get_logger() + if em.epochs is None: + logger.error('%s (on %s) got None in epochs! ignoring', em.sid, em.date) + return + + yield cls(**{ + k: getattr(em, k) for k in Emfit.__annotations__ + }) + +# TODO very nice! +@cachew def iter_datas() -> Iterator[Emfit]: - for f in PATH.glob('*.json'): + for f in sorted(PATH.glob('*.json')): sid = f.stem if sid in EXCLUDED: continue - yield get_emfit(sid, f) + em = EmfitOld(sid=sid, jj=json.loads(f.read_text())) + yield from Emfit.make(em) def get_datas() -> List[Emfit]: @@ -282,11 +315,6 @@ def by_night() -> Dict[date, Emfit]: logger.warning("multiple sleeps per night, not handled yet: %s", sleeps) continue [s] = sleeps - - if s.epochs is None: - logger.error('%s (on %s) got None in epochs! ignoring', s.sid, dd) - continue - res[s.date] = s return res @@ -319,10 +347,10 @@ def test_tz(): def main(): + from kython.klogging import setup_logzero + setup_logzero(get_logger(), level=logging.DEBUG) for k, v in by_night().items(): print(k, v.start, v.end) if __name__ == '__main__': - from kython.klogging import setup_logzero - setup_logzero(get_logger()) main() From e874d46ece79a92a4f2fa24649f2fccbad23be0e Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 4 Aug 2019 13:02:56 +0100 Subject: [PATCH 14/19] Use proper cache --- emfit/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index 4de056a..e8374d5 100755 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -4,7 +4,6 @@ import logging from collections import OrderedDict as odict from dataclasses import dataclass from datetime import date, datetime, time, timedelta -from functools import lru_cache from pathlib import Path from typing import Dict, Iterator, List, NamedTuple @@ -287,11 +286,15 @@ class Emfit(Mixin): }) +# TODO move to common? +def dir_hash(path: Path): + mtimes = tuple(p.stat().st_mtime for p in sorted(path.glob('*.json'))) + return mtimes -# TODO very nice! -@cachew -def iter_datas() -> Iterator[Emfit]: - for f in sorted(PATH.glob('*.json')): + +@cachew(db_path=Path('/L/data/.cache/emfit.cache'), hashf=dir_hash) +def iter_datas(path: Path) -> Iterator[Emfit]: + for f in sorted(path.glob('*.json')): sid = f.stem if sid in EXCLUDED: continue @@ -301,7 +304,7 @@ def iter_datas() -> Iterator[Emfit]: def get_datas() -> List[Emfit]: - return list(sorted(iter_datas(), key=lambda e: e.start)) + return list(sorted(iter_datas(PATH), key=lambda e: e.start)) # TODO move away old entries if there is a diff?? From 419d072f8cf97b4b6594b9e12d128b0075ab3780 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 4 Aug 2019 16:15:39 +0100 Subject: [PATCH 15/19] use more fields --- emfit/__init__.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index e8374d5..dcd7486 100755 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -57,6 +57,16 @@ class Mixin: def recovery(self): return self.hrv_morning - self.hrv_evening + @property + def summary(self): + return f"""in bed for {hhmm(self.time_in_bed)} +emfit time: {hhmm(self.sleep_minutes_emfit)}; covered: {self.sleep_hr_coverage:.0f} +hrv morning: {self.hrv_morning:.0f} +hrv evening: {self.hrv_evening:.0f} +avg hr: {self.measured_hr_avg:.0f} +recovery: {self.recovery:3.0f} +{self.hrv_lf}/{self.hrv_hf}""" + # TODO def use multiple threads for that.. class EmfitOld(Mixin): @@ -134,16 +144,6 @@ class EmfitOld(Mixin): def hrv_hf(self): return self.jj['hrv_hf'] - @property - def summary(self): - return f"""in bed for {hhmm(self.time_in_bed)} -emfit time: {hhmm(self.sleep_minutes_emfit)}; covered: {self.sleep_hr_coverage:.0f} -hrv morning: {self.hrv_morning:.0f} -hrv evening: {self.hrv_evening:.0f} -avg hr: {self.measured_hr_avg:.0f} -recovery: {self.recovery:3.0f} -{self.hrv_lf}/{self.hrv_hf}""" - @property def strip_awakes(self): ff = None @@ -272,6 +272,9 @@ class Emfit(Mixin): sleep_end : datetime sleep_hr_coverage: float measured_hr_avg: float + sleep_minutes_emfit: int + hrv_lf: float + hrv_hf: float @classmethod def make(cls, em) -> Iterator['Emfit']: From 9b6e857bbbd0e59092c00d0eb25cb1ae4384e356 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 4 Aug 2019 16:23:25 +0100 Subject: [PATCH 16/19] fix yielding --- emfit/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index dcd7486..da1d831 100755 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -296,7 +296,7 @@ def dir_hash(path: Path): @cachew(db_path=Path('/L/data/.cache/emfit.cache'), hashf=dir_hash) -def iter_datas(path: Path) -> Iterator[Emfit]: +def iter_datas_cached(path: Path) -> Iterator[Emfit]: for f in sorted(path.glob('*.json')): sid = f.stem if sid in EXCLUDED: @@ -306,8 +306,12 @@ def iter_datas(path: Path) -> Iterator[Emfit]: yield from Emfit.make(em) +def iter_datas(path=PATH) -> Iterator[Emfit]: + yield from iter_datas_cached(path) + + def get_datas() -> List[Emfit]: - return list(sorted(iter_datas(PATH), key=lambda e: e.start)) + return list(sorted(iter_datas(), key=lambda e: e.start)) # TODO move away old entries if there is a diff?? From 69d88a53153d6bbbb789c14e561e22abbecf4a7e Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 5 Aug 2019 18:41:31 +0100 Subject: [PATCH 17/19] fix ruci --- emfit/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index da1d831..aa652ca 100755 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -285,6 +285,7 @@ class Emfit(Mixin): return yield cls(**{ + # pylint: disable=no-member k: getattr(em, k) for k in Emfit.__annotations__ }) @@ -333,7 +334,7 @@ def by_night() -> Dict[date, Emfit]: def test(): datas = get_datas() for d in datas: - assert len(d.epochs) > 0 + pass def test_tz(): From d3661d65b5c6b7cc757df17e581ef2964a4c4d39 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 14 Aug 2019 23:45:54 +0100 Subject: [PATCH 18/19] fix cachew --- emfit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index aa652ca..1238aaf 100755 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -296,7 +296,7 @@ def dir_hash(path: Path): return mtimes -@cachew(db_path=Path('/L/data/.cache/emfit.cache'), hashf=dir_hash) +@cachew(cache_path=Path('/L/data/.cache/emfit.cache'), hashf=dir_hash) def iter_datas_cached(path: Path) -> Iterator[Emfit]: for f in sorted(path.glob('*.json')): sid = f.stem From d25de5de6f91abe543d53f337daa06feb41c122d Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 2 Nov 2019 14:41:17 +0000 Subject: [PATCH 19/19] fix ruci --- emfit/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/emfit/__init__.py b/emfit/__init__.py index 1238aaf..370f926 100755 --- a/emfit/__init__.py +++ b/emfit/__init__.py @@ -5,7 +5,7 @@ from collections import OrderedDict as odict from dataclasses import dataclass from datetime import date, datetime, time, timedelta from pathlib import Path -from typing import Dict, Iterator, List, NamedTuple +from typing import Dict, Iterator, List, NamedTuple, Any, cast import kython import pytz @@ -44,18 +44,21 @@ def fromts(ts) -> datetime: class Mixin: + # TODO ugh. tricking mypy... + sleep_minutes_emfit: int + @property # ok, I guess that's reasonable way of defining sleep date def date(self): - return self.end.date() + return self.end.date() # type: ignore[attr-defined] @cproperty def time_in_bed(self): - return int((self.sleep_end - self.sleep_start).total_seconds()) // 60 + return int((self.sleep_end - self.sleep_start).total_seconds()) // 60 # type: ignore[attr-defined] @property def recovery(self): - return self.hrv_morning - self.hrv_evening + return self.hrv_morning - self.hrv_evening # type: ignore[attr-defined] @property def summary(self): @@ -65,7 +68,7 @@ hrv morning: {self.hrv_morning:.0f} hrv evening: {self.hrv_evening:.0f} avg hr: {self.measured_hr_avg:.0f} recovery: {self.recovery:3.0f} -{self.hrv_lf}/{self.hrv_hf}""" +{self.hrv_lf}/{self.hrv_hf}""" # type: ignore[attr-defined] # TODO def use multiple threads for that..