diff --git a/.projectile b/.projectile new file mode 100644 index 0000000..89335b7 --- /dev/null +++ b/.projectile @@ -0,0 +1 @@ +- /.mypy_cache/ diff --git a/Jawbone/features.csv b/Jawbone/features.csv new file mode 100644 index 0000000..7adb374 --- /dev/null +++ b/Jawbone/features.csv @@ -0,0 +1,81 @@ +Field,Definition +DATE,Date of column data (in year/month/day format). +age,Your age on the indicated date. +avg_bg,Average passive heart rate (in beats per minute). +bmr,Basal Metabolic Rate (in calories). +body_fat,Body fat percentage. +e_caffeine,Caffeine content consumed (in milligrams). +e_calcium,Calcium content consumed (in milligrams). +e_calories,Calories consumed. +e_carbs,Carbohydrates consumed (in grams). +e_cholesterol,Cholesterol consumed (in milligrams). +e_count,Number of meals logged. +e_fat,Fat consumed (in grams). +e_fiber,Fiber consumed (in grams). +e_iron,Percentage of recommended iron consumed (based on a 2000 calorie per day diet). +e_monounsaturated_fat,Monounsaturated fat consumed (in grams). +e_num_drinks,Number of drinks logged. +e_num_foods,Number of meal items logged. +e_num_mealitems_green,Number of meal items logged with a green UP Food Score. +e_num_mealitems_red,Number of meal items logged with a red UP Food Score. +e_num_mealitems_with_score,Number of meal items logged with an UP Food Score. +e_num_mealitems_yellow,Number of meal items logged with a yellow UP Food Score. +e_num_water,Glasses of water logged. +e_polyunsaturated_fat,Polyunsaturated fat consumed (in grams). +e_potassium,Potassium consumed (in milligrams). +e_protein,Protein consumed (in grams). +e_sat_fat,Saturated fat consumed (in grams). +e_sodium,Sodium consumed (in milligrams). +e_sugar,Sugar consumed (in grams). +e_trans_fat,Trans fat consumed (in grams). +e_unsat_fat,Unsaturated fat consumed (in grams). +e_vitamin_a,Percentage of recommended Vitamin A consumed (based on a 2000 calorie per day diet). +e_vitamin_c,Percentage of recommended Vitamin C consumed (based on a 2000 calorie per day diet). +gender,Your specified gender (0=male 1=female). +goal_body_weight,Weight goal (in kilograms). +goal_body_weight_intent,Weight goal preference (0=lose 1=maintain 2=gain). +height,Your specified height (in meters). +m_active_time,Amount of total active time (in seconds). +m_calories,Total number of calories burned during active time (in seconds). +m_distance,Total distance traveled (in meters). +m_inactive_time,Total inactive time (in seconds). +m_lcat,Longest consecutive active time (in seconds). +m_lcit,Longest consecutive inactive time (in seconds). +m_steps,Total number of steps taken. +m_steps_3am,Total number of steps taken (before 3am). +m_total_calories,Total number of calories burned in the day +m_workout_count,Number of workouts logged. +m_workout_time,Length of logged workouts (in seconds). +max_bg,Highest passive heart rate (in beats per minute). +min_bg,Lowest passive heart rate (in beats per minute). +n_asleep_time,Duration of naps/secondary sleep (in seconds). +n_awake,Total time awake during naps/secondary sleep (in seconds). +n_awake_time,Total time awake after waking from naps/secondary sleep (in seconds). +n_awakenings,Number of times awoken during naps/secondary sleep. +n_bedtime,Length of time band in sleep mode during naps/secondary sleep (in seconds). +n_clinical_deep,Length of Deep sleep during naps/secondary sleep (in seconds UP3 and UP4 only). +n_count,Number of naps/secondary sleep entries logged. +n_deep,Length of Sound sleep during naps/secondary sleep (in seconds). +n_duration,Duration of naps/secondary sleep (in seconds). +n_light,Length of Light sleep during naps/secondary sleep (in seconds). +n_quality,Not applicable. +n_rem,Length of REM sleep during naps/secondary sleep (in seconds). +n_to_bed_phr,Average passive heart rate 1 hour before naps/secondary sleep (in beats per minute). +num_readings,Number of background heart rate readings. +o_count,Number of moods logged. +o_mood,Average score for moods logged (10 = lowest 80 = highest). +rhr,Resting heart rate (in beats per minute). +s_asleep_time,Duration of primary sleep (in seconds). +s_awake,Total time awake during primary sleep (in seconds). +s_awake_time,Total time awake after waking from primary sleep (in seconds). +s_awakenings,Number of times awoken during primary sleep. +s_bedtime,Length of time band was in primary sleep mode (in seconds). +s_clinical_deep,Length of primary Deep sleep (in seconds UP3 and UP4 only). +s_count,Number of primary sleep entries logged. +s_deep,Length of primary Sound sleep (in seconds). +s_duration,Duration of primary sleep (in seconds). +s_light,Length of primary Light sleep (in seconds). +s_quality,Not applicable. +s_rem,Length of primary REM sleep (in seconds). +s_to_bed_phr,Average passive heart rate 1 hour before primary sleep (in beats per minute). +weight,Weight (in kilograms). diff --git a/TODO.org b/TODO.org new file mode 100644 index 0000000..ccea496 --- /dev/null +++ b/TODO.org @@ -0,0 +1,14 @@ +https://github.com/crowoy/Health-Analysis +https://github.com/joytafty-work/SleepModel +https://github.com/search?l=Jupyter+Notebook&q=s_awakenings&type=Code&utf8=%E2%9C%93 +https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/colifer.py +https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/reportextenders/jawbone/jawbone_sleep.py +https://github.com/GlenCrawford/ruby_jawbone + +* https://nyquist212.wordpress.com/2015/06/22/visualizing-jawbone-up-data-with-d3-js/ + + +* TODO ok, so shoud really do a week of consistent bedtime/waking up to make some final decision on jawbone? + +* TODO figure out timezones +* TODO post on reddit? release and ask people to run against their data? diff --git a/jawbone_provider/__init__.py b/jawbone_provider/__init__.py new file mode 100755 index 0000000..75c4f06 --- /dev/null +++ b/jawbone_provider/__init__.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +from typing import Dict, Any, List +import json +from functools import lru_cache +from datetime import datetime, date, time, timedelta +from pathlib import Path +import logging +import pytz + +from kython.klogging import setup_logzero + +BDIR = Path('/L/backups/jawbone') +PHASES_FILE = BDIR / 'phases.json' +SLEEPS_FILE = BDIR / 'sleeps.json' +GRAPHS_DIR = BDIR / 'graphs' + + +def get_logger(): + return logging.getLogger('jawbone-provider') + + +XID = str # TODO how to shared with backup thing? + +Phases = Dict[XID, Any] +@lru_cache(1) +def get_phases() -> Phases: + return json.loads(PHASES_FILE.read_text()) + +# TODO use awakenings and quality +class SleepEntry: + def __init__(self, js) -> None: + self.js = js + + # TODO @memoize decorator? + @property + def date_(self) -> date: + return self.sleep_end.date() + + def _fromts(self, ts: int) -> datetime: + return pytz.utc.localize(datetime.utcfromtimestamp(ts)).astimezone(self._tz).astimezone(self._tz) + @property + def _tz(self): + return pytz.timezone(self._details['tz']) + + @property + def title(self) -> str: + return self.js['title'] + + @property + def xid(self) -> XID: + return self.js['xid'] + + @property + def _details(self): + return self.js['details'] + + # TODO figure out timezones.. + # not sure how.. I guess by the american ones + @property + def created(self) -> datetime: + return self._fromts(self.js['time_created']) + + @property + def completed(self) -> datetime: + return self._fromts(self.js['time_completed']) + + @property + def asleep(self) -> datetime: + return self._fromts(self._details['asleep_time']) + + @property + def sleep_start(self) -> datetime: + return self.asleep # TODO careful, maybe use same logic as emfit + + @property + def bed_time(self) -> int: + return int((self.sleep_end - self.sleep_start).total_seconds()) // 60 + + @property + def sleep_end(self) -> datetime: + return self._fromts(self._details['awake_time']) + + @property + def graph(self) -> Path: + return GRAPHS_DIR / (self.xid + ".png") + + # TODO might be useful to cache these?? + @property + def phases(self) -> List[datetime]: + # TODO make sure they are consistent with emfit? + return [self._fromts(i['time']) for i in get_phases()[self.xid]] + + def __str__(self) -> str: + return f"{self.date_.strftime('%a %d %b')} {self.title}" + + def __repr__(self) -> str: + return str(self) + + +def load_sleeps() -> List[SleepEntry]: + sleeps = json.loads(SLEEPS_FILE.read_text()) + return [SleepEntry(js) for js in sleeps] + + +import numpy as np # type: ignore +import matplotlib.pyplot as plt # type: ignore +from matplotlib.figure import Figure # type: ignore +from matplotlib.axes import Axes # type: ignore + +# pip install imageio +# from imageio import imread # type: ignore + + +def hhmm(time: datetime): + return time.strftime("%H:%M") + + +# def xpos(time: datetime) -> float: +# tick = span / width +# fromstart = time - sleep.created +# return fromstart / tick + +import matplotlib.dates as mdates # type: ignore +from matplotlib.ticker import MultipleLocator, FixedLocator # type: ignore + +def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None, showtext=True): + span = sleep.completed - sleep.created + print(f"{sleep.xid} span: {span}") + + img = imread(sleep.graph) + # all of them are 300x300 images apparently + # span for image + xspan = [sleep.created, sleep.completed] + xspan = [mdates.date2num(i) for i in xspan] + if xlims is None: + tt = sleep.created + hour = tt.hour + # TODO maybe assert that hour is somewhere between 20 and 8 or something + start: datetime + starttime = time(23, 00) + if hour >= 20: + # went to bed before midnight + start = datetime.combine(tt.date(), starttime) + elif hour <= 8: + # went to bed after midnight + start = datetime.combine(tt.date() - timedelta(days=1), starttime) + else: + print("wtf??? weird time for sleep...") + # choosing at random + start = datetime.combine(tt.date(), starttime) + end = start + timedelta(hours=10) + xlims = [start, end] + + # axes.figure(figsize=(10, 5)) + axes.set_xlim(xlims) + hhmm_fmt = mdates.DateFormatter('%H:%M') + axes.xaxis.set_major_formatter(hhmm_fmt) + ticks = sleep.phases if showtext else [] + axes.xaxis.set_ticks(ticks) + axes.yaxis.set_ticks([]) + axes.tick_params( + axis='both', + which='major', + length=0, + labelsize=7, + rotation=30, + pad=-14, # err... hacky + ) + + ylims = [0, 50] + axes.set_ylim(ylims) + + axes.imshow( + img, + zorder=0, + extent=[ + xspan[0], xspan[1], + ylims[0], ylims[1], + ], + aspect='auto', + ) + # axes.set_title(str(sleep)) + # axes.title.set_size(10) + + if showtext: + axes.text(xlims[1] - timedelta(hours=1.5), 20, str(sleep),) + # plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) + +from kython import make_dict, group_by_key + +def sleeps_by_date() -> Dict[date, SleepEntry]: + logger = get_logger() + + sleeps = load_sleeps() + sleeps = [s for s in sleeps if s.graph.exists()] # TODO careful.. + res = {} + for dd, group in group_by_key(sleeps, key=lambda s: s.date_).items(): + if len(group) == 1: + res[dd] = group[0] + else: + # TODO short ones I can ignore I guess. but won't bother now + logger.error('multiple sleeps on %s: %s', dd, group) + return res + +# sleeps_count = 35 # len(sleeps) # apparently MPL fails at 298 with outofmemory or something +# start = 40 +# 65 is arount 1 july +# sleeps = sleeps[start: start + sleeps_count] +# sleeps = sleeps[:sleeps_count] +# dt = {k: v for k, v in dt.items() if v is not None} + +# TODO not really sure it belongs here... +# import melatonin +# dt = melatonin.get_data() + +def predicate(sleep: SleepEntry): + """ + Filter for comparing similar sleep sesssions + """ + start = sleep.created.time() + end = sleep.completed.time() + if (time(23, 0) <= start <= time(23, 30)) and (time(5, 30) <= end <= time(6, 30)): + return True + return False + + +def plot(): + # TODO ?? + sleeps = lfilter(predicate, sleeps) + sleeps_count = len(sleeps) + print(sleeps_count) + + fig: Figure = plt.figure(figsize=(15, sleeps_count * 1)) + + axarr = fig.subplots(nrows=len(sleeps)) + for i, (sleep, axes) in enumerate(zip(sleeps, axarr)): + plot_one(sleep, fig, axes, showtext=True) + used = dt.get(sleep.date_, None) + sused: str + color: str + # used = True if used is None else False # TODO? + if used is True: + sused = "YES" + color = 'green' + elif used is False: + sused = "NO" + color = 'red' + else: + sused = "??" + color = 'white' + axes.text(axes.get_xlim()[0], 20, sused) + axes.patch.set_alpha(0.5) + axes.set_facecolor(color) + + + plt.tight_layout() + plt.subplots_adjust(hspace=0.0) + # er... this saves with a different aspect ratio for some reason. + # tap 'ctrl-s' on mpl plot window to save.. + # plt.savefig('res.png', asp) + plt.show() + +import pandas as pd # type: ignore +def get_dataframe(): + sleeps = sleeps_by_date() + items = [] + for dd, s in sleeps.items(): + items.append({ + 'date' : dd, # TODO not sure... # TODO would also be great to sync column names... + 'sleep_start': s.sleep_start, + 'sleep_end' : s.sleep_end, + 'bed_time' : s.bed_time, + }) + # TODO tz is in sleeps json + res = pd.DataFrame(items) + return res + + +def test_tz(): + sleeps = sleeps_by_date() + for s in sleeps.values(): + assert s.sleep_start.tzinfo is not None + assert s.sleep_end.tzinfo is not None + + for dd, exp in [ + (date(year=2015, month=8 , day=28), time(hour=7, minute=20)), + (date(year=2015, month=9 , day=15), time(hour=6, minute=10)), + ]: + sleep = sleeps[dd] + end = sleep.sleep_end + + assert end.time() == exp + + # TODO fuck. on 0909 I woke up at around 6 according to google timeline + # but according to jawbone, it was on 0910?? eh. I guess it's jus shitty tracking. + + +def main(): + setup_logzero(get_logger()) + test_tz() + # print(get_dataframe()) + + +if __name__ == '__main__': + main() diff --git a/main.py b/main.py new file mode 100755 index 0000000..968e16e --- /dev/null +++ b/main.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# TODO +from kython import * +# from kython.plotting import * +from csv import DictReader +from itertools import islice + +from typing import Dict + +# sleep = [] +# with open('2017.csv', 'r') as fo: +# reader = DictReader(fo) +# for line in islice(reader, 0, 10): +# sleep +# print(line) + +import numpy as np +import matplotlib.pyplot as plt +from numpy import genfromtxt +import matplotlib.pylab as pylab + +pylab.rcParams['figure.figsize'] = (32.0, 24.0) +pylab.rcParams['font.size'] = 10 + +jawboneDataFeatures = "Jawbone/features.csv" # Data File Path +featureDesc: Dict[str, str] = {} +for x in genfromtxt(jawboneDataFeatures, dtype='unicode', delimiter=','): + featureDesc[x[0]] = x[1] + +def _safe_float(s: str): + if len(s) == 0: + return None + return float(s) + +def _safe_int(s: str): + if len(s) == 0: + return None + return int(float(s)) # TODO meh + +def _safe_mins(s: float): + if s is None: + return None + return s / 60 + +class SleepData(NamedTuple): + date: str + asleep_time: float + awake_time: float + total: float + awake: float # 'awake for' from app, time awake duing sleep (seconds) + awakenings: int + light: float # 'light sleep' from app (seconds) + deep: float # 'deep sleep' from app (sec) + quality: float # ??? + + @classmethod + def from_jawbone_dict(cls, d: Dict[str, Any]): + return cls( + date=d['DATE'], + asleep_time=_safe_mins(_safe_float(d['s_asleep_time'])), + awake_time=_safe_mins(_safe_float(d['s_awake_time'])), + total=_safe_mins(_safe_float(d['s_duration'])), + light=_safe_mins(_safe_float(d['s_light'])), + deep =_safe_mins(_safe_float(d['s_deep'])), + awake=_safe_mins(_safe_float(d['s_awake'])), + awakenings=_safe_int(d['s_awakenings']), + quality=_safe_float(d['s_quality']), + ) + + def is_bad(self): + return self.deep is None and self.light is None + + # @property + # def total(self) -> float: + # return self.light + self.deep + + + +def iter_useful(data_file: str): + from csv import DictReader + with open(data_file) as fo: + reader = DictReader(fo) + for d in reader: + dt = SleepData.from_jawbone_dict(d) + if not dt.is_bad(): + yield dt + +# TODO <<< hmm. these files do contain deep and light sleep?? +# also steps stats?? +p = Path('/L/backups/jawbone/old_csv') +# TODO with_my? +files = [ + p / "2015.csv", + p / "2016.csv", + p / "2017.csv", +] + +useful = concat(*(list(iter_useful(f)) for f in files)) + +# for u in useful: +# print(f"{u.total} {u.asleep_time} {u.awake_time}") +# # pprint(u.total) +# pprint(u) +# pprint("---") + +dates = [parse_date(u.date, yearfirst=True, dayfirst=False) for u in useful] +# TODO filter outliers? + +# TODO don't need this anymore? it's gonna be in dashboards package +from kython.plotting import plot_timestamped +for attr, lims, mavg, fig in [ + ('light', (0, 400), 5, None), + ('deep', (0, 600), 5, None), + ('total', (200, 600), 5, None), + ('awake_time', (0, 1200), None, 1), + ('asleep_time', (-100, 1000), None, 1), + # ('awakenings', (0, 5)), +]: + dates_wkd = [d for d in dates if d.weekday() < 5] + dates_wke = [d for d in dates if d.weekday() >= 5] + for dts, dn in [ + (dates, 'total'), + (dates_wkd, 'weekday'), + (dates_wke, 'weekend') + ]: + mavgs = [] + if mavg is not None: + mavgs.append((mavg, 'green')) + fig = plot_timestamped( + dts, + [getattr(u, attr) for u in useful], + marker='.', + ratio=(16, 4), + mavgs=mavgs, + ylimits=lims, + ytick_size=60, + # figure=1, + ) + plt.savefig(f'{attr}_{dn}.png') + +# TODO use proper names? +# plt.savefig('res.png') +# fig.show()