Merge remote-tracking branch 'jb/master'

.projectile

@@ -0,0 +1 @@
+- /.mypy_cache/

Jawbone/features.csv

@@ -0,0 +1,81 @@
+Field,Definition
+DATE,Date of column data (in year/month/day format). 
+age,Your age on the indicated date. 
+avg_bg,Average passive heart rate (in beats per minute).
+bmr,Basal Metabolic Rate (in calories). 
+body_fat,Body fat percentage. 
+e_caffeine,Caffeine content consumed (in milligrams).
+e_calcium,Calcium content consumed (in milligrams). 
+e_calories,Calories consumed. 
+e_carbs,Carbohydrates consumed (in grams). 
+e_cholesterol,Cholesterol consumed (in milligrams). 
+e_count,Number of meals logged. 
+e_fat,Fat consumed (in grams). 
+e_fiber,Fiber consumed (in grams). 
+e_iron,Percentage of recommended iron consumed (based on a 2000 calorie per day diet). 
+e_monounsaturated_fat,Monounsaturated fat consumed (in grams). 
+e_num_drinks,Number of drinks logged. 
+e_num_foods,Number of meal items logged. 
+e_num_mealitems_green,Number of meal items logged with a green UP Food Score.
+e_num_mealitems_red,Number of meal items logged with a red UP Food Score. 
+e_num_mealitems_with_score,Number of meal items logged with an UP Food Score. 
+e_num_mealitems_yellow,Number of meal items logged with a yellow UP Food Score. 
+e_num_water,Glasses of water logged. 
+e_polyunsaturated_fat,Polyunsaturated fat consumed (in grams). 
+e_potassium,Potassium consumed (in milligrams). 
+e_protein,Protein consumed (in grams). 
+e_sat_fat,Saturated fat consumed (in grams). 
+e_sodium,Sodium consumed (in milligrams). 
+e_sugar,Sugar consumed (in grams). 
+e_trans_fat,Trans fat consumed (in grams). 
+e_unsat_fat,Unsaturated fat consumed (in grams). 
+e_vitamin_a,Percentage of recommended Vitamin A consumed (based on a 2000 calorie per day diet). 
+e_vitamin_c,Percentage of recommended Vitamin C consumed (based on a 2000 calorie per day diet). 
+gender,Your specified gender (0=male 1=female). 
+goal_body_weight,Weight goal (in kilograms). 
+goal_body_weight_intent,Weight goal preference (0=lose 1=maintain 2=gain). 
+height,Your specified height (in meters). 
+m_active_time,Amount of total active time (in seconds). 
+m_calories,Total number of calories burned during active time (in seconds). 
+m_distance,Total distance traveled (in meters). 
+m_inactive_time,Total inactive time (in seconds). 
+m_lcat,Longest consecutive active time (in seconds). 
+m_lcit,Longest consecutive inactive time (in seconds). 
+m_steps,Total number of steps taken. 
+m_steps_3am,Total number of steps taken (before 3am). 
+m_total_calories,Total number of calories burned in the day
+m_workout_count,Number of workouts logged. 
+m_workout_time,Length of logged workouts (in seconds). 
+max_bg,Highest passive heart rate (in beats per minute). 
+min_bg,Lowest passive heart rate (in beats per minute). 
+n_asleep_time,Duration of naps/secondary sleep (in seconds). 
+n_awake,Total time awake during naps/secondary sleep (in seconds). 
+n_awake_time,Total time awake after waking from naps/secondary sleep (in seconds). 
+n_awakenings,Number of times awoken during naps/secondary sleep. 
+n_bedtime,Length of time band in sleep mode during naps/secondary sleep (in seconds). 
+n_clinical_deep,Length of Deep sleep during naps/secondary sleep (in seconds UP3 and UP4 only). 
+n_count,Number of naps/secondary sleep entries logged. 
+n_deep,Length of Sound sleep during naps/secondary sleep (in seconds). 
+n_duration,Duration of naps/secondary sleep (in seconds). 
+n_light,Length of Light sleep during naps/secondary sleep (in seconds). 
+n_quality,Not applicable. 
+n_rem,Length of REM sleep during naps/secondary sleep (in seconds). 
+n_to_bed_phr,Average passive heart rate 1 hour before naps/secondary sleep (in beats per minute). 
+num_readings,Number of background heart rate readings. 
+o_count,Number of moods logged. 
+o_mood,Average score for moods logged (10 = lowest 80 = highest). 
+rhr,Resting heart rate (in beats per minute). 
+s_asleep_time,Duration of primary sleep (in seconds). 
+s_awake,Total time awake during primary sleep (in seconds). 
+s_awake_time,Total time awake after waking from primary sleep (in seconds). 
+s_awakenings,Number of times awoken during primary sleep. 
+s_bedtime,Length of time band was in primary sleep mode (in seconds). 
+s_clinical_deep,Length of primary Deep sleep (in seconds UP3 and UP4 only). 
+s_count,Number of primary sleep entries logged. 
+s_deep,Length of primary Sound sleep (in seconds). 
+s_duration,Duration of primary sleep (in seconds). 
+s_light,Length of primary Light sleep (in seconds). 
+s_quality,Not applicable. 
+s_rem,Length of primary REM sleep (in seconds). 
+s_to_bed_phr,Average passive heart rate 1 hour before primary sleep (in beats per minute). 
+weight,Weight (in kilograms). 

TODO.org

@@ -0,0 +1,14 @@
+https://github.com/crowoy/Health-Analysis
+https://github.com/joytafty-work/SleepModel
+https://github.com/search?l=Jupyter+Notebook&q=s_awakenings&type=Code&utf8=%E2%9C%93
+https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/colifer.py
+https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/reportextenders/jawbone/jawbone_sleep.py
+https://github.com/GlenCrawford/ruby_jawbone
+
+* https://nyquist212.wordpress.com/2015/06/22/visualizing-jawbone-up-data-with-d3-js/
+
+
+* TODO ok, so shoud really do a week of consistent bedtime/waking up to make some final decision on jawbone?
+
+* TODO figure out timezones
+* TODO post on reddit? release and ask people to run against their data?

jawbone_provider/__init__.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+from typing import Dict, Any, List
+import json
+from functools import lru_cache
+from datetime import datetime, date, time, timedelta
+from pathlib import Path
+import logging
+import pytz
+
+from kython.klogging import setup_logzero
+
+BDIR = Path('/L/backups/jawbone')
+PHASES_FILE = BDIR / 'phases.json'
+SLEEPS_FILE = BDIR / 'sleeps.json'
+GRAPHS_DIR = BDIR / 'graphs'
+
+
+def get_logger():
+    return logging.getLogger('jawbone-provider')
+
+
+XID = str # TODO how to shared with backup thing?
+
+Phases = Dict[XID, Any]
+@lru_cache(1)
+def get_phases() -> Phases:
+    return json.loads(PHASES_FILE.read_text())
+
+# TODO use awakenings and quality
+class SleepEntry:
+    def __init__(self, js) -> None:
+        self.js = js
+
+    # TODO @memoize decorator?
+    @property
+    def date_(self) -> date:
+        return self.sleep_end.date()
+
+    def _fromts(self, ts: int) -> datetime:
+        return pytz.utc.localize(datetime.utcfromtimestamp(ts)).astimezone(self._tz).astimezone(self._tz)
+    @property
+    def _tz(self):
+        return pytz.timezone(self._details['tz'])
+
+    @property
+    def title(self) -> str:
+        return self.js['title']
+
+    @property
+    def xid(self) -> XID:
+        return self.js['xid']
+
+    @property
+    def _details(self):
+        return self.js['details']
+
+    # TODO figure out timezones..
+    # not sure how.. I guess by the american ones
+    @property
+    def created(self) -> datetime:
+        return self._fromts(self.js['time_created'])
+
+    @property
+    def completed(self) -> datetime:
+        return self._fromts(self.js['time_completed'])
+
+    @property
+    def asleep(self) -> datetime:
+        return self._fromts(self._details['asleep_time'])
+
+    @property
+    def sleep_start(self) -> datetime:
+        return self.asleep # TODO careful, maybe use same logic as emfit
+
+    @property
+    def bed_time(self) -> int:
+        return int((self.sleep_end - self.sleep_start).total_seconds()) // 60
+
+    @property
+    def sleep_end(self) -> datetime:
+        return self._fromts(self._details['awake_time'])
+
+    @property
+    def graph(self) -> Path:
+        return GRAPHS_DIR / (self.xid + ".png")
+
+    # TODO might be useful to cache these??
+    @property
+    def phases(self) -> List[datetime]:
+        # TODO make sure they are consistent with emfit?
+        return [self._fromts(i['time']) for i in get_phases()[self.xid]]
+
+    def __str__(self) -> str:
+        return f"{self.date_.strftime('%a %d %b')} {self.title}"
+
+    def __repr__(self) -> str:
+        return str(self)
+
+
+def load_sleeps() -> List[SleepEntry]:
+    sleeps = json.loads(SLEEPS_FILE.read_text())
+    return [SleepEntry(js) for js in sleeps]
+
+
+import numpy as np # type: ignore
+import matplotlib.pyplot as plt # type: ignore
+from matplotlib.figure import Figure # type: ignore
+from matplotlib.axes import Axes # type: ignore
+
+# pip install imageio
+# from imageio import imread # type: ignore
+
+
+def hhmm(time: datetime):
+    return time.strftime("%H:%M")
+
+
+# def xpos(time: datetime) -> float:
+#     tick = span / width
+#     fromstart = time - sleep.created
+#     return fromstart / tick
+
+import matplotlib.dates as mdates # type: ignore
+from matplotlib.ticker import MultipleLocator, FixedLocator # type: ignore
+
+def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None, showtext=True):
+    span = sleep.completed - sleep.created
+    print(f"{sleep.xid} span: {span}")
+
+    img = imread(sleep.graph)
+    # all of them are 300x300 images apparently
+    # span for image
+    xspan = [sleep.created, sleep.completed]
+    xspan = [mdates.date2num(i) for i in xspan]
+    if xlims is None:
+        tt = sleep.created
+        hour = tt.hour
+        # TODO maybe assert that hour is somewhere between 20 and 8 or something
+        start: datetime
+        starttime = time(23, 00)
+        if hour >= 20:
+            # went to bed before midnight
+            start = datetime.combine(tt.date(), starttime)
+        elif hour <= 8:
+            # went to bed after midnight
+            start = datetime.combine(tt.date() - timedelta(days=1), starttime)
+        else:
+            print("wtf??? weird time for sleep...")
+            # choosing at random
+            start = datetime.combine(tt.date(), starttime)
+        end = start + timedelta(hours=10)
+        xlims = [start, end]
+
+    # axes.figure(figsize=(10, 5))
+    axes.set_xlim(xlims)
+    hhmm_fmt = mdates.DateFormatter('%H:%M')
+    axes.xaxis.set_major_formatter(hhmm_fmt)
+    ticks = sleep.phases if showtext else []
+    axes.xaxis.set_ticks(ticks)
+    axes.yaxis.set_ticks([])
+    axes.tick_params(
+        axis='both',
+        which='major',
+        length=0,
+        labelsize=7,
+        rotation=30,
+        pad=-14, # err... hacky
+    )
+
+    ylims = [0, 50]
+    axes.set_ylim(ylims)
+
+    axes.imshow(
+        img,
+        zorder=0,
+        extent=[
+            xspan[0], xspan[1],
+            ylims[0], ylims[1],
+        ],
+        aspect='auto',
+    )
+    # axes.set_title(str(sleep))
+    # axes.title.set_size(10)
+
+    if showtext:
+        axes.text(xlims[1] - timedelta(hours=1.5), 20, str(sleep),)
+    # plt.text(sleep.asleep(), 0, hhmm(sleep.asleep()))
+
+from kython import make_dict, group_by_key
+
+def sleeps_by_date() -> Dict[date, SleepEntry]:
+    logger = get_logger()
+
+    sleeps = load_sleeps()
+    sleeps = [s for s in sleeps if s.graph.exists()] # TODO careful..
+    res = {}
+    for dd, group in group_by_key(sleeps, key=lambda s: s.date_).items():
+        if len(group) == 1:
+            res[dd] = group[0]
+        else:
+            # TODO short ones I can ignore I guess. but won't bother now
+            logger.error('multiple sleeps on %s: %s', dd, group)
+    return res
+
+# sleeps_count = 35 # len(sleeps) # apparently MPL fails at 298 with outofmemory or something
+# start = 40
+# 65 is arount 1 july
+# sleeps = sleeps[start: start + sleeps_count]
+# sleeps = sleeps[:sleeps_count]
+# dt = {k: v for k, v in dt.items() if v is not None}
+
+# TODO not really sure it belongs here...
+# import melatonin
+# dt = melatonin.get_data()
+
+def predicate(sleep: SleepEntry):
+    """
+       Filter for comparing similar sleep sesssions
+    """
+    start = sleep.created.time()
+    end = sleep.completed.time()
+    if (time(23, 0) <= start <= time(23, 30)) and (time(5, 30) <= end <= time(6, 30)):
+        return True
+    return False
+
+
+def plot():
+    # TODO ??
+    sleeps = lfilter(predicate, sleeps)
+    sleeps_count = len(sleeps)
+    print(sleeps_count)
+
+    fig: Figure = plt.figure(figsize=(15, sleeps_count * 1))
+
+    axarr = fig.subplots(nrows=len(sleeps))
+    for i, (sleep, axes) in enumerate(zip(sleeps, axarr)):
+        plot_one(sleep, fig, axes, showtext=True)
+        used = dt.get(sleep.date_, None)
+        sused: str
+        color: str
+        # used = True if used is None else False # TODO?
+        if used is True:
+            sused = "YES"
+            color = 'green'
+        elif used is False:
+            sused = "NO"
+            color = 'red'
+        else:
+            sused = "??"
+            color = 'white'
+        axes.text(axes.get_xlim()[0], 20, sused)
+        axes.patch.set_alpha(0.5)
+        axes.set_facecolor(color)
+
+
+    plt.tight_layout()
+    plt.subplots_adjust(hspace=0.0)
+    # er... this saves with a different aspect ratio for some reason.
+    # tap 'ctrl-s' on mpl plot window to save..
+    # plt.savefig('res.png', asp)
+    plt.show()
+
+import pandas as pd # type: ignore
+def get_dataframe():
+    sleeps = sleeps_by_date()
+    items = []
+    for dd, s in sleeps.items():
+        items.append({
+            'date'       : dd, # TODO not sure... # TODO would also be great to sync column names...
+            'sleep_start': s.sleep_start,
+            'sleep_end'  : s.sleep_end,
+            'bed_time'   : s.bed_time,
+        })
+        # TODO tz is in sleeps json
+    res = pd.DataFrame(items)
+    return res
+
+
+def test_tz():
+    sleeps = sleeps_by_date()
+    for s in sleeps.values():
+        assert s.sleep_start.tzinfo is not None
+        assert s.sleep_end.tzinfo is not None
+
+    for dd, exp in [
+            (date(year=2015, month=8 , day=28), time(hour=7, minute=20)),
+            (date(year=2015, month=9 , day=15), time(hour=6, minute=10)),
+    ]:
+        sleep = sleeps[dd]
+        end = sleep.sleep_end
+
+        assert end.time() == exp
+
+        # TODO fuck. on 0909 I woke up at around 6 according to google timeline
+        # but according to jawbone, it was on 0910?? eh. I guess it's jus shitty tracking.
+
+
+def main():
+    setup_logzero(get_logger())
+    test_tz()
+    # print(get_dataframe())
+
+
+if __name__ == '__main__':
+    main()

main.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+# TODO
+from kython import *
+# from kython.plotting import *
+from csv import DictReader
+from itertools import islice
+
+from typing import Dict
+
+# sleep = []
+# with open('2017.csv', 'r') as fo:
+#     reader = DictReader(fo)
+#     for line in islice(reader, 0, 10):
+#         sleep
+#         print(line)
+
+import numpy as np
+import matplotlib.pyplot as plt
+from numpy import genfromtxt
+import matplotlib.pylab as pylab
+
+pylab.rcParams['figure.figsize'] = (32.0, 24.0)
+pylab.rcParams['font.size'] = 10
+
+jawboneDataFeatures = "Jawbone/features.csv" # Data File Path
+featureDesc: Dict[str, str] = {}
+for x in genfromtxt(jawboneDataFeatures, dtype='unicode', delimiter=','):
+    featureDesc[x[0]] = x[1]
+
+def _safe_float(s: str):
+    if len(s) == 0:
+        return None
+    return float(s)
+
+def _safe_int(s: str):
+    if len(s) == 0:
+        return None
+    return int(float(s)) # TODO meh
+
+def _safe_mins(s: float):
+    if s is None:
+        return None
+    return s / 60
+
+class SleepData(NamedTuple):
+    date: str
+    asleep_time: float
+    awake_time: float
+    total: float
+    awake: float # 'awake for' from app, time awake duing sleep (seconds)
+    awakenings: int
+    light: float # 'light sleep' from app (seconds)
+    deep: float  # 'deep sleep' from app (sec)
+    quality: float # ???
+
+    @classmethod
+    def from_jawbone_dict(cls, d: Dict[str, Any]):
+        return cls(
+            date=d['DATE'],
+            asleep_time=_safe_mins(_safe_float(d['s_asleep_time'])),
+            awake_time=_safe_mins(_safe_float(d['s_awake_time'])),
+            total=_safe_mins(_safe_float(d['s_duration'])),
+            light=_safe_mins(_safe_float(d['s_light'])),
+            deep =_safe_mins(_safe_float(d['s_deep'])),
+            awake=_safe_mins(_safe_float(d['s_awake'])),
+            awakenings=_safe_int(d['s_awakenings']),
+            quality=_safe_float(d['s_quality']),
+        )
+
+    def is_bad(self):
+        return self.deep is None and self.light is None
+
+    # @property
+    # def total(self) -> float:
+    #     return self.light + self.deep
+
+
+
+def iter_useful(data_file: str):
+    from csv import DictReader
+    with open(data_file) as fo:
+        reader = DictReader(fo)
+        for d in reader:
+            dt = SleepData.from_jawbone_dict(d)
+            if not dt.is_bad():
+                yield dt
+
+# TODO <<< hmm. these files do contain deep and light sleep??
+# also steps stats??
+p = Path('/L/backups/jawbone/old_csv')
+# TODO with_my?
+files = [
+    p / "2015.csv",
+    p / "2016.csv",
+    p / "2017.csv",
+]
+
+useful = concat(*(list(iter_useful(f)) for f in files))
+
+# for u in useful:
+#     print(f"{u.total} {u.asleep_time} {u.awake_time}")
+#     # pprint(u.total)
+#     pprint(u)
+#     pprint("---")
+
+dates = [parse_date(u.date, yearfirst=True, dayfirst=False) for u in useful]
+# TODO filter outliers?
+
+# TODO don't need this anymore? it's gonna be in dashboards package
+from kython.plotting import plot_timestamped
+for attr, lims, mavg, fig in [
+        ('light', (0, 400), 5, None),
+        ('deep', (0, 600), 5, None),
+        ('total', (200, 600), 5, None),
+        ('awake_time', (0, 1200), None, 1),
+        ('asleep_time', (-100, 1000), None, 1),
+        # ('awakenings', (0, 5)),
+]:
+    dates_wkd = [d for d in dates if d.weekday() < 5]
+    dates_wke = [d for d in dates if d.weekday() >= 5]
+    for dts, dn in [
+            (dates, 'total'),
+            (dates_wkd, 'weekday'),
+            (dates_wke, 'weekend')
+    ]:
+        mavgs = []
+        if mavg is not None:
+            mavgs.append((mavg, 'green'))
+        fig = plot_timestamped(
+            dts,
+            [getattr(u, attr) for u in useful],
+            marker='.',
+            ratio=(16, 4),
+            mavgs=mavgs,
+            ylimits=lims,
+            ytick_size=60,
+            # figure=1,
+           )
+        plt.savefig(f'{attr}_{dn}.png')
+
+# TODO use proper names?
+# plt.savefig('res.png')
+# fig.show()