From 9efae51d3789b7f13f5165fb75fc2001370fcf41 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 29 Jul 2017 13:55:26 +0100 Subject: [PATCH 01/23] initial --- .gitignore | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++ .projectile | 1 + ci.sh | 3 ++ 3 files changed, 108 insertions(+) create mode 100644 .gitignore create mode 100644 .projectile create mode 100755 ci.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7ecbf91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,104 @@ + +# Created by https://www.gitignore.io/api/python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# End of https://www.gitignore.io/api/python diff --git a/.projectile b/.projectile new file mode 100644 index 0000000..89335b7 --- /dev/null +++ b/.projectile @@ -0,0 +1 @@ +- /.mypy_cache/ diff --git a/ci.sh b/ci.sh new file mode 100755 index 0000000..f584ed9 --- /dev/null +++ b/ci.sh @@ -0,0 +1,3 @@ +#!/bin/bash +python3.6 -mpylint -E main.py common.py products.py +python3.6 -mmypy main.py common.py products.py From 7899a67d08349ec4cc867dcd8f4ebc1a83da74ed Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 29 Jul 2017 17:33:14 +0100 Subject: [PATCH 02/23] initial --- Jawbone/features.csv | 81 ++++++++++++++++ TODO.org | 6 ++ main.py | 219 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 306 insertions(+) create mode 100644 Jawbone/features.csv create mode 100644 TODO.org create mode 100755 main.py diff --git a/Jawbone/features.csv b/Jawbone/features.csv new file mode 100644 index 0000000..7adb374 --- /dev/null +++ b/Jawbone/features.csv @@ -0,0 +1,81 @@ +Field,Definition +DATE,Date of column data (in year/month/day format). +age,Your age on the indicated date. +avg_bg,Average passive heart rate (in beats per minute). +bmr,Basal Metabolic Rate (in calories). +body_fat,Body fat percentage. +e_caffeine,Caffeine content consumed (in milligrams). +e_calcium,Calcium content consumed (in milligrams). +e_calories,Calories consumed. +e_carbs,Carbohydrates consumed (in grams). +e_cholesterol,Cholesterol consumed (in milligrams). +e_count,Number of meals logged. +e_fat,Fat consumed (in grams). +e_fiber,Fiber consumed (in grams). +e_iron,Percentage of recommended iron consumed (based on a 2000 calorie per day diet). +e_monounsaturated_fat,Monounsaturated fat consumed (in grams). +e_num_drinks,Number of drinks logged. +e_num_foods,Number of meal items logged. +e_num_mealitems_green,Number of meal items logged with a green UP Food Score. +e_num_mealitems_red,Number of meal items logged with a red UP Food Score. +e_num_mealitems_with_score,Number of meal items logged with an UP Food Score. +e_num_mealitems_yellow,Number of meal items logged with a yellow UP Food Score. +e_num_water,Glasses of water logged. +e_polyunsaturated_fat,Polyunsaturated fat consumed (in grams). +e_potassium,Potassium consumed (in milligrams). +e_protein,Protein consumed (in grams). +e_sat_fat,Saturated fat consumed (in grams). +e_sodium,Sodium consumed (in milligrams). +e_sugar,Sugar consumed (in grams). +e_trans_fat,Trans fat consumed (in grams). +e_unsat_fat,Unsaturated fat consumed (in grams). +e_vitamin_a,Percentage of recommended Vitamin A consumed (based on a 2000 calorie per day diet). +e_vitamin_c,Percentage of recommended Vitamin C consumed (based on a 2000 calorie per day diet). +gender,Your specified gender (0=male 1=female). +goal_body_weight,Weight goal (in kilograms). +goal_body_weight_intent,Weight goal preference (0=lose 1=maintain 2=gain). +height,Your specified height (in meters). +m_active_time,Amount of total active time (in seconds). +m_calories,Total number of calories burned during active time (in seconds). +m_distance,Total distance traveled (in meters). +m_inactive_time,Total inactive time (in seconds). +m_lcat,Longest consecutive active time (in seconds). +m_lcit,Longest consecutive inactive time (in seconds). +m_steps,Total number of steps taken. +m_steps_3am,Total number of steps taken (before 3am). +m_total_calories,Total number of calories burned in the day +m_workout_count,Number of workouts logged. +m_workout_time,Length of logged workouts (in seconds). +max_bg,Highest passive heart rate (in beats per minute). +min_bg,Lowest passive heart rate (in beats per minute). +n_asleep_time,Duration of naps/secondary sleep (in seconds). +n_awake,Total time awake during naps/secondary sleep (in seconds). +n_awake_time,Total time awake after waking from naps/secondary sleep (in seconds). +n_awakenings,Number of times awoken during naps/secondary sleep. +n_bedtime,Length of time band in sleep mode during naps/secondary sleep (in seconds). +n_clinical_deep,Length of Deep sleep during naps/secondary sleep (in seconds UP3 and UP4 only). +n_count,Number of naps/secondary sleep entries logged. +n_deep,Length of Sound sleep during naps/secondary sleep (in seconds). +n_duration,Duration of naps/secondary sleep (in seconds). +n_light,Length of Light sleep during naps/secondary sleep (in seconds). +n_quality,Not applicable. +n_rem,Length of REM sleep during naps/secondary sleep (in seconds). +n_to_bed_phr,Average passive heart rate 1 hour before naps/secondary sleep (in beats per minute). +num_readings,Number of background heart rate readings. +o_count,Number of moods logged. +o_mood,Average score for moods logged (10 = lowest 80 = highest). +rhr,Resting heart rate (in beats per minute). +s_asleep_time,Duration of primary sleep (in seconds). +s_awake,Total time awake during primary sleep (in seconds). +s_awake_time,Total time awake after waking from primary sleep (in seconds). +s_awakenings,Number of times awoken during primary sleep. +s_bedtime,Length of time band was in primary sleep mode (in seconds). +s_clinical_deep,Length of primary Deep sleep (in seconds UP3 and UP4 only). +s_count,Number of primary sleep entries logged. +s_deep,Length of primary Sound sleep (in seconds). +s_duration,Duration of primary sleep (in seconds). +s_light,Length of primary Light sleep (in seconds). +s_quality,Not applicable. +s_rem,Length of primary REM sleep (in seconds). +s_to_bed_phr,Average passive heart rate 1 hour before primary sleep (in beats per minute). +weight,Weight (in kilograms). diff --git a/TODO.org b/TODO.org new file mode 100644 index 0000000..d2c848d --- /dev/null +++ b/TODO.org @@ -0,0 +1,6 @@ +https://github.com/crowoy/Health-Analysis +https://github.com/joytafty-work/SleepModel +https://github.com/carleshf/jawboneup2tex +https://github.com/search?l=Jupyter+Notebook&q=s_awakenings&type=Code&utf8=%E2%9C%93 +https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/colifer.py +https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/reportextenders/jawbone/jawbone_sleep.py diff --git a/main.py b/main.py new file mode 100755 index 0000000..8106bd4 --- /dev/null +++ b/main.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3.6 +# TODO +from csv import DictReader +from itertools import islice + +from typing import Dict + +# sleep = [] +# with open('2017.csv', 'r') as fo: +# reader = DictReader(fo) +# for line in islice(reader, 0, 10): +# sleep +# print(line) + +import numpy as np +import matplotlib.pyplot as plt +from numpy import genfromtxt +import matplotlib.pylab as pylab + +pylab.rcParams['figure.figsize'] = (32.0, 24.0) +pylab.rcParams['font.size'] = 10 + + + +dimensions = 3 # Number of dimensions to reduce to +jawboneDataFile = "/L/Dropbox/backups/jawbone/2017.csv" # Data File Path + +jawboneDataFeatures = "Jawbone/features.csv" # Data File Path +featureDesc: Dict[str, str] = {} +for x in genfromtxt(jawboneDataFeatures, dtype='unicode', delimiter=','): + featureDesc[x[0]] = x[1] + +def filterData_Jawbone (data): + #Removes null data (and corresponding features) + data = data[0:,:] + # for i in range(16): + # data = np.delete(data, 0, 1) + # print(data) + h, w = data.shape + data = np.where((data == ''), 0, data) + allZero = [np.all(np.delete([0 if col[i] == '' else col[i] for col in data], [0]).astype(float) + == 0) for i in range(w)] + allSame = [np.all(np.delete([0 if col[i] == '' else col[i] for col in data], [0]).astype(float) + == np.delete([0 if col[i] == '' else col[i] for col in data], [0]).astype(float)[0]) for i in range(w)] + empty = np.logical_or(allZero, allSame) + n = [i for i in range(np.array(empty).size) if empty[i] == True] + return np.delete(data, n, axis=1) + +dataAll = filterData_Jawbone(genfromtxt(jawboneDataFile, dtype='unicode', delimiter=',')) +features = dataAll[0] +features = [ + 's_light', # 'light sleep' from app + 's_awake', # 'woke up' from app (how many times you were awake) + 's_deep' # 'sound sleep' from app +] +# TODO filter more carefully... + + +def getIndex (data, features): + index = [] + for f in features: + index.append(np.where((data[0] == f) == True)[0][0]) + return index + +def getFeatures (data, features): + h, w = data.shape + index = getIndex(data, features) + extracted = np.zeros(h-1) + for i in index: + temp = np.delete([0 if col[i] == '' else col[i] for col in data], [0]).astype(float) + temp /= np.amax(temp) + extracted = np.vstack((extracted, temp)) + extracted = np.delete(extracted, 0, 0) + return extracted + + +# print(dataAll) +data = getFeatures(dataAll, features) + + +def remNull(x, y): + nx = np.where(x == 0) + ny = np.where(y == 0) + nulli = np.concatenate((nx[0], ny[0])) + x = np.delete(x, nulli, 0) + y = np.delete(y, nulli, 0) + return x, y + +def calculateVar(x, y) -> float: + x, y = remNull(x,y) + if len(x) == 0: + # TODO needs date? + print("Warning") + return 0.0 # TODO ??? + meanX = np.mean(x) + meanY = np.mean(y) + n = float(x.shape[0]) + print(n) + return ((1/n)*(np.sum((x-meanX)*(y-meanY)))) + # return ((1/(n + 1))*(np.sum((x-meanX)*(y-meanY)))) # TODO fixme.. + +def calculateCov(data): + h, w = data.shape + cov = np.zeros([h, h]) + + for i in range(h): + for j in range(h): + cov[i][j] = calculateVar(data[i], data[j]) + return cov + + +# In[119]: +# a = np.array([[1, 2, 3], [1, 2, 3]]) +# print(a) +# print(calculateCov(a)) +# print(np.cov(a)) +# print("VAR") +# print(np.var(a[0])) + +# print("DATA") +# print(data) + +# print("NPCOV") +# print(np.cov(data)) + +# cov = calculateCov (data) +# print("COV") +# print(cov) +cov = np.cov(data) # TODO ??? + + +# In[120]: + + +def plotFeatures (title, label1, label2, feature1, feature2): + plt.scatter(feature1, feature2) + + plt.title(title) + plt.xlabel(label1) + plt.ylabel(label2) + + plt.xlim(0, 1) + plt.ylim(0, 1) + + plt.show() + +def plotMatrix(data): + r, c = data.shape + c=2 + fig = plt.figure() + plotID = 1 + for i in range(c): + for j in range(c): + f1 = getFeature(data, data[0][i]) + f2 = getFeature(data, data[0][j]) + ax = fig.add_subplot( c, c, plotID ) + ax.scatter(f1, f2) + ax.set_title(data[0][i] + ' vs ' + data[0][j]) + ax.axis('off') + plotID += 1 + plt.show() + +def plotMatrix1(features, data): + for f in features: + print(f"{f}: {featureDesc[f]}") + r, c = data.shape + fig = plt.figure() + plotID = 1 + for i in range(r): + for j in range(r): + ax = fig.add_subplot( r, r, plotID ) + x,y = remNull(data[i], data[j]) + ax.scatter(x, y, s=2) + ax.set_title(features[i] + ' vs ' + features[j], fontsize=15) + ax.tick_params(axis='x', which='major', labelsize=8) + ax.tick_params(axis='y', which='major', labelsize=8) +# ax.set_xlim(0,1) +# ax.set_ylim(0,1) + plotID += 1 + plt.show() + + +# In[121]: + + +# plotMatrix1(features, data) + + +# In[ ]: + + +def rankF(features, cov): + n = len(features) + eigenV = np.linalg.eig(cov) + eigVal = np.matrix(eigenV[0]) + eigVec = np.matrix(eigenV[1]) + order = (n-1) - np.argsort(eigVal) + + rankFeatures = np.empty(n, dtype=' Date: Sat, 29 Jul 2017 17:40:59 +0100 Subject: [PATCH 03/23] more todos --- TODO.org | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/TODO.org b/TODO.org index d2c848d..2899dd4 100644 --- a/TODO.org +++ b/TODO.org @@ -4,3 +4,12 @@ https://github.com/carleshf/jawboneup2tex https://github.com/search?l=Jupyter+Notebook&q=s_awakenings&type=Code&utf8=%E2%9C%93 https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/colifer.py https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/reportextenders/jawbone/jawbone_sleep.py +https://github.com/GlenCrawford/ruby_jawbone + +* TODO extract sleep intervals +https://jawbone.com/up/developer/endpoints/sleeps +sleep phases here + + +* https://nyquist212.wordpress.com/2015/06/22/visualizing-jawbone-up-data-with-d3-js/ + From dfabd728052af3f15e287ee41d51c41c8c3b8d15 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 9 Aug 2017 21:21:07 +0100 Subject: [PATCH 04/23] alal --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 8106bd4..28bf2c2 100755 --- a/main.py +++ b/main.py @@ -51,7 +51,7 @@ features = dataAll[0] features = [ 's_light', # 'light sleep' from app 's_awake', # 'woke up' from app (how many times you were awake) - 's_deep' # 'sound sleep' from app + 's_deep' # 'sound sleep' from app ] # TODO filter more carefully... From 9795c058fbddd50c875a5b725650b711185a738f Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 9 Aug 2017 23:12:35 +0100 Subject: [PATCH 05/23] Lots of plots! --- main.py | 287 ++++++++++++++++++++------------------------------------ 1 file changed, 102 insertions(+), 185 deletions(-) diff --git a/main.py b/main.py index 28bf2c2..a700c10 100755 --- a/main.py +++ b/main.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3.6 # TODO +from kython import * +from kython.plotting import * from csv import DictReader from itertools import islice @@ -20,200 +22,115 @@ import matplotlib.pylab as pylab pylab.rcParams['figure.figsize'] = (32.0, 24.0) pylab.rcParams['font.size'] = 10 - - -dimensions = 3 # Number of dimensions to reduce to -jawboneDataFile = "/L/Dropbox/backups/jawbone/2017.csv" # Data File Path - jawboneDataFeatures = "Jawbone/features.csv" # Data File Path featureDesc: Dict[str, str] = {} for x in genfromtxt(jawboneDataFeatures, dtype='unicode', delimiter=','): featureDesc[x[0]] = x[1] -def filterData_Jawbone (data): - #Removes null data (and corresponding features) - data = data[0:,:] - # for i in range(16): - # data = np.delete(data, 0, 1) - # print(data) - h, w = data.shape - data = np.where((data == ''), 0, data) - allZero = [np.all(np.delete([0 if col[i] == '' else col[i] for col in data], [0]).astype(float) - == 0) for i in range(w)] - allSame = [np.all(np.delete([0 if col[i] == '' else col[i] for col in data], [0]).astype(float) - == np.delete([0 if col[i] == '' else col[i] for col in data], [0]).astype(float)[0]) for i in range(w)] - empty = np.logical_or(allZero, allSame) - n = [i for i in range(np.array(empty).size) if empty[i] == True] - return np.delete(data, n, axis=1) +def _safe_float(s: str): + if len(s) == 0: + return None + return float(s) -dataAll = filterData_Jawbone(genfromtxt(jawboneDataFile, dtype='unicode', delimiter=',')) -features = dataAll[0] -features = [ - 's_light', # 'light sleep' from app - 's_awake', # 'woke up' from app (how many times you were awake) - 's_deep' # 'sound sleep' from app +def _safe_int(s: str): + if len(s) == 0: + return None + return int(float(s)) # TODO meh + +def _safe_mins(s: float): + if s is None: + return None + return s / 60 + +class SleepData(NamedTuple): + date: str + asleep_time: float + awake_time: float + total: float + awake: float # 'awake for' from app, time awake duing sleep (seconds) + awakenings: int + light: float # 'light sleep' from app (seconds) + deep: float # 'deep sleep' from app (sec) + quality: float # ??? + + @classmethod + def from_jawbone_dict(cls, d: Dict[str, Any]): + return cls( + date=d['DATE'], + asleep_time=_safe_mins(_safe_float(d['s_asleep_time'])), + awake_time=_safe_mins(_safe_float(d['s_awake_time'])), + total=_safe_mins(_safe_float(d['s_duration'])), + light=_safe_mins(_safe_float(d['s_light'])), + deep =_safe_mins(_safe_float(d['s_deep'])), + awake=_safe_mins(_safe_float(d['s_awake'])), + awakenings=_safe_int(d['s_awakenings']), + quality=_safe_float(d['s_quality']), + ) + + def is_bad(self): + return self.deep is None and self.light is None + + # @property + # def total(self) -> float: + # return self.light + self.deep + + + +def iter_useful(data_file: str): + from csv import DictReader + with open(data_file) as fo: + reader = DictReader(fo) + for d in reader: + dt = SleepData.from_jawbone_dict(d) + if not dt.is_bad(): + yield dt + +files = [ + "/L/Dropbox/backups/jawbone/2015.csv", + "/L/Dropbox/backups/jawbone/2016.csv", + "/L/Dropbox/backups/jawbone/2017.csv", ] -# TODO filter more carefully... +useful = concat(*(list(iter_useful(f)) for f in files)) -def getIndex (data, features): - index = [] - for f in features: - index.append(np.where((data[0] == f) == True)[0][0]) - return index +# for u in useful: +# print(f"{u.total} {u.asleep_time} {u.awake_time}") +# # pprint(u.total) +# pprint(u) +# pprint("---") -def getFeatures (data, features): - h, w = data.shape - index = getIndex(data, features) - extracted = np.zeros(h-1) - for i in index: - temp = np.delete([0 if col[i] == '' else col[i] for col in data], [0]).astype(float) - temp /= np.amax(temp) - extracted = np.vstack((extracted, temp)) - extracted = np.delete(extracted, 0, 0) - return extracted +dates = [parse_date(u.date, yearfirst=True, dayfirst=False) for u in useful] +# TODO filter outliers? +for attr, lims, mavg, fig in [ + # ('light', (0, 400), 5, None), + # ('deep', (0, 600), 5, None), + # ('total', (200, 600), 5, None), + ('awake_time', (0, 1200), None, 1), + ('asleep_time', (-100, 1000), None, 1), + # ('awakenings', (0, 5)), +]: + dates_wkd = [d for d in dates if d.weekday() < 5] + dates_wke = [d for d in dates if d.weekday() >= 5] + for dts, dn in [ + (dates, 'total'), + # (dates_wkd, 'weekday'), + # (dates_wke, 'weekend') + ]: + mavgs = [] + if mavg is not None: + mavgs.append((mavg, 'green')) + fig = plot_timestamped( + dts, + [getattr(u, attr) for u in useful], + marker='.', + ratio=(16, 4), + mavgs=mavgs, + ylimits=lims, + ytick_size=60, + figure=1, + ) + # plt.savefig(f'{attr}_{dn}.png') -# print(dataAll) -data = getFeatures(dataAll, features) - - -def remNull(x, y): - nx = np.where(x == 0) - ny = np.where(y == 0) - nulli = np.concatenate((nx[0], ny[0])) - x = np.delete(x, nulli, 0) - y = np.delete(y, nulli, 0) - return x, y - -def calculateVar(x, y) -> float: - x, y = remNull(x,y) - if len(x) == 0: - # TODO needs date? - print("Warning") - return 0.0 # TODO ??? - meanX = np.mean(x) - meanY = np.mean(y) - n = float(x.shape[0]) - print(n) - return ((1/n)*(np.sum((x-meanX)*(y-meanY)))) - # return ((1/(n + 1))*(np.sum((x-meanX)*(y-meanY)))) # TODO fixme.. - -def calculateCov(data): - h, w = data.shape - cov = np.zeros([h, h]) - - for i in range(h): - for j in range(h): - cov[i][j] = calculateVar(data[i], data[j]) - return cov - - -# In[119]: -# a = np.array([[1, 2, 3], [1, 2, 3]]) -# print(a) -# print(calculateCov(a)) -# print(np.cov(a)) -# print("VAR") -# print(np.var(a[0])) - -# print("DATA") -# print(data) - -# print("NPCOV") -# print(np.cov(data)) - -# cov = calculateCov (data) -# print("COV") -# print(cov) -cov = np.cov(data) # TODO ??? - - -# In[120]: - - -def plotFeatures (title, label1, label2, feature1, feature2): - plt.scatter(feature1, feature2) - - plt.title(title) - plt.xlabel(label1) - plt.ylabel(label2) - - plt.xlim(0, 1) - plt.ylim(0, 1) - - plt.show() - -def plotMatrix(data): - r, c = data.shape - c=2 - fig = plt.figure() - plotID = 1 - for i in range(c): - for j in range(c): - f1 = getFeature(data, data[0][i]) - f2 = getFeature(data, data[0][j]) - ax = fig.add_subplot( c, c, plotID ) - ax.scatter(f1, f2) - ax.set_title(data[0][i] + ' vs ' + data[0][j]) - ax.axis('off') - plotID += 1 - plt.show() - -def plotMatrix1(features, data): - for f in features: - print(f"{f}: {featureDesc[f]}") - r, c = data.shape - fig = plt.figure() - plotID = 1 - for i in range(r): - for j in range(r): - ax = fig.add_subplot( r, r, plotID ) - x,y = remNull(data[i], data[j]) - ax.scatter(x, y, s=2) - ax.set_title(features[i] + ' vs ' + features[j], fontsize=15) - ax.tick_params(axis='x', which='major', labelsize=8) - ax.tick_params(axis='y', which='major', labelsize=8) -# ax.set_xlim(0,1) -# ax.set_ylim(0,1) - plotID += 1 - plt.show() - - -# In[121]: - - -# plotMatrix1(features, data) - - -# In[ ]: - - -def rankF(features, cov): - n = len(features) - eigenV = np.linalg.eig(cov) - eigVal = np.matrix(eigenV[0]) - eigVec = np.matrix(eigenV[1]) - order = (n-1) - np.argsort(eigVal) - - rankFeatures = np.empty(n, dtype=' Date: Mon, 13 Nov 2017 23:06:51 +0000 Subject: [PATCH 06/23] initial --- TODO.org | 10 +++++----- analyse.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) create mode 100755 analyse.py diff --git a/TODO.org b/TODO.org index 2899dd4..238e5bd 100644 --- a/TODO.org +++ b/TODO.org @@ -6,10 +6,10 @@ https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/c https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/reportextenders/jawbone/jawbone_sleep.py https://github.com/GlenCrawford/ruby_jawbone -* TODO extract sleep intervals -https://jawbone.com/up/developer/endpoints/sleeps -sleep phases here - - * https://nyquist212.wordpress.com/2015/06/22/visualizing-jawbone-up-data-with-d3-js/ + +* jawbone sleeps meta: +** time_created: apparently starto of sleep (leftmost point of plot) +** time_updated: rightmost point of plot +** details -> asleep_time, awake_time diff --git a/analyse.py b/analyse.py new file mode 100755 index 0000000..aea34ff --- /dev/null +++ b/analyse.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3.6 +from kython import * + +from backup_config import SLEEPS_FILE + +from datetime import datetime +from datetime import date + +XID = str # TODO how to shared with backup thing? + +class SleepEntry: + def __init__(self, js) -> None: + self.js = js + + # TODO @memoize decorator? + def date_(self) -> date: + dates = str(self.js['date']) + return datetime.strptime(dates, "%Y%m%d").date() + + def title(self) -> str: + return self.js['title'] + + def xid(self) -> XID: + return self.js['xid'] + + def _details(self): + return self.js['details'] + + # TODO take timezones into account? + def created(self) -> datetime: + return datetime.fromtimestamp(self.js['time_created']) + + def completed(self) -> datetime: + return datetime.fromtimestamp(self.js['time_completed']) + + def __str__(self) -> str: + return f"{self.date_()} {self.title()}" + + def __repr__(self) -> str: + return str(self) + +def load_sleeps() -> List[SleepEntry]: + with open(SLEEPS_FILE, 'r') as fo: + sleeps = json_load(fo) + return [SleepEntry(js) for js in sleeps] + +sleeps = load_sleeps() +pprint(sleeps[:2]) From 25414b4284cc8cbd4dc9b73ca3e8a0bdfe4bb73d Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 13 Nov 2017 23:22:31 +0000 Subject: [PATCH 07/23] display single sleep on plot --- analyse.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/analyse.py b/analyse.py index aea34ff..1a09b88 100755 --- a/analyse.py +++ b/analyse.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3.6 from kython import * -from backup_config import SLEEPS_FILE +from backup_config import SLEEPS_FILE, GRAPHS_DIR from datetime import datetime from datetime import date @@ -33,6 +33,9 @@ class SleepEntry: def completed(self) -> datetime: return datetime.fromtimestamp(self.js['time_completed']) + def graph(self) -> str: + return os.path.join(GRAPHS_DIR, self.xid() + ".png") + def __str__(self) -> str: return f"{self.date_()} {self.title()}" @@ -45,4 +48,33 @@ def load_sleeps() -> List[SleepEntry]: return [SleepEntry(js) for js in sleeps] sleeps = load_sleeps() -pprint(sleeps[:2]) +# TODO use map? +sleep = sleeps[0] +# pprint(sleeps[:2]) + + +span = sleep.completed() - sleep.created() +print(f"span: {span}") +import numpy as np # type: ignore +import matplotlib.pyplot as plt # type: ignore +# pip install imageio +from imageio import imread # type: ignore + +img = imread(sleep.graph()) +# all of them are 300x300 images apparently +(a, b, _) = img.shape +# print(img.shape) +# print(img.size) +print(a) +print(b) + +np.random.seed(0) +x = np.random.uniform(0.0, a, 100) +y = np.random.uniform(0.0, b, 100) +plt.scatter(x,y,zorder=1) +plt.imshow(img,zorder=0) +plt.title(str(sleep)) +plt.text(0, 0, str(sleep.created().time())) +plt.text(300, 0, str(sleep.completed().time())) +plt.show() + From ad94741c5b96f1e82836aec00606b1894ae5ce64 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 14 Nov 2017 08:24:11 +0000 Subject: [PATCH 08/23] draw timestamps on axis --- TODO.org | 3 +++ analyse.py | 79 +++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/TODO.org b/TODO.org index 238e5bd..62f6537 100644 --- a/TODO.org +++ b/TODO.org @@ -13,3 +13,6 @@ https://github.com/GlenCrawford/ruby_jawbone ** time_created: apparently starto of sleep (leftmost point of plot) ** time_updated: rightmost point of plot ** details -> asleep_time, awake_time + + +* TODO align multiple on the same plot/picture to see patterns diff --git a/analyse.py b/analyse.py index 1a09b88..22eabf6 100755 --- a/analyse.py +++ b/analyse.py @@ -6,6 +6,8 @@ from backup_config import SLEEPS_FILE, GRAPHS_DIR from datetime import datetime from datetime import date +fromtimestamp = datetime.fromtimestamp + XID = str # TODO how to shared with backup thing? class SleepEntry: @@ -28,10 +30,13 @@ class SleepEntry: # TODO take timezones into account? def created(self) -> datetime: - return datetime.fromtimestamp(self.js['time_created']) + return fromtimestamp(self.js['time_created']) def completed(self) -> datetime: - return datetime.fromtimestamp(self.js['time_completed']) + return fromtimestamp(self.js['time_completed']) + + def asleep(self) -> datetime: + return fromtimestamp(self._details()['asleep_time']) def graph(self) -> str: return os.path.join(GRAPHS_DIR, self.xid() + ".png") @@ -59,22 +64,70 @@ import numpy as np # type: ignore import matplotlib.pyplot as plt # type: ignore # pip install imageio from imageio import imread # type: ignore +from scipy.misc import imresize # type: ignore img = imread(sleep.graph()) # all of them are 300x300 images apparently -(a, b, _) = img.shape +size = img.shape +(height, width, depth) = size +size = (height, width * 5, depth) +(height, width, depth) = size + # print(img.shape) # print(img.size) -print(a) -print(b) -np.random.seed(0) -x = np.random.uniform(0.0, a, 100) -y = np.random.uniform(0.0, b, 100) -plt.scatter(x,y,zorder=1) -plt.imshow(img,zorder=0) -plt.title(str(sleep)) -plt.text(0, 0, str(sleep.created().time())) -plt.text(300, 0, str(sleep.completed().time())) +img = imresize(img, size) + +def hhmm(time: datetime): + return time.strftime("%H:%M") + + +def xpos(time: datetime) -> float: + tick = span / width + fromstart = time - sleep.created() + return fromstart / tick + +print(xpos(sleep.asleep())) + +import matplotlib.dates as mdates # type: ignore +from matplotlib.axes import Axes # type: ignore +from matplotlib.ticker import MultipleLocator, FixedLocator # type: ignore + +fig = plt.figure(figsize=(10, 5)) +axes: Axes = fig.add_subplot(1,1,1) + +xlims = [sleep.created(), sleep.completed()] +xlims = [mdates.date2num(i) for i in xlims] +# axes.figure(figsize=(10, 5)) +axes.set_xlim(xlims) + +ylims = [0, 100] +axes.set_ylim(ylims) +# axes.set_xlim((sleep.created(), sleep.completed())) + +hhmm_fmt = mdates.DateFormatter('%H:%M') +axes.xaxis.set_major_formatter(hhmm_fmt) +ticks = [ + sleep.created(), + sleep.asleep(), + sleep.completed(), +] +axes.xaxis.set_ticks(ticks) +# axes.set_ +# plt.gca().xaxis.set_major_formatter(myFmt) +# plt.gca().xaxis_date() + +# axes.imshow(img, zorder=0) +# plt.figure(figsize=(10, 5)) +plt.imshow(img, zorder=0, extent=[ + xlims[0], xlims[1], + ylims[0], ylims[1], +] +, aspect='auto' +) +# plt.title(str(sleep)) + +# plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) + plt.show() From 37531a725984cbf2d58c6a1dec4da145e25839f6 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 14 Nov 2017 08:36:09 +0000 Subject: [PATCH 09/23] Display all sleep phases --- analyse.py | 166 +++++++++++++++++++++++++++++------------------------ 1 file changed, 91 insertions(+), 75 deletions(-) diff --git a/analyse.py b/analyse.py index 22eabf6..659fb38 100755 --- a/analyse.py +++ b/analyse.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3.6 from kython import * -from backup_config import SLEEPS_FILE, GRAPHS_DIR +from backup_config import SLEEPS_FILE, GRAPHS_DIR, PHASES_FILE from datetime import datetime from datetime import date @@ -9,40 +9,55 @@ from datetime import date fromtimestamp = datetime.fromtimestamp XID = str # TODO how to shared with backup thing? +Phases = Dict[XID, Any] + +phases: Phases +with open(PHASES_FILE, 'r') as fo: + phases = json_load(fo) class SleepEntry: def __init__(self, js) -> None: self.js = js # TODO @memoize decorator? + @property def date_(self) -> date: dates = str(self.js['date']) return datetime.strptime(dates, "%Y%m%d").date() + @property def title(self) -> str: return self.js['title'] + @property def xid(self) -> XID: return self.js['xid'] def _details(self): return self.js['details'] - # TODO take timezones into account? + @property def created(self) -> datetime: return fromtimestamp(self.js['time_created']) + @property def completed(self) -> datetime: return fromtimestamp(self.js['time_completed']) + @property def asleep(self) -> datetime: return fromtimestamp(self._details()['asleep_time']) + @property def graph(self) -> str: - return os.path.join(GRAPHS_DIR, self.xid() + ".png") + return os.path.join(GRAPHS_DIR, self.xid + ".png") + + @property + def phases(self) -> List[datetime]: + return [fromtimestamp(i['time']) for i in phases[self.xid]] def __str__(self) -> str: - return f"{self.date_()} {self.title()}" + return f"{self.date_} {self.title}" def __repr__(self) -> str: return str(self) @@ -51,6 +66,77 @@ def load_sleeps() -> List[SleepEntry]: with open(SLEEPS_FILE, 'r') as fo: sleeps = json_load(fo) return [SleepEntry(js) for js in sleeps] +import numpy as np # type: ignore +import matplotlib.pyplot as plt # type: ignore +# pip install imageio +from imageio import imread # type: ignore +from scipy.misc import imresize # type: ignore + + +def hhmm(time: datetime): + return time.strftime("%H:%M") + + +# def xpos(time: datetime) -> float: +# tick = span / width +# fromstart = time - sleep.created +# return fromstart / tick + +import matplotlib.dates as mdates # type: ignore +from matplotlib.figure import Figure # type: ignore +from matplotlib.axes import Axes # type: ignore +from matplotlib.ticker import MultipleLocator, FixedLocator # type: ignore + +def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes): + span = sleep.completed - sleep.created + print(f"span: {span}") + + img = imread(sleep.graph) + # all of them are 300x300 images apparently + # size = img.shape + # (height, width, depth) = size + # size = (height, width * 5, depth) + # (height, width, depth) = size + # img = imresize(img, size) + + xlims = [sleep.created, sleep.completed] + xlims = [mdates.date2num(i) for i in xlims] + # axes.figure(figsize=(10, 5)) + axes.set_xlim(xlims) + + ylims = [0, 50] + axes.set_ylim(ylims) + # axes.set_xlim((sleep.created(), sleep.completed())) + + hhmm_fmt = mdates.DateFormatter('%H:%M') + axes.xaxis.set_major_formatter(hhmm_fmt) + ticks = [ + # sleep.created, + # sleep.asleep, + # sleep.completed, + ] + sleep.phases + axes.xaxis.set_ticks(ticks) + # axes.set_ + # plt.gca().xaxis.set_major_formatter(myFmt) + # plt.gca().xaxis_date() + + # axes.imshow(img, zorder=0) + # plt.figure(figsize=(10, 5)) + axes.imshow( + img, + zorder=0, + extent=[ + xlims[0], xlims[1], + ylims[0], ylims[1], + ], + aspect='auto', + ) + axes.set_title(str(sleep)) + + # plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) + +fig: Figure = plt.figure(figsize=(15, 5)) +axes: Axes = fig.add_subplot(1,1,1) sleeps = load_sleeps() # TODO use map? @@ -58,76 +144,6 @@ sleep = sleeps[0] # pprint(sleeps[:2]) -span = sleep.completed() - sleep.created() -print(f"span: {span}") -import numpy as np # type: ignore -import matplotlib.pyplot as plt # type: ignore -# pip install imageio -from imageio import imread # type: ignore -from scipy.misc import imresize # type: ignore - -img = imread(sleep.graph()) -# all of them are 300x300 images apparently -size = img.shape -(height, width, depth) = size -size = (height, width * 5, depth) -(height, width, depth) = size - -# print(img.shape) -# print(img.size) - -img = imresize(img, size) - -def hhmm(time: datetime): - return time.strftime("%H:%M") - - -def xpos(time: datetime) -> float: - tick = span / width - fromstart = time - sleep.created() - return fromstart / tick - -print(xpos(sleep.asleep())) - -import matplotlib.dates as mdates # type: ignore -from matplotlib.axes import Axes # type: ignore -from matplotlib.ticker import MultipleLocator, FixedLocator # type: ignore - -fig = plt.figure(figsize=(10, 5)) -axes: Axes = fig.add_subplot(1,1,1) - -xlims = [sleep.created(), sleep.completed()] -xlims = [mdates.date2num(i) for i in xlims] -# axes.figure(figsize=(10, 5)) -axes.set_xlim(xlims) - -ylims = [0, 100] -axes.set_ylim(ylims) -# axes.set_xlim((sleep.created(), sleep.completed())) - -hhmm_fmt = mdates.DateFormatter('%H:%M') -axes.xaxis.set_major_formatter(hhmm_fmt) -ticks = [ - sleep.created(), - sleep.asleep(), - sleep.completed(), -] -axes.xaxis.set_ticks(ticks) -# axes.set_ -# plt.gca().xaxis.set_major_formatter(myFmt) -# plt.gca().xaxis_date() - -# axes.imshow(img, zorder=0) -# plt.figure(figsize=(10, 5)) -plt.imshow(img, zorder=0, extent=[ - xlims[0], xlims[1], - ylims[0], ylims[1], -] -, aspect='auto' -) -# plt.title(str(sleep)) - -# plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) - +plot_one(sleep, fig, axes) plt.show() From c31660ff444ab9b39599b209ee0f08d034f38729 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 14 Nov 2017 08:41:05 +0000 Subject: [PATCH 10/23] Subplots! --- analyse.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/analyse.py b/analyse.py index 659fb38..cb480d5 100755 --- a/analyse.py +++ b/analyse.py @@ -136,14 +136,17 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes): # plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) fig: Figure = plt.figure(figsize=(15, 5)) -axes: Axes = fig.add_subplot(1,1,1) sleeps = load_sleeps() +sleeps = sleeps[:5] + +for i, sleep in enumerate(sleeps): + axes: Axes = fig.add_subplot(len(sleeps), 1, i + 1) + plot_one(sleep, fig, axes) + # TODO use map? -sleep = sleeps[0] # pprint(sleeps[:2]) - -plot_one(sleep, fig, axes) +plt.tight_layout() plt.show() From e689e63b7b5fb5f094e40ca63e366fede90e14d8 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 14 Nov 2017 21:51:28 +0000 Subject: [PATCH 11/23] all on one plot --- analyse.py | 73 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/analyse.py b/analyse.py index cb480d5..234defa 100755 --- a/analyse.py +++ b/analyse.py @@ -3,8 +3,7 @@ from kython import * from backup_config import SLEEPS_FILE, GRAPHS_DIR, PHASES_FILE -from datetime import datetime -from datetime import date +from datetime import datetime, date, time fromtimestamp = datetime.fromtimestamp @@ -57,7 +56,7 @@ class SleepEntry: return [fromtimestamp(i['time']) for i in phases[self.xid]] def __str__(self) -> str: - return f"{self.date_} {self.title}" + return f"{self.date_.strftime('%a %d %b')} {self.title}" def __repr__(self) -> str: return str(self) @@ -87,7 +86,7 @@ from matplotlib.figure import Figure # type: ignore from matplotlib.axes import Axes # type: ignore from matplotlib.ticker import MultipleLocator, FixedLocator # type: ignore -def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes): +def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None): span = sleep.completed - sleep.created print(f"span: {span}") @@ -99,15 +98,30 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes): # (height, width, depth) = size # img = imresize(img, size) - xlims = [sleep.created, sleep.completed] - xlims = [mdates.date2num(i) for i in xlims] + # span for image + xspan = [sleep.created, sleep.completed] + xspan = [mdates.date2num(i) for i in xspan] + if xlims is None: + # TODO soo, + tt = sleep.created + hour = tt.hour + # TODO maybe assert that hour is somewhere between 20 and 8 or something + start: datetime + starttime = time(23, 00) + if hour >= 20: + # went to bed before midnight + start = datetime.combine(tt.date(), starttime) + elif hour <= 8: + # went to bed after midnight + start = datetime.combine(tt.date() - timedelta(days=1), starttime) + pass + else: + raise RuntimeError("wtf???") + end = start + timedelta(hours=10) + xlims = [start, end] + # axes.figure(figsize=(10, 5)) axes.set_xlim(xlims) - - ylims = [0, 50] - axes.set_ylim(ylims) - # axes.set_xlim((sleep.created(), sleep.completed())) - hhmm_fmt = mdates.DateFormatter('%H:%M') axes.xaxis.set_major_formatter(hhmm_fmt) ticks = [ @@ -116,37 +130,50 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes): # sleep.completed, ] + sleep.phases axes.xaxis.set_ticks(ticks) - # axes.set_ - # plt.gca().xaxis.set_major_formatter(myFmt) - # plt.gca().xaxis_date() + axes.yaxis.set_ticks([]) + axes.tick_params( + axis='both', + which='major', + length=0, + labelsize=7, + rotation=30, + pad=-14, # err... hacky + ) + + ylims = [0, 50] + axes.set_ylim(ylims) - # axes.imshow(img, zorder=0) - # plt.figure(figsize=(10, 5)) axes.imshow( img, zorder=0, extent=[ - xlims[0], xlims[1], + xspan[0], xspan[1], ylims[0], ylims[1], ], aspect='auto', ) - axes.set_title(str(sleep)) - + # axes.set_title(str(sleep)) + # axes.title.set_size(10) + + axes.text(xlims[1] - timedelta(hours=1.5), 20, str(sleep),) # plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) -fig: Figure = plt.figure(figsize=(15, 5)) +sleeps_count = 30 +fig: Figure = plt.figure(figsize=(15, sleeps_count * 1)) sleeps = load_sleeps() -sleeps = sleeps[:5] +sleeps = sleeps[:sleeps_count] -for i, sleep in enumerate(sleeps): - axes: Axes = fig.add_subplot(len(sleeps), 1, i + 1) +axarr = fig.subplots(nrows=len(sleeps)) +for i, (sleep, axes) in enumerate(zip(sleeps, axarr)): + # axes: Axes = fig.add_subplot(len(sleeps), 1, i + 1) + # ok, have to adjust days a bit... plot_one(sleep, fig, axes) # TODO use map? # pprint(sleeps[:2]) plt.tight_layout() +plt.subplots_adjust(hspace=0.0) plt.show() From 17b3b927f6c25899fc935c8bffc53fa8294d5ae7 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 14 Nov 2017 23:01:00 +0000 Subject: [PATCH 12/23] Add for plotting textless composite plot --- analyse.py | 58 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/analyse.py b/analyse.py index 234defa..ad10f9d 100755 --- a/analyse.py +++ b/analyse.py @@ -4,9 +4,10 @@ from kython import * from backup_config import SLEEPS_FILE, GRAPHS_DIR, PHASES_FILE from datetime import datetime, date, time - fromtimestamp = datetime.fromtimestamp +import os.path + XID = str # TODO how to shared with backup thing? Phases = Dict[XID, Any] @@ -86,23 +87,16 @@ from matplotlib.figure import Figure # type: ignore from matplotlib.axes import Axes # type: ignore from matplotlib.ticker import MultipleLocator, FixedLocator # type: ignore -def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None): +def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None, showtext=True): span = sleep.completed - sleep.created - print(f"span: {span}") + print(f"{sleep.xid} span: {span}") img = imread(sleep.graph) # all of them are 300x300 images apparently - # size = img.shape - # (height, width, depth) = size - # size = (height, width * 5, depth) - # (height, width, depth) = size - # img = imresize(img, size) - # span for image xspan = [sleep.created, sleep.completed] xspan = [mdates.date2num(i) for i in xspan] if xlims is None: - # TODO soo, tt = sleep.created hour = tt.hour # TODO maybe assert that hour is somewhere between 20 and 8 or something @@ -114,9 +108,10 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None): elif hour <= 8: # went to bed after midnight start = datetime.combine(tt.date() - timedelta(days=1), starttime) - pass else: - raise RuntimeError("wtf???") + print("wtf??? weird time for sleep...") + # choosing at random + start = datetime.combine(tt.date(), starttime) end = start + timedelta(hours=10) xlims = [start, end] @@ -124,11 +119,7 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None): axes.set_xlim(xlims) hhmm_fmt = mdates.DateFormatter('%H:%M') axes.xaxis.set_major_formatter(hhmm_fmt) - ticks = [ - # sleep.created, - # sleep.asleep, - # sleep.completed, - ] + sleep.phases + ticks = sleep.phases if showtext else [] axes.xaxis.set_ticks(ticks) axes.yaxis.set_ticks([]) axes.tick_params( @@ -155,25 +146,40 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None): # axes.set_title(str(sleep)) # axes.title.set_size(10) - axes.text(xlims[1] - timedelta(hours=1.5), 20, str(sleep),) + if showtext: + axes.text(xlims[1] - timedelta(hours=1.5), 20, str(sleep),) # plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) -sleeps_count = 30 +sleeps = load_sleeps() + +sleeps = [s for s in sleeps if os.path.lexists(s.graph)] +sleeps_count = 290 # len(sleeps) # apparently MPL fails at 298 with outofmemory or something +sleeps = sleeps[:sleeps_count] + + fig: Figure = plt.figure(figsize=(15, sleeps_count * 1)) -sleeps = load_sleeps() -sleeps = sleeps[:sleeps_count] +def predicate(sleep: SleepEntry): + """ + Filter for comparing similar sleep sesssions + """ + start = sleep.created.time() + end = sleep.completed.time() + if (time(23, 0) <= start <= time(23, 30)) and (time(5, 30) <= end <= time(6, 30)): + return True + return False + +# sleeps = lfilter(predicate, sleeps) axarr = fig.subplots(nrows=len(sleeps)) for i, (sleep, axes) in enumerate(zip(sleeps, axarr)): - # axes: Axes = fig.add_subplot(len(sleeps), 1, i + 1) - # ok, have to adjust days a bit... - plot_one(sleep, fig, axes) + plot_one(sleep, fig, axes, showtext=False) -# TODO use map? -# pprint(sleeps[:2]) plt.tight_layout() plt.subplots_adjust(hspace=0.0) +# er... this saves with a different aspect ratio for some reason. +# tap 'ctrl-s' on mpl plot window to save.. +# plt.savefig('res.png', asp) plt.show() From 5a9d34b42900bd8cec13433f4a61991ef527785f Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 16 Nov 2017 23:13:54 +0000 Subject: [PATCH 13/23] Some melatonin stats... --- analyse.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/analyse.py b/analyse.py index ad10f9d..8300a6b 100755 --- a/analyse.py +++ b/analyse.py @@ -20,6 +20,8 @@ class SleepEntry: self.js = js # TODO @memoize decorator? + # date is going to be the date of the day you mostly slept on + # e.g. if you went to bed at 01:30 24 august and work up at 8AM, date is gonna be 24 august @property def date_(self) -> date: dates = str(self.js['date']) @@ -153,8 +155,19 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None, showtext=Tr sleeps = load_sleeps() sleeps = [s for s in sleeps if os.path.lexists(s.graph)] -sleeps_count = 290 # len(sleeps) # apparently MPL fails at 298 with outofmemory or something -sleeps = sleeps[:sleeps_count] +sleeps_by_date = {s.date_: s for s in sleeps} + +# sleeps_count = 35 # len(sleeps) # apparently MPL fails at 298 with outofmemory or something +# start = 40 +# 65 is arount 1 july +# sleeps = sleeps[start: start + sleeps_count] +# sleeps = sleeps[:sleeps_count] +import melatonin +dt = melatonin.get_data() +# dt = {k: v for k, v in dt.items() if v is not None} + +sleeps = [sleeps_by_date[d] for d in dt if d in sleeps_by_date] +sleeps_count = len(sleeps) fig: Figure = plt.figure(figsize=(15, sleeps_count * 1)) @@ -173,7 +186,23 @@ def predicate(sleep: SleepEntry): axarr = fig.subplots(nrows=len(sleeps)) for i, (sleep, axes) in enumerate(zip(sleeps, axarr)): - plot_one(sleep, fig, axes, showtext=False) + plot_one(sleep, fig, axes, showtext=True) + used = dt[sleep.date_] + sused: str + color: str + used = True if used is None else False # TODO? + if used is True: + sused = "YES" + color = 'green' + elif used is False: + sused = "NO" + color = 'red' + else: + sused = "??" + color = 'blue' + axes.text(axes.get_xlim()[0], 20, sused) + axes.patch.set_alpha(0.5) + axes.set_facecolor(color) plt.tight_layout() From 3d28c3ca6e0115070d0a33d7bfd1e9f40363faf0 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 11 Apr 2019 11:35:07 +0100 Subject: [PATCH 14/23] some wip --- TODO.org | 10 +++------- analyse.py | 23 ++++++++++++----------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/TODO.org b/TODO.org index 62f6537..ccea496 100644 --- a/TODO.org +++ b/TODO.org @@ -1,6 +1,5 @@ https://github.com/crowoy/Health-Analysis https://github.com/joytafty-work/SleepModel -https://github.com/carleshf/jawboneup2tex https://github.com/search?l=Jupyter+Notebook&q=s_awakenings&type=Code&utf8=%E2%9C%93 https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/colifer.py https://github.com/oshev/colifer/blob/592cc6b4d1ac9005c52fccdfb4e207513812baaa/reportextenders/jawbone/jawbone_sleep.py @@ -9,10 +8,7 @@ https://github.com/GlenCrawford/ruby_jawbone * https://nyquist212.wordpress.com/2015/06/22/visualizing-jawbone-up-data-with-d3-js/ -* jawbone sleeps meta: -** time_created: apparently starto of sleep (leftmost point of plot) -** time_updated: rightmost point of plot -** details -> asleep_time, awake_time +* TODO ok, so shoud really do a week of consistent bedtime/waking up to make some final decision on jawbone? - -* TODO align multiple on the same plot/picture to see patterns +* TODO figure out timezones +* TODO post on reddit? release and ask people to run against their data? diff --git a/analyse.py b/analyse.py index 8300a6b..6be3997 100755 --- a/analyse.py +++ b/analyse.py @@ -70,6 +70,8 @@ def load_sleeps() -> List[SleepEntry]: return [SleepEntry(js) for js in sleeps] import numpy as np # type: ignore import matplotlib.pyplot as plt # type: ignore +from matplotlib.figure import Figure # type: ignore +from matplotlib.axes import Axes # type: ignore # pip install imageio from imageio import imread # type: ignore from scipy.misc import imresize # type: ignore @@ -85,8 +87,6 @@ def hhmm(time: datetime): # return fromstart / tick import matplotlib.dates as mdates # type: ignore -from matplotlib.figure import Figure # type: ignore -from matplotlib.axes import Axes # type: ignore from matplotlib.ticker import MultipleLocator, FixedLocator # type: ignore def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None, showtext=True): @@ -166,11 +166,7 @@ import melatonin dt = melatonin.get_data() # dt = {k: v for k, v in dt.items() if v is not None} -sleeps = [sleeps_by_date[d] for d in dt if d in sleeps_by_date] -sleeps_count = len(sleeps) - - -fig: Figure = plt.figure(figsize=(15, sleeps_count * 1)) +sleeps = list(sleeps_by_date.values()) # [sleeps_by_date[d] for d in dt if d in sleeps_by_date] def predicate(sleep: SleepEntry): """ @@ -182,15 +178,20 @@ def predicate(sleep: SleepEntry): return True return False -# sleeps = lfilter(predicate, sleeps) +sleeps = lfilter(predicate, sleeps) +sleeps_count = len(sleeps) +print(sleeps_count) + + +fig: Figure = plt.figure(figsize=(15, sleeps_count * 1)) axarr = fig.subplots(nrows=len(sleeps)) for i, (sleep, axes) in enumerate(zip(sleeps, axarr)): plot_one(sleep, fig, axes, showtext=True) - used = dt[sleep.date_] + used = dt.get(sleep.date_, None) sused: str color: str - used = True if used is None else False # TODO? + # used = True if used is None else False # TODO? if used is True: sused = "YES" color = 'green' @@ -199,7 +200,7 @@ for i, (sleep, axes) in enumerate(zip(sleeps, axarr)): color = 'red' else: sused = "??" - color = 'blue' + color = 'white' axes.text(axes.get_xlim()[0], 20, sused) axes.patch.set_alpha(0.5) axes.set_facecolor(color) From 5831dc26db7f432e15772678dde1604e37e88de5 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 11 Apr 2019 11:38:58 +0100 Subject: [PATCH 15/23] move to __init__ --- analyse.py => jawbone_provider/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename analyse.py => jawbone_provider/__init__.py (100%) diff --git a/analyse.py b/jawbone_provider/__init__.py similarity index 100% rename from analyse.py rename to jawbone_provider/__init__.py From b471a337aac220811b19dab99d148e49c117c5dc Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 11 Apr 2019 11:46:51 +0100 Subject: [PATCH 16/23] fix it to retrieve sleeps again --- jawbone_provider/__init__.py | 120 +++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 54 deletions(-) diff --git a/jawbone_provider/__init__.py b/jawbone_provider/__init__.py index 6be3997..67cff7b 100755 --- a/jawbone_provider/__init__.py +++ b/jawbone_provider/__init__.py @@ -1,20 +1,27 @@ -#!/usr/bin/env python3.6 -from kython import * +#!/usr/bin/env python3 +from typing import Dict, Any, List +import json +from datetime import datetime, date, time +from pathlib import Path -from backup_config import SLEEPS_FILE, GRAPHS_DIR, PHASES_FILE +BDIR = Path('/L/backups/jawbone') +PHASES_FILE = BDIR / 'phases.json' +SLEEPS_FILE = BDIR / 'sleeps.json' +GRAPHS_DIR = BDIR / 'graphs' -from datetime import datetime, date, time -fromtimestamp = datetime.fromtimestamp -import os.path +fromtimestamp = datetime.fromtimestamp # TODO careful XID = str # TODO how to shared with backup thing? + + Phases = Dict[XID, Any] -phases: Phases -with open(PHASES_FILE, 'r') as fo: - phases = json_load(fo) +phases: Phases = json.loads(PHASES_FILE.read_text()) + +# TODO namedtuple, cproperty? +# TODO cache? class SleepEntry: def __init__(self, js) -> None: self.js = js @@ -51,8 +58,8 @@ class SleepEntry: return fromtimestamp(self._details()['asleep_time']) @property - def graph(self) -> str: - return os.path.join(GRAPHS_DIR, self.xid + ".png") + def graph(self) -> Path: + return GRAPHS_DIR / (self.xid + ".png") @property def phases(self) -> List[datetime]: @@ -65,15 +72,17 @@ class SleepEntry: return str(self) def load_sleeps() -> List[SleepEntry]: - with open(SLEEPS_FILE, 'r') as fo: - sleeps = json_load(fo) - return [SleepEntry(js) for js in sleeps] + sleeps = json.loads(SLEEPS_FILE.read_text()) + return [SleepEntry(js) for js in sleeps] + + import numpy as np # type: ignore import matplotlib.pyplot as plt # type: ignore from matplotlib.figure import Figure # type: ignore from matplotlib.axes import Axes # type: ignore + # pip install imageio -from imageio import imread # type: ignore +# from imageio import imread # type: ignore from scipy.misc import imresize # type: ignore @@ -152,21 +161,22 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None, showtext=Tr axes.text(xlims[1] - timedelta(hours=1.5), 20, str(sleep),) # plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) -sleeps = load_sleeps() - -sleeps = [s for s in sleeps if os.path.lexists(s.graph)] -sleeps_by_date = {s.date_: s for s in sleeps} +from kython import make_dict +def sleeps_by_date() -> Dict[date, SleepEntry]: + sleeps = load_sleeps() + sleeps = [s for s in sleeps if s.graph.exists()] # TODO careful.. + return make_dict(sleeps, key=SleepEntry.date_) # sleeps_count = 35 # len(sleeps) # apparently MPL fails at 298 with outofmemory or something # start = 40 # 65 is arount 1 july # sleeps = sleeps[start: start + sleeps_count] # sleeps = sleeps[:sleeps_count] -import melatonin -dt = melatonin.get_data() # dt = {k: v for k, v in dt.items() if v is not None} -sleeps = list(sleeps_by_date.values()) # [sleeps_by_date[d] for d in dt if d in sleeps_by_date] +# TODO ?? +# import melatonin +# dt = melatonin.get_data() def predicate(sleep: SleepEntry): """ @@ -178,38 +188,40 @@ def predicate(sleep: SleepEntry): return True return False -sleeps = lfilter(predicate, sleeps) -sleeps_count = len(sleeps) -print(sleeps_count) + +def plot(): + # TODO ?? + sleeps = lfilter(predicate, sleeps) + sleeps_count = len(sleeps) + print(sleeps_count) + + fig: Figure = plt.figure(figsize=(15, sleeps_count * 1)) + + axarr = fig.subplots(nrows=len(sleeps)) + for i, (sleep, axes) in enumerate(zip(sleeps, axarr)): + plot_one(sleep, fig, axes, showtext=True) + used = dt.get(sleep.date_, None) + sused: str + color: str + # used = True if used is None else False # TODO? + if used is True: + sused = "YES" + color = 'green' + elif used is False: + sused = "NO" + color = 'red' + else: + sused = "??" + color = 'white' + axes.text(axes.get_xlim()[0], 20, sused) + axes.patch.set_alpha(0.5) + axes.set_facecolor(color) -fig: Figure = plt.figure(figsize=(15, sleeps_count * 1)) - -axarr = fig.subplots(nrows=len(sleeps)) -for i, (sleep, axes) in enumerate(zip(sleeps, axarr)): - plot_one(sleep, fig, axes, showtext=True) - used = dt.get(sleep.date_, None) - sused: str - color: str - # used = True if used is None else False # TODO? - if used is True: - sused = "YES" - color = 'green' - elif used is False: - sused = "NO" - color = 'red' - else: - sused = "??" - color = 'white' - axes.text(axes.get_xlim()[0], 20, sused) - axes.patch.set_alpha(0.5) - axes.set_facecolor(color) - - -plt.tight_layout() -plt.subplots_adjust(hspace=0.0) -# er... this saves with a different aspect ratio for some reason. -# tap 'ctrl-s' on mpl plot window to save.. -# plt.savefig('res.png', asp) -plt.show() + plt.tight_layout() + plt.subplots_adjust(hspace=0.0) + # er... this saves with a different aspect ratio for some reason. + # tap 'ctrl-s' on mpl plot window to save.. + # plt.savefig('res.png', asp) + plt.show() From 9c94431791c5232a05577f4813b06d0f76d94ab1 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 11 Apr 2019 13:31:43 +0100 Subject: [PATCH 17/23] proper pandas frame --- jawbone_provider/__init__.py | 65 +++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/jawbone_provider/__init__.py b/jawbone_provider/__init__.py index 67cff7b..e5e9a1a 100755 --- a/jawbone_provider/__init__.py +++ b/jawbone_provider/__init__.py @@ -3,6 +3,9 @@ from typing import Dict, Any, List import json from datetime import datetime, date, time from pathlib import Path +import logging + +from kython.klogging import setup_logzero BDIR = Path('/L/backups/jawbone') PHASES_FILE = BDIR / 'phases.json' @@ -10,14 +13,16 @@ SLEEPS_FILE = BDIR / 'sleeps.json' GRAPHS_DIR = BDIR / 'graphs' +def get_logger(): + return logging.getLogger('jawbone-provider') + + fromtimestamp = datetime.fromtimestamp # TODO careful XID = str # TODO how to shared with backup thing? Phases = Dict[XID, Any] - - phases: Phases = json.loads(PHASES_FILE.read_text()) # TODO namedtuple, cproperty? @@ -42,9 +47,12 @@ class SleepEntry: def xid(self) -> XID: return self.js['xid'] + @property def _details(self): return self.js['details'] + # TODO figure out timezones.. + # not sure how.. I guess by the american ones @property def created(self) -> datetime: return fromtimestamp(self.js['time_created']) @@ -55,14 +63,28 @@ class SleepEntry: @property def asleep(self) -> datetime: - return fromtimestamp(self._details()['asleep_time']) + return fromtimestamp(self._details['asleep_time']) + + @property + def sleep_start(self) -> datetime: + return self.asleep # TODO careful, maybe use same logic as emfit + + @property + def bed_time(self) -> int: + return int((self.sleep_end - self.sleep_start).total_seconds()) // 60 + + @property + def sleep_end(self) -> datetime: + return fromtimestamp(self._details['awake_time']) @property def graph(self) -> Path: return GRAPHS_DIR / (self.xid + ".png") + # TODO might be useful to cache these?? @property def phases(self) -> List[datetime]: + # TODO make sure they are consistent with emfit? return [fromtimestamp(i['time']) for i in phases[self.xid]] def __str__(self) -> str: @@ -161,11 +183,21 @@ def plot_one(sleep: SleepEntry, fig: Figure, axes: Axes, xlims=None, showtext=Tr axes.text(xlims[1] - timedelta(hours=1.5), 20, str(sleep),) # plt.text(sleep.asleep(), 0, hhmm(sleep.asleep())) -from kython import make_dict +from kython import make_dict, group_by_key + def sleeps_by_date() -> Dict[date, SleepEntry]: + logger = get_logger() + sleeps = load_sleeps() sleeps = [s for s in sleeps if s.graph.exists()] # TODO careful.. - return make_dict(sleeps, key=SleepEntry.date_) + res = {} + for dd, group in group_by_key(sleeps, key=lambda s: s.date_).items(): + if len(group) == 1: + res[dd] = group[0] + else: + # TODO short ones I can ignore I guess. but won't bother now + logger.error('multiple sleeps on %s: %s', dd, group) + return res # sleeps_count = 35 # len(sleeps) # apparently MPL fails at 298 with outofmemory or something # start = 40 @@ -225,3 +257,26 @@ def plot(): # plt.savefig('res.png', asp) plt.show() +import pandas as pd # type: ignore +def get_dataframe(): + sleeps = sleeps_by_date() + items = [] + for dd, s in sleeps.items(): + items.append({ + 'date' : dd, # TODO not sure... # TODO would also be great to sync column names... + 'sleep_start': s.sleep_start, + 'sleep_end' : s.sleep_end, + 'bed_time' : s.bed_time, + }) + # TODO tz is in sleeps json + res = pd.DataFrame(items) + return res + + +def main(): + setup_logzero(get_logger()) + print(get_dataframe()) + + +if __name__ == '__main__': + main() From 85bec7373d728e006409c433bc95f343e5acf69b Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 11 Apr 2019 14:37:08 +0100 Subject: [PATCH 18/23] fix timezones --- jawbone_provider/__init__.py | 47 +++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/jawbone_provider/__init__.py b/jawbone_provider/__init__.py index e5e9a1a..2826eff 100755 --- a/jawbone_provider/__init__.py +++ b/jawbone_provider/__init__.py @@ -4,6 +4,7 @@ import json from datetime import datetime, date, time from pathlib import Path import logging +import pytz from kython.klogging import setup_logzero @@ -17,8 +18,6 @@ def get_logger(): return logging.getLogger('jawbone-provider') -fromtimestamp = datetime.fromtimestamp # TODO careful - XID = str # TODO how to shared with backup thing? @@ -32,12 +31,16 @@ class SleepEntry: self.js = js # TODO @memoize decorator? - # date is going to be the date of the day you mostly slept on - # e.g. if you went to bed at 01:30 24 august and work up at 8AM, date is gonna be 24 august @property def date_(self) -> date: - dates = str(self.js['date']) - return datetime.strptime(dates, "%Y%m%d").date() + return self.sleep_end.date() + + def _fromts(self, ts: int) -> datetime: + return pytz.utc.localize(datetime.utcfromtimestamp(ts)).astimezone(self._tz).astimezone(self._tz) + + @property + def _tz(self): + return pytz.timezone(self._details['tz']) @property def title(self) -> str: @@ -55,15 +58,15 @@ class SleepEntry: # not sure how.. I guess by the american ones @property def created(self) -> datetime: - return fromtimestamp(self.js['time_created']) + return self._fromts(self.js['time_created']) @property def completed(self) -> datetime: - return fromtimestamp(self.js['time_completed']) + return self._fromts(self.js['time_completed']) @property def asleep(self) -> datetime: - return fromtimestamp(self._details['asleep_time']) + return self._fromts(self._details['asleep_time']) @property def sleep_start(self) -> datetime: @@ -75,7 +78,7 @@ class SleepEntry: @property def sleep_end(self) -> datetime: - return fromtimestamp(self._details['awake_time']) + return self._fromts(self._details['awake_time']) @property def graph(self) -> Path: @@ -85,7 +88,7 @@ class SleepEntry: @property def phases(self) -> List[datetime]: # TODO make sure they are consistent with emfit? - return [fromtimestamp(i['time']) for i in phases[self.xid]] + return [self._fromts(i['time']) for i in phases[self.xid]] def __str__(self) -> str: return f"{self.date_.strftime('%a %d %b')} {self.title}" @@ -273,9 +276,29 @@ def get_dataframe(): return res +def test_tz(): + sleeps = sleeps_by_date() + for s in sleeps.values(): + assert s.sleep_start.tzinfo is not None + assert s.sleep_end.tzinfo is not None + + for dd, exp in [ + (date(year=2015, month=8 , day=28), time(hour=7, minute=20)), + (date(year=2015, month=9 , day=15), time(hour=6, minute=10)), + ]: + sleep = sleeps[dd] + end = sleep.sleep_end + + assert end.time() == exp + + # TODO fuck. on 0909 I woke up at around 6 according to google timeline + # but according to jawbone, it was on 0910?? eh. I guess it's jus shitty tracking. + + def main(): setup_logzero(get_logger()) - print(get_dataframe()) + test_tz() + # print(get_dataframe()) if __name__ == '__main__': From 6b94f54600012c8ea1b02b67243572729ad8f23d Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 11 Apr 2019 14:53:39 +0100 Subject: [PATCH 19/23] some todos --- jawbone_provider/__init__.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/jawbone_provider/__init__.py b/jawbone_provider/__init__.py index 2826eff..ff98498 100755 --- a/jawbone_provider/__init__.py +++ b/jawbone_provider/__init__.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from typing import Dict, Any, List import json +from functools import lru_cache from datetime import datetime, date, time from pathlib import Path import logging @@ -20,12 +21,12 @@ def get_logger(): XID = str # TODO how to shared with backup thing? - Phases = Dict[XID, Any] -phases: Phases = json.loads(PHASES_FILE.read_text()) +@lru_cache(1) +def get_phases() -> Phases: + return json.loads(PHASES_FILE.read_text()) -# TODO namedtuple, cproperty? -# TODO cache? +# TODO use awakenings and quality class SleepEntry: def __init__(self, js) -> None: self.js = js @@ -37,7 +38,6 @@ class SleepEntry: def _fromts(self, ts: int) -> datetime: return pytz.utc.localize(datetime.utcfromtimestamp(ts)).astimezone(self._tz).astimezone(self._tz) - @property def _tz(self): return pytz.timezone(self._details['tz']) @@ -88,7 +88,7 @@ class SleepEntry: @property def phases(self) -> List[datetime]: # TODO make sure they are consistent with emfit? - return [self._fromts(i['time']) for i in phases[self.xid]] + return [self._fromts(i['time']) for i in get_phases()[self.xid]] def __str__(self) -> str: return f"{self.date_.strftime('%a %d %b')} {self.title}" @@ -96,6 +96,7 @@ class SleepEntry: def __repr__(self) -> str: return str(self) + def load_sleeps() -> List[SleepEntry]: sleeps = json.loads(SLEEPS_FILE.read_text()) return [SleepEntry(js) for js in sleeps] From c9702a325b8bc7485f81cfd941fb07c7acaa2995 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 11 Apr 2019 15:01:39 +0100 Subject: [PATCH 20/23] remove old ci.sh --- ci.sh | 3 --- main.py | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 100755 ci.sh diff --git a/ci.sh b/ci.sh deleted file mode 100755 index f584ed9..0000000 --- a/ci.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -python3.6 -mpylint -E main.py common.py products.py -python3.6 -mmypy main.py common.py products.py diff --git a/main.py b/main.py index a700c10..4b79727 100755 --- a/main.py +++ b/main.py @@ -85,6 +85,8 @@ def iter_useful(data_file: str): if not dt.is_bad(): yield dt +# TODO <<< hmm. these files do contain deep and light sleep?? +# also steps stats?? files = [ "/L/Dropbox/backups/jawbone/2015.csv", "/L/Dropbox/backups/jawbone/2016.csv", From 7b39a2089920b45c779bdd21743f38de53cf879c Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 20 Jul 2019 13:19:19 +0100 Subject: [PATCH 21/23] Fix imports --- jawbone_provider/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jawbone_provider/__init__.py b/jawbone_provider/__init__.py index ff98498..5cc6381 100755 --- a/jawbone_provider/__init__.py +++ b/jawbone_provider/__init__.py @@ -2,7 +2,7 @@ from typing import Dict, Any, List import json from functools import lru_cache -from datetime import datetime, date, time +from datetime import datetime, date, time, timedelta from pathlib import Path import logging import pytz @@ -109,7 +109,6 @@ from matplotlib.axes import Axes # type: ignore # pip install imageio # from imageio import imread # type: ignore -from scipy.misc import imresize # type: ignore def hhmm(time: datetime): From 8ebb93e2427d1d80e5ae576e7b60a9b828981624 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 11 Dec 2019 21:52:55 +0000 Subject: [PATCH 23/23] minor jawbone fixes --- jawbone_provider/__init__.py | 2 +- main.py | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/jawbone_provider/__init__.py b/jawbone_provider/__init__.py index 5cc6381..75c4f06 100755 --- a/jawbone_provider/__init__.py +++ b/jawbone_provider/__init__.py @@ -209,7 +209,7 @@ def sleeps_by_date() -> Dict[date, SleepEntry]: # sleeps = sleeps[:sleeps_count] # dt = {k: v for k, v in dt.items() if v is not None} -# TODO ?? +# TODO not really sure it belongs here... # import melatonin # dt = melatonin.get_data() diff --git a/main.py b/main.py index 4b79727..968e16e 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3.6 +#!/usr/bin/env python3 # TODO from kython import * -from kython.plotting import * +# from kython.plotting import * from csv import DictReader from itertools import islice @@ -87,10 +87,12 @@ def iter_useful(data_file: str): # TODO <<< hmm. these files do contain deep and light sleep?? # also steps stats?? +p = Path('/L/backups/jawbone/old_csv') +# TODO with_my? files = [ - "/L/Dropbox/backups/jawbone/2015.csv", - "/L/Dropbox/backups/jawbone/2016.csv", - "/L/Dropbox/backups/jawbone/2017.csv", + p / "2015.csv", + p / "2016.csv", + p / "2017.csv", ] useful = concat(*(list(iter_useful(f)) for f in files)) @@ -104,10 +106,12 @@ useful = concat(*(list(iter_useful(f)) for f in files)) dates = [parse_date(u.date, yearfirst=True, dayfirst=False) for u in useful] # TODO filter outliers? +# TODO don't need this anymore? it's gonna be in dashboards package +from kython.plotting import plot_timestamped for attr, lims, mavg, fig in [ - # ('light', (0, 400), 5, None), - # ('deep', (0, 600), 5, None), - # ('total', (200, 600), 5, None), + ('light', (0, 400), 5, None), + ('deep', (0, 600), 5, None), + ('total', (200, 600), 5, None), ('awake_time', (0, 1200), None, 1), ('asleep_time', (-100, 1000), None, 1), # ('awakenings', (0, 5)), @@ -116,8 +120,8 @@ for attr, lims, mavg, fig in [ dates_wke = [d for d in dates if d.weekday() >= 5] for dts, dn in [ (dates, 'total'), - # (dates_wkd, 'weekday'), - # (dates_wke, 'weekend') + (dates_wkd, 'weekday'), + (dates_wke, 'weekend') ]: mavgs = [] if mavg is not None: @@ -130,9 +134,10 @@ for attr, lims, mavg, fig in [ mavgs=mavgs, ylimits=lims, ytick_size=60, - figure=1, + # figure=1, ) - # plt.savefig(f'{attr}_{dn}.png') + plt.savefig(f'{attr}_{dn}.png') -plt.savefig('res.png') +# TODO use proper names? +# plt.savefig('res.png') # fig.show()