From 0835fef493d20f7cad23ef6d064861aed1548d6e Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 31 May 2018 21:05:56 +0100 Subject: [PATCH 1/6] Initial --- .gitignore | 170 +++++++++++++++++++++++++++++++++++++++++ rescuetime/__init__.py | 0 rescuetime/__main__.py | 102 +++++++++++++++++++++++++ run | 3 + 4 files changed, 275 insertions(+) create mode 100644 .gitignore create mode 100644 rescuetime/__init__.py create mode 100644 rescuetime/__main__.py create mode 100755 run diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..07d89cb --- /dev/null +++ b/.gitignore @@ -0,0 +1,170 @@ + +# Created by https://www.gitignore.io/api/python,emacs + +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile +projectile-bookmarks.eld + +# directory configuration +.dir-locals.el + +# saveplace +places + +# url cache +url/cache/ + +# cedet +ede-projects.el + +# smex +smex-items + +# company-statistics +company-statistics-cache.el + +# anaconda-mode +anaconda-mode/ + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + + +# End of https://www.gitignore.io/api/python,emacs diff --git a/rescuetime/__init__.py b/rescuetime/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rescuetime/__main__.py b/rescuetime/__main__.py new file mode 100644 index 0000000..28d32c0 --- /dev/null +++ b/rescuetime/__main__.py @@ -0,0 +1,102 @@ +import logging +from os import listdir +from os.path import join + +from kython import json_load, JSONType +from kython.logging import setup_logzero + +logger = logging.getLogger("rescuetime-provider") +setup_logzero(logger) + + +PATH = "/L/backups/rescuetime" + +def try_load(fp: str): + try: + j: JSONType + with open(fp, 'r') as fo: + j = json_load(fo) + + if j is None: + logger.warning(f"Corrupted: {fp}") + return None + + return j + except Exception as e: + if 'Expecting value' in str(e): + logger.warning(f"Corrupted: {fp}") + else: + raise e + return None + +from datetime import datetime, timedelta +from typing import NamedTuple, Dict, List + +_DT_FMT = "%Y-%m-%dT%H:%M:%S" + +class Entry(NamedTuple): + dt: datetime # TODO mm, no timezone? + duration_s: int + + @staticmethod + def from_dict(row: Dict): + dt_s = row['Date'] + dur = row['Time Spent (seconds)'] + dt = datetime.strptime(dt_s, _DT_FMT) + return Entry(dt=dt, duration_s=dur) + + @staticmethod + def from_row(row: List): + COL_DT = 0 + COL_DUR = 1 + dt_s = row[COL_DT] + dur = row[COL_DUR] + dt = datetime.strptime(dt_s, _DT_FMT) + return Entry(dt=dt, duration_s=dur) + +from typing import Set +all_entries: Set[Entry] = set() + +for f in sorted(listdir(PATH))[-5:]: # TODO FIXME + fp = join(PATH, f) + j = try_load(fp) + if j is None: + continue + + cols = j['row_headers'] + seen = 0 + total = 0 + for row in j['rows']: + e = Entry.from_row(row) + total += 1 + if e in all_entries: + seen += 1 + else: + all_entries.add(e) + print(f"{f}: {seen}/{total}") + # import ipdb; ipdb.set_trace() + # print(len(j)) + +all_sorted = sorted(all_entries, key=lambda e: e.dt) +gap = timedelta(hours=3) + +groups = [] +group = [] + +for e in all_sorted: + if len(group) > 0 and e.dt - group[-1].dt > gap: + groups.append(group) + group = [] + group.append(e) + +if len(group) > 0: + groups.append(group) + group = [] + +for gr in groups: + print(f"{gr[0].dt}--{gr[-1].dt}") + + +# TODO merged db? +# TODO ok, it summarises my sleep intervals pretty well. I guess should adjust it for the fact I don't sleep during the day, and it would be ok! + diff --git a/run b/run new file mode 100755 index 0000000..0d0e25d --- /dev/null +++ b/run @@ -0,0 +1,3 @@ +#!/bin/bash +cd "$(dirname "$0")" +python3 -m rescuetime \ No newline at end of file From d8568587b44156aa065ea08aa18efa8582187b7e Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 6 Apr 2019 16:37:06 +0100 Subject: [PATCH 2/6] more kython --- rescuetime/__init__.py | 82 +++++++++++++++++++++++++++++++++++ rescuetime/__main__.py | 97 ++---------------------------------------- run | 3 +- 3 files changed, 88 insertions(+), 94 deletions(-) diff --git a/rescuetime/__init__.py b/rescuetime/__init__.py index e69de29..a1a201b 100644 --- a/rescuetime/__init__.py +++ b/rescuetime/__init__.py @@ -0,0 +1,82 @@ +import logging +from pathlib import Path +import json +from datetime import datetime, timedelta +from typing import NamedTuple, Dict, List, Set +from functools import lru_cache + + +from kython import JSONType, fget, group_by_cmp + + +def get_logger(): + return logging.getLogger("rescuetime-provider") + +_PATH = Path("/L/backups/rescuetime") + +def try_load(fp: Path): + logger = get_logger() + try: + return json.loads(fp.read_text()) + except Exception as e: + if 'Expecting value' in str(e): + logger.warning(f"Corrupted: {fp}") + else: + raise e + return None + + +_DT_FMT = "%Y-%m-%dT%H:%M:%S" + +class Entry(NamedTuple): + # TODO ugh, appears to be local time... + dt: datetime + duration_s: int + + @staticmethod + def from_dict(row: Dict): + dt_s = row['Date'] + dur = row['Time Spent (seconds)'] + dt = datetime.strptime(dt_s, _DT_FMT) + return Entry(dt=dt, duration_s=dur) + + @staticmethod + def from_row(row: List): + COL_DT = 0 + COL_DUR = 1 + dt_s = row[COL_DT] + dur = row[COL_DUR] + dt = datetime.strptime(dt_s, _DT_FMT) + return Entry(dt=dt, duration_s=dur) + + +@lru_cache() +def get_rescuetime(): + entries: Set[Entry] = set() + + for fp in list(sorted(_PATH.glob('*.json')))[-5:]: + j = try_load(fp) + if j is None: + continue + + cols = j['row_headers'] + seen = 0 + total = 0 + for row in j['rows']: + e = Entry.from_row(row) + total += 1 + if e in entries: + seen += 1 + else: + entries.add(e) + print(f"{fp}: {seen}/{total}") + # import ipdb; ipdb.set_trace() + # print(len(j)) + res = sorted(entries, key=fget(Entry.dt)) + return res + + +def get_groups(gap=timedelta(hours=3)): + data = get_rescuetime() + return group_by_cmp(data, lambda a, b: (b.dt - a.dt) <= gap, dist=1) + diff --git a/rescuetime/__main__.py b/rescuetime/__main__.py index 28d32c0..b68177d 100644 --- a/rescuetime/__main__.py +++ b/rescuetime/__main__.py @@ -1,102 +1,13 @@ -import logging -from os import listdir -from os.path import join +from kython.klogging import setup_logzero -from kython import json_load, JSONType -from kython.logging import setup_logzero +from . import get_logger, get_groups -logger = logging.getLogger("rescuetime-provider") +logger = get_logger() setup_logzero(logger) - -PATH = "/L/backups/rescuetime" - -def try_load(fp: str): - try: - j: JSONType - with open(fp, 'r') as fo: - j = json_load(fo) - - if j is None: - logger.warning(f"Corrupted: {fp}") - return None - - return j - except Exception as e: - if 'Expecting value' in str(e): - logger.warning(f"Corrupted: {fp}") - else: - raise e - return None - -from datetime import datetime, timedelta -from typing import NamedTuple, Dict, List - -_DT_FMT = "%Y-%m-%dT%H:%M:%S" - -class Entry(NamedTuple): - dt: datetime # TODO mm, no timezone? - duration_s: int - - @staticmethod - def from_dict(row: Dict): - dt_s = row['Date'] - dur = row['Time Spent (seconds)'] - dt = datetime.strptime(dt_s, _DT_FMT) - return Entry(dt=dt, duration_s=dur) - - @staticmethod - def from_row(row: List): - COL_DT = 0 - COL_DUR = 1 - dt_s = row[COL_DT] - dur = row[COL_DUR] - dt = datetime.strptime(dt_s, _DT_FMT) - return Entry(dt=dt, duration_s=dur) - -from typing import Set -all_entries: Set[Entry] = set() - -for f in sorted(listdir(PATH))[-5:]: # TODO FIXME - fp = join(PATH, f) - j = try_load(fp) - if j is None: - continue - - cols = j['row_headers'] - seen = 0 - total = 0 - for row in j['rows']: - e = Entry.from_row(row) - total += 1 - if e in all_entries: - seen += 1 - else: - all_entries.add(e) - print(f"{f}: {seen}/{total}") - # import ipdb; ipdb.set_trace() - # print(len(j)) - -all_sorted = sorted(all_entries, key=lambda e: e.dt) -gap = timedelta(hours=3) - -groups = [] -group = [] - -for e in all_sorted: - if len(group) > 0 and e.dt - group[-1].dt > gap: - groups.append(group) - group = [] - group.append(e) - -if len(group) > 0: - groups.append(group) - group = [] - -for gr in groups: +for gr in get_groups(): print(f"{gr[0].dt}--{gr[-1].dt}") - # TODO merged db? # TODO ok, it summarises my sleep intervals pretty well. I guess should adjust it for the fact I don't sleep during the day, and it would be ok! diff --git a/run b/run index 0d0e25d..9ec68dc 100755 --- a/run +++ b/run @@ -1,3 +1,4 @@ #!/bin/bash +set -eu cd "$(dirname "$0")" -python3 -m rescuetime \ No newline at end of file +python3 -m rescuetime From 2043bea317ec517d422be8545cdb1bca992f6ec0 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 6 Apr 2019 17:07:56 +0100 Subject: [PATCH 3/6] add checker for latest entries --- check | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 check diff --git a/check b/check new file mode 100755 index 0000000..b6207e1 --- /dev/null +++ b/check @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +from datetime import datetime, timedelta + +from rescuetime import get_rescuetime + +rs = get_rescuetime(latest=1) +latest_dt = rs[-1].dt + +assert (datetime.now() - latest_dt) < timedelta(days=1) From 5363248dcbbca85f10c244fd6f07ece45533e044 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 6 Apr 2019 17:09:16 +0100 Subject: [PATCH 4/6] add activity --- rescuetime/__init__.py | 21 +++++++++------------ rescuetime/__main__.py | 8 +++++--- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/rescuetime/__init__.py b/rescuetime/__init__.py index a1a201b..d40c83b 100644 --- a/rescuetime/__init__.py +++ b/rescuetime/__init__.py @@ -32,29 +32,26 @@ class Entry(NamedTuple): # TODO ugh, appears to be local time... dt: datetime duration_s: int - - @staticmethod - def from_dict(row: Dict): - dt_s = row['Date'] - dur = row['Time Spent (seconds)'] - dt = datetime.strptime(dt_s, _DT_FMT) - return Entry(dt=dt, duration_s=dur) + activity: str @staticmethod def from_row(row: List): COL_DT = 0 COL_DUR = 1 - dt_s = row[COL_DT] - dur = row[COL_DUR] + COL_ACTIVITY = 3 + dt_s = row[COL_DT] + dur = row[COL_DUR] + activity = row[COL_ACTIVITY] + # TODO utc?? dt = datetime.strptime(dt_s, _DT_FMT) - return Entry(dt=dt, duration_s=dur) + return Entry(dt=dt, duration_s=dur, activity=activity) @lru_cache() -def get_rescuetime(): +def get_rescuetime(latest=None): entries: Set[Entry] = set() - for fp in list(sorted(_PATH.glob('*.json')))[-5:]: + for fp in list(sorted(_PATH.glob('*.json')))[(0 if latest is None else -latest):]: j = try_load(fp) if j is None: continue diff --git a/rescuetime/__main__.py b/rescuetime/__main__.py index b68177d..a50fc32 100644 --- a/rescuetime/__main__.py +++ b/rescuetime/__main__.py @@ -1,12 +1,14 @@ from kython.klogging import setup_logzero -from . import get_logger, get_groups +from . import get_logger, get_groups, get_rescuetime logger = get_logger() setup_logzero(logger) -for gr in get_groups(): - print(f"{gr[0].dt}--{gr[-1].dt}") +# for gr in get_groups(): +# print(f"{gr[0].dt}--{gr[-1].dt}") +for e in get_rescuetime(latest=2): + print(e) # TODO merged db? # TODO ok, it summarises my sleep intervals pretty well. I guess should adjust it for the fact I don't sleep during the day, and it would be ok! From 7b2ff3627c51f5375982b632c5e587e4ccc0084d Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 6 Apr 2019 18:18:50 +0100 Subject: [PATCH 5/6] SIP on filling influxdb --- rescuetime/__init__.py | 17 +++++++++++++++++ rescuetime/__main__.py | 7 ++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/rescuetime/__init__.py b/rescuetime/__init__.py index d40c83b..3f81d1a 100644 --- a/rescuetime/__init__.py +++ b/rescuetime/__init__.py @@ -77,3 +77,20 @@ def get_groups(gap=timedelta(hours=3)): data = get_rescuetime() return group_by_cmp(data, lambda a, b: (b.dt - a.dt) <= gap, dist=1) + + +def fill_influxdb(): + from influxdb import InfluxDBClient # type: ignore + client = InfluxDBClient() + # client.delete_series(database='lastfm', measurement='phone') + db = 'test' + client.drop_database(db) + client.create_database(db) + jsons = [{ + "measurement": 'phone', + "tags": {}, + "time": str(e.dt), + "fields": {"name": e.activity}, + } for e in get_rescuetime()] + client.write_points(jsons, database=db) # TODO?? + diff --git a/rescuetime/__main__.py b/rescuetime/__main__.py index a50fc32..5568d64 100644 --- a/rescuetime/__main__.py +++ b/rescuetime/__main__.py @@ -1,14 +1,15 @@ from kython.klogging import setup_logzero -from . import get_logger, get_groups, get_rescuetime +from . import get_logger, get_groups, get_rescuetime, fill_influxdb logger = get_logger() setup_logzero(logger) # for gr in get_groups(): # print(f"{gr[0].dt}--{gr[-1].dt}") -for e in get_rescuetime(latest=2): - print(e) +# for e in get_rescuetime(latest=2): +# print(e) +fill_influxdb() # TODO merged db? # TODO ok, it summarises my sleep intervals pretty well. I guess should adjust it for the fact I don't sleep during the day, and it would be ok! From 71eb70eaa8f731310bf89a131ccbfe4f93e231b5 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 8 May 2019 21:17:22 +0100 Subject: [PATCH 6/6] fix ruci --- check | 12 +++++++++--- rescuetime/__init__.py | 12 ++++++++---- rescuetime/__main__.py | 21 ++++++++++++--------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/check b/check index b6207e1..0477a01 100755 --- a/check +++ b/check @@ -3,7 +3,13 @@ from datetime import datetime, timedelta from rescuetime import get_rescuetime -rs = get_rescuetime(latest=1) -latest_dt = rs[-1].dt -assert (datetime.now() - latest_dt) < timedelta(days=1) +def main(): + rs = get_rescuetime(latest=1) + latest_dt = rs[-1].dt + + assert (datetime.now() - latest_dt) < timedelta(days=1) + + +if __name__ == '__main__': + main() diff --git a/rescuetime/__init__.py b/rescuetime/__init__.py index 3f81d1a..dd9b9ba 100644 --- a/rescuetime/__init__.py +++ b/rescuetime/__init__.py @@ -2,7 +2,7 @@ import logging from pathlib import Path import json from datetime import datetime, timedelta -from typing import NamedTuple, Dict, List, Set +from typing import NamedTuple, Dict, List, Set, Optional from functools import lru_cache @@ -47,11 +47,15 @@ class Entry(NamedTuple): return Entry(dt=dt, duration_s=dur, activity=activity) -@lru_cache() -def get_rescuetime(latest=None): +@lru_cache(1) +def get_rescuetime(latest: Optional[int]=None): + if latest is None: + latest = 0 + entries: Set[Entry] = set() - for fp in list(sorted(_PATH.glob('*.json')))[(0 if latest is None else -latest):]: + # pylint: disable=invalid-unary-operand-type + for fp in list(sorted(_PATH.glob('*.json')))[-latest:]: j = try_load(fp) if j is None: continue diff --git a/rescuetime/__main__.py b/rescuetime/__main__.py index 5568d64..2e5edbd 100644 --- a/rescuetime/__main__.py +++ b/rescuetime/__main__.py @@ -2,15 +2,18 @@ from kython.klogging import setup_logzero from . import get_logger, get_groups, get_rescuetime, fill_influxdb -logger = get_logger() -setup_logzero(logger) +def main(): + logger = get_logger() + setup_logzero(logger) -# for gr in get_groups(): -# print(f"{gr[0].dt}--{gr[-1].dt}") -# for e in get_rescuetime(latest=2): -# print(e) -fill_influxdb() + # for gr in get_groups(): + # print(f"{gr[0].dt}--{gr[-1].dt}") + # for e in get_rescuetime(latest=2): + # print(e) + fill_influxdb() -# TODO merged db? -# TODO ok, it summarises my sleep intervals pretty well. I guess should adjust it for the fact I don't sleep during the day, and it would be ok! + # TODO merged db? + # TODO ok, it summarises my sleep intervals pretty well. I guess should adjust it for the fact I don't sleep during the day, and it would be ok! +if __name__ == '__main__': + main()