From 1e20420ce5196579b4588081512b5cc584934487 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 12 Feb 2019 21:53:05 +0000 Subject: [PATCH 01/11] initial impl --- bluemaestro/__init__.py | 68 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100755 bluemaestro/__init__.py diff --git a/bluemaestro/__init__.py b/bluemaestro/__init__.py new file mode 100755 index 0000000..cc3a88e --- /dev/null +++ b/bluemaestro/__init__.py @@ -0,0 +1,68 @@ +#!/usr/bin/python3 +import sqlite3 +# ugh, dataset stumpled over date format + +from datetime import datetime +import logging +from kython import setup_logzero +from pathlib import Path +DIR = Path("/L/zzz_syncthing_backups/bluemaestro/") + +def get_logger(): + return logging.getLogger('bluemaestro') + + +def get_data(): + logger = get_logger() + merged = {} + # TODO could get pretty big ? + + for f in sorted(DIR.glob('*.db')): + def reg(dt: datetime, value): + v = merged.get(dt, None) + if v is None: + merged[dt] = value + return + if value == v or (isinstance(v, set) and value in v): + return + if isinstance(v, set): + v.add(value) + else: + merged[dt] = {v, value} + # err = f'{f}: mismatch: {v} vs {value}' + # if abs(v - value) > 0.4: + # logger.warning(err) + # # TODO mm. dunno how to mark errors properly.. + # # raise AssertionError(err) + # else: + # pass + + db = sqlite3.connect(str(f)) + + datas = list(db.execute('select * from data')) + + for _, tss, temp, hum, pres, dew in datas: + # TODO is that utc??? + tss = tss.replace('Juli', 'Jul').replace('Aug.', 'Aug') + + + dt = datetime.strptime(tss, '%Y-%b-%d %H:%M') + reg(dt, temp) + + db.close() + + logger.info('total items: %d', len(merged)) + # TODO assert frequency? + for k, v in merged.items(): + # TODO shit. quite a few of them have varying values... how is that freaking possible???? + # most of them are withing 0.5 degree though... so just ignore? + if isinstance(v, set) and len(v) > 1: + print(k, v) + +def main(): + setup_logzero(get_logger(), level=logging.DEBUG) + get_data() + + +if __name__ == '__main__': + main() From bce5fd99b6332128ef632a088265f7c1078efecb Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 12 Feb 2019 22:29:23 +0000 Subject: [PATCH 02/11] checking script --- bluemaestro/__init__.py | 20 +++++++++++--------- bluemaestro/check.py | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 bluemaestro/check.py diff --git a/bluemaestro/__init__.py b/bluemaestro/__init__.py index cc3a88e..10c8b41 100755 --- a/bluemaestro/__init__.py +++ b/bluemaestro/__init__.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import sqlite3 # ugh, dataset stumpled over date format +from itertools import islice from datetime import datetime import logging @@ -12,12 +13,12 @@ def get_logger(): return logging.getLogger('bluemaestro') -def get_data(): +def get_temperature(): logger = get_logger() merged = {} - # TODO could get pretty big ? - for f in sorted(DIR.glob('*.db')): + # TODO how to trigger data download manually? + for f in list(sorted(DIR.glob('*.db'))): def reg(dt: datetime, value): v = merged.get(dt, None) if v is None: @@ -53,15 +54,16 @@ def get_data(): logger.info('total items: %d', len(merged)) # TODO assert frequency? - for k, v in merged.items(): - # TODO shit. quite a few of them have varying values... how is that freaking possible???? - # most of them are withing 0.5 degree though... so just ignore? - if isinstance(v, set) and len(v) > 1: - print(k, v) + # for k, v in merged.items(): + # # TODO shit. quite a few of them have varying values... how is that freaking possible???? + # # most of them are withing 0.5 degree though... so just ignore? + # if isinstance(v, set) and len(v) > 1: + # print(k, v) + return merged def main(): setup_logzero(get_logger(), level=logging.DEBUG) - get_data() + get_temperature() if __name__ == '__main__': diff --git a/bluemaestro/check.py b/bluemaestro/check.py new file mode 100644 index 0000000..92ca2bb --- /dev/null +++ b/bluemaestro/check.py @@ -0,0 +1,24 @@ +#!/usr/bin/python3 +import logging +from datetime import timedelta, datetime +from kython import setup_logzero + +from my.bluemaestro import get_temperature, get_logger + +def main(): + setup_logzero(get_logger(), level=logging.DEBUG) + + temps = get_temperature() + latest = sorted(temps.items())[:-2] + + prev, _ = latest[-2] + last, _ = latest[-1] + assert last - prev < timedelta(minutes=3), f'bad interval! {last - prev}' + single = (last - prev).seconds + + NOW = datetime.now() + assert NOW - last < timedelta(days=5), f'old backup! {last}' + + +if __name__ == '__main__': + main() From 773156bf220d5ae085acb56bdea06eb71fc1dc48 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 12 Feb 2019 22:30:10 +0000 Subject: [PATCH 03/11] fix mode --- bluemaestro/check.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bluemaestro/check.py diff --git a/bluemaestro/check.py b/bluemaestro/check.py old mode 100644 new mode 100755 From c7c6d9d4f6415eae009dbd3c0f41ea77c7a7b218 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 13 Feb 2019 21:55:01 +0000 Subject: [PATCH 04/11] add automatic backup path --- bluemaestro/__init__.py | 12 +++++++++--- bluemaestro/check.py | 3 ++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bluemaestro/__init__.py b/bluemaestro/__init__.py index 10c8b41..f8c006d 100755 --- a/bluemaestro/__init__.py +++ b/bluemaestro/__init__.py @@ -1,14 +1,18 @@ #!/usr/bin/python3 import sqlite3 # ugh, dataset stumpled over date format -from itertools import islice +from itertools import islice, chain from datetime import datetime import logging from kython import setup_logzero from pathlib import Path + DIR = Path("/L/zzz_syncthing_backups/bluemaestro/") +# TODO how to move them back? +DIR2 = Path("/L/zzz_syncthing_phone/phone-syncthing/backups/bluemaestro/") + def get_logger(): return logging.getLogger('bluemaestro') @@ -17,8 +21,10 @@ def get_temperature(): logger = get_logger() merged = {} - # TODO how to trigger data download manually? - for f in list(sorted(DIR.glob('*.db'))): + for f in list(sorted(chain( + DIR.glob('*.db'), + DIR2.glob('*.db'), + ))): def reg(dt: datetime, value): v = merged.get(dt, None) if v is None: diff --git a/bluemaestro/check.py b/bluemaestro/check.py index 92ca2bb..028d6d0 100755 --- a/bluemaestro/check.py +++ b/bluemaestro/check.py @@ -17,7 +17,8 @@ def main(): single = (last - prev).seconds NOW = datetime.now() - assert NOW - last < timedelta(days=5), f'old backup! {last}' + assert NOW - last < timedelta(days=2), f'old backup! {last}' + # TODO change to 1 later? if __name__ == '__main__': From 07b3388cefe0b0b7d98733fe2434583e106705cf Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 11 Mar 2019 23:23:03 +0000 Subject: [PATCH 05/11] Fix ruci --- bluemaestro/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bluemaestro/__init__.py b/bluemaestro/__init__.py index f8c006d..ee000bc 100755 --- a/bluemaestro/__init__.py +++ b/bluemaestro/__init__.py @@ -2,6 +2,7 @@ import sqlite3 # ugh, dataset stumpled over date format from itertools import islice, chain +from typing import Dict, Any from datetime import datetime import logging @@ -19,7 +20,7 @@ def get_logger(): def get_temperature(): logger = get_logger() - merged = {} + merged: Dict[datetime, Any] = {} for f in list(sorted(chain( DIR.glob('*.db'), From f34c796ee859155f86a248cc43d19500c2e30354 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 16 Apr 2019 08:21:13 +0100 Subject: [PATCH 06/11] better detection --- bluemaestro/__init__.py | 15 +++++++++++---- bluemaestro/check.py | 14 ++++++++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/bluemaestro/__init__.py b/bluemaestro/__init__.py index ee000bc..55974b8 100755 --- a/bluemaestro/__init__.py +++ b/bluemaestro/__init__.py @@ -18,14 +18,18 @@ def get_logger(): return logging.getLogger('bluemaestro') -def get_temperature(): +def get_temperature(all_=False): logger = get_logger() - merged: Dict[datetime, Any] = {} - for f in list(sorted(chain( + backups = list(sorted(chain( DIR.glob('*.db'), DIR2.glob('*.db'), - ))): + ))) + if not all_: + backups = [backups[-1]] + + merged: Dict[datetime, Any] = {} + for f in backups: def reg(dt: datetime, value): v = merged.get(dt, None) if v is None: @@ -68,6 +72,9 @@ def get_temperature(): # print(k, v) return merged +def test(): + get_temperature(all_=False) + def main(): setup_logzero(get_logger(), level=logging.DEBUG) get_temperature() diff --git a/bluemaestro/check.py b/bluemaestro/check.py index 028d6d0..34861a6 100755 --- a/bluemaestro/check.py +++ b/bluemaestro/check.py @@ -8,17 +8,23 @@ from my.bluemaestro import get_temperature, get_logger def main(): setup_logzero(get_logger(), level=logging.DEBUG) - temps = get_temperature() + temps = get_temperature(all_=False) latest = sorted(temps.items())[:-2] prev, _ = latest[-2] last, _ = latest[-1] + + POINTS_STORED = 6000 + FREQ_SEC = 60 + SECS_STORED = POINTS_STORED * FREQ_SEC + HOURS_STORED = POINTS_STORED / (60 * 60 / FREQ_SEC) # around 4 days + NOW = datetime.now() + assert NOW - last < timedelta(hours=HOURS_STORED / 2), f'old backup! {last}' + + assert last - prev < timedelta(minutes=3), f'bad interval! {last - prev}' single = (last - prev).seconds - NOW = datetime.now() - assert NOW - last < timedelta(days=2), f'old backup! {last}' - # TODO change to 1 later? if __name__ == '__main__': From 1d7c0905e17e01ac6835b202534f1fbbf353b079 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 4 Aug 2019 20:20:19 +0100 Subject: [PATCH 07/11] cachew for bluemaestro --- bluemaestro/__init__.py | 103 +++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 48 deletions(-) diff --git a/bluemaestro/__init__.py b/bluemaestro/__init__.py index 55974b8..252421e 100755 --- a/bluemaestro/__init__.py +++ b/bluemaestro/__init__.py @@ -1,46 +1,43 @@ #!/usr/bin/python3 -import sqlite3 -# ugh, dataset stumpled over date format -from itertools import islice, chain -from typing import Dict, Any - -from datetime import datetime import logging -from kython import setup_logzero +import sqlite3 +from datetime import datetime +from itertools import chain, islice from pathlib import Path +from typing import Any, Dict, Iterable, NamedTuple, Set + +from cachew import cachew +from kython import dictify +from kython.klogging import LazyLogger + + +CACHE = Path('/L/data/.cache/bluemaestro.cache') DIR = Path("/L/zzz_syncthing_backups/bluemaestro/") - # TODO how to move them back? DIR2 = Path("/L/zzz_syncthing_phone/phone-syncthing/backups/bluemaestro/") -def get_logger(): - return logging.getLogger('bluemaestro') +logger = LazyLogger('bluemaestro', level=logging.DEBUG) -def get_temperature(all_=False): - logger = get_logger() - - backups = list(sorted(chain( +def get_backup_files(): + return list(sorted(chain( DIR.glob('*.db'), DIR2.glob('*.db'), ))) - if not all_: - backups = [backups[-1]] - merged: Dict[datetime, Any] = {} - for f in backups: - def reg(dt: datetime, value): - v = merged.get(dt, None) - if v is None: - merged[dt] = value - return - if value == v or (isinstance(v, set) and value in v): - return - if isinstance(v, set): - v.add(value) - else: - merged[dt] = {v, value} + +class Point(NamedTuple): + dt: datetime + temp: float + + +@cachew(db_path=CACHE) +def iter_points(dbs) -> Iterable[Point]: + # I guess we can affort keeping them in sorted order + points: Set[Point] = set() + # TODO do some sanity check?? + for f in dbs: # err = f'{f}: mismatch: {v} vs {value}' # if abs(v - value) > 0.4: # logger.warning(err) @@ -48,36 +45,46 @@ def get_temperature(all_=False): # # raise AssertionError(err) # else: # pass + with sqlite3.connect(str(f)) as db: + datas = list(db.execute('select * from data')) + for _, tss, temp, hum, pres, dew in datas: + # TODO is that utc??? + tss = tss.replace('Juli', 'Jul').replace('Aug.', 'Aug') + dt = datetime.strptime(tss, '%Y-%b-%d %H:%M') + p = Point( + dt=dt, + temp=temp, + ) + if p in points: + continue + points.add(p) + for p in sorted(points, key=lambda p: p.dt): + yield p - db = sqlite3.connect(str(f)) - - datas = list(db.execute('select * from data')) - - for _, tss, temp, hum, pres, dew in datas: - # TODO is that utc??? - tss = tss.replace('Juli', 'Jul').replace('Aug.', 'Aug') - - - dt = datetime.strptime(tss, '%Y-%b-%d %H:%M') - reg(dt, temp) - - db.close() - - logger.info('total items: %d', len(merged)) + # logger.info('total items: %d', len(merged)) # TODO assert frequency? # for k, v in merged.items(): # # TODO shit. quite a few of them have varying values... how is that freaking possible???? # # most of them are withing 0.5 degree though... so just ignore? # if isinstance(v, set) and len(v) > 1: # print(k, v) - return merged + # for k, v in merged.items(): + # yield Point(dt=k, temp=v) # meh? + +# TODO does it even have to be a dict? +@dictify(key=lambda p: p.dt) +def get_temperature(backups=get_backup_files()): + yield from iter_points(backups) + def test(): - get_temperature(all_=False) + get_temperature(get_backup_files()[-1:]) def main(): - setup_logzero(get_logger(), level=logging.DEBUG) - get_temperature() + ll = list(iter_points(get_backup_files())) + print(len(ll)) + # print(get_temperature(get_backup_files()[-1:])) + # print(type(t)) if __name__ == '__main__': From a836cfccb5b28e7e8161dc40c81e9c433c780fee Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 4 Aug 2019 21:52:18 +0100 Subject: [PATCH 08/11] return list --- bluemaestro/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bluemaestro/__init__.py b/bluemaestro/__init__.py index 252421e..5c831e7 100755 --- a/bluemaestro/__init__.py +++ b/bluemaestro/__init__.py @@ -72,9 +72,9 @@ def iter_points(dbs) -> Iterable[Point]: # yield Point(dt=k, temp=v) # meh? # TODO does it even have to be a dict? -@dictify(key=lambda p: p.dt) +# @dictify(key=lambda p: p.dt) def get_temperature(backups=get_backup_files()): - yield from iter_points(backups) + return list(iter_points(backups)) def test(): From 827a57dd032cbc2c80f0b5a690877d8e39fa1b51 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 5 Aug 2019 18:44:07 +0100 Subject: [PATCH 09/11] fix checker script --- bluemaestro/check.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bluemaestro/check.py b/bluemaestro/check.py index 34861a6..9a0816e 100755 --- a/bluemaestro/check.py +++ b/bluemaestro/check.py @@ -3,13 +3,11 @@ import logging from datetime import timedelta, datetime from kython import setup_logzero -from my.bluemaestro import get_temperature, get_logger +from my.bluemaestro import get_temperature, logger def main(): - setup_logzero(get_logger(), level=logging.DEBUG) - - temps = get_temperature(all_=False) - latest = sorted(temps.items())[:-2] + temps = get_temperature() + latest = temps[:-2] prev, _ = latest[-2] last, _ = latest[-1] From fdec500458bda16bcc14c48296d9e0981a1a7cd1 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 14 Aug 2019 23:43:05 +0100 Subject: [PATCH 10/11] fix cachew --- bluemaestro/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bluemaestro/__init__.py b/bluemaestro/__init__.py index 5c831e7..4771bec 100755 --- a/bluemaestro/__init__.py +++ b/bluemaestro/__init__.py @@ -32,7 +32,7 @@ class Point(NamedTuple): temp: float -@cachew(db_path=CACHE) +@cachew(cache_path=CACHE) def iter_points(dbs) -> Iterable[Point]: # I guess we can affort keeping them in sorted order points: Set[Point] = set() From e2efe3bddaaaa146b7b941e2743b1507716c2f8a Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 17 Sep 2019 17:19:20 +0400 Subject: [PATCH 11/11] update bluemaestro path to prevent clashing --- bluemaestro/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bluemaestro/__init__.py b/bluemaestro/__init__.py index 4771bec..b250678 100755 --- a/bluemaestro/__init__.py +++ b/bluemaestro/__init__.py @@ -13,7 +13,7 @@ from kython.klogging import LazyLogger CACHE = Path('/L/data/.cache/bluemaestro.cache') -DIR = Path("/L/zzz_syncthing_backups/bluemaestro/") +DIR = Path("/L/zzz_syncthing_backups/bluemaestro-all/") # TODO how to move them back? DIR2 = Path("/L/zzz_syncthing_phone/phone-syncthing/backups/bluemaestro/")