fz0x1/HPI
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

my.bluemaestro: workaround weird timestamps by keeping track of the latest timestamp

Browse source
This commit is contained in:
Dima Gerasimov 2020-10-03 23:24:56 +01:00 • committed by karlicoss
parent 8e8d9702f3
commit 5babbb44d0
2 changed files with 12 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
my/bluemaestro/__init__.py
View file

@ -9,7 +9,7 @@ from datetime import datetime, timedelta
from pathlib import Path from pathlib import Path
import re import re
import sqlite3 import sqlite3
from typing import Iterable, NamedTuple, Sequence, Set from typing import Iterable, NamedTuple, Sequence, Set, Optional
from ..core.common import mcachew, LazyLogger, get_files from ..core.common import mcachew, LazyLogger, get_files
@ -37,7 +37,7 @@ tz = pytz.timezone('Europe/London')
@mcachew(cache_path=cache_dir() / 'bluemaestro.cache') @mcachew(cache_path=cache_dir() / 'bluemaestro.cache')
def measurements(dbs=inputs()) -> Iterable[Measurement]: def measurements(dbs=inputs()) -> Iterable[Measurement]:
emitted: Set[datetime] = set() last: Optional[datetime] = None
# tables are immutable, so can save on processing.. # tables are immutable, so can save on processing..
processed_tables: Set[str] = set() processed_tables: Set[str] = set()
@ -107,14 +107,18 @@ def measurements(dbs=inputs()) -> Iterable[Measurement]:
dt = datetime.fromtimestamp(tsc / 1000, tz=tz) dt = datetime.fromtimestamp(tsc / 1000, tz=tz)
temp = tempc / 10 # for some reason it's in tenths of degrees temp = tempc / 10 # for some reason it's in tenths of degrees
# need to exclude bad databases? some have weird years like 2000
# sanity check ## sanity checks (todo make defensive/configurable?)
# not sure how that happens.. but basically they'd better be excluded
assert dt.year >= 2015, (f, name, dt)
assert -60 <= temp <= 60, (f, dt, temp) assert -60 <= temp <= 60, (f, dt, temp)
##
tot += 1 tot += 1
if dt in emitted: if last is not None and last >= dt:
continue continue
emitted.add(dt) # todo for performance, pass 'last' to sqlite instead?
last = dt
new += 1 new += 1
p = Measurement( p = Measurement(
dt=dt, dt=dt,

6
tests/bluemaestro.py
View file

@ -17,12 +17,10 @@ def test() -> None:
# check that timezone is set properly # check that timezone is set properly
assert dts == '20200824 22' assert dts == '20200824 22'
# NOTE: broken at the moment due to weirdness with timestamping assert len(tp) == 1 # should be unique
# assert len(tp) == 1 # should be unique
# 2.5 K + 4 K datapoints, somwhat overlapping # 2.5 K + 4 K datapoints, somwhat overlapping
# NOTE: boken at the moment due to weirdness with timestamping assert len(res) < 6000
# assert len(res) < 6000
import pytest # type: ignore import pytest # type: ignore