bluemaestro: include humidity, pressure and dewpoint data

my/bluemaestro/__init__.py

@@ -26,7 +26,10 @@ def inputs() -> Sequence[Path]:
 
 class Measurement(NamedTuple):
     dt: datetime
-    temp: float
+    temp    : float # Celsius
+    humidity: float # percent
+    pressure: float # mBar
+    dewpoint: float # Celsius
 
 
 # fixme: later, rely on the timezone provider
@@ -48,9 +51,7 @@ def measurements(dbs=inputs()) -> Iterable[Measurement]:
         # todo assert increasing timestamp?
         with sqlite3.connect(f'file:{f}?immutable=1', uri=True) as db:
             try:
-                # try old format first
+                datas = db.execute(f'SELECT "{f.name}" as name, Time, Temperature, Humidity, Pressure, Dewpoint FROM data ORDER BY log_index')
-                # todo Humidity, Pressure, Dewpoint
-                datas = db.execute(f'SELECT "{f.name}" as name, Time, Temperature FROM data ORDER BY log_index')
                 oldfmt = True
             except sqlite3.OperationalError:
                 # Right, this looks really bad.
@@ -83,30 +84,29 @@ def measurements(dbs=inputs()) -> Iterable[Measurement]:
                 # todo could just filter out the older datapoints?? dunno.
 
                 # eh. a bit horrible, but seems the easiest way to do it?
-                # todo could exclude logs that we already processed??
+                # note: for some reason everything in the new table multiplied by 10
-                # todo humiReadings, pressReadings, dewpReadings
+                query = ' UNION '.join(
-                query = ' UNION '.join(f'SELECT "{t}" AS name, unix, tempReadings FROM {t}' for t in log_tables)
+                    f'SELECT "{t}" AS name, unix, tempReadings / 10.0, humiReadings / 10.0, pressReadings / 10.0, dewpReadings / 10.0 FROM {t}'
+                    for t in log_tables
+                )
                 if len(log_tables) > 0: # ugh. otherwise end up with syntax error..
                     query = f'SELECT * FROM ({query}) ORDER BY name, unix'
                 datas = db.execute(query)
                 oldfmt = False
 
-            for i, (name, tsc, tempc) in enumerate(datas):
+            for i, (name, tsc, temp, hum, pres, dewp) in enumerate(datas):
+                # note: bluemaestro keeps local datetime
                 if oldfmt:
-                    # TODO double check the timezone
                     tss = tsc.replace('Juli', 'Jul').replace('Aug.', 'Aug')
-                    dt = tz.localize(datetime.strptime(tss, '%Y-%b-%d %H:%M'))
+                    dt = datetime.strptime(tss, '%Y-%b-%d %H:%M')
-                    temp = tempc
+                    dt = tz.localize(dt)
                 else:
                     m = re.search(r'_(\d+)_', name)
                     assert m is not None
                     export_ts = int(m.group(1))
                     edt = datetime.fromtimestamp(export_ts / 1000, tz=tz)
 
-                    # right, seems that it stores local datetime
                     dt = datetime.fromtimestamp(tsc / 1000, tz=tz)
-                    temp = tempc / 10 # for some reason it's in tenths of degrees
-
 
                 ## sanity checks (todo make defensive/configurable?)
                 # not sure how that happens.. but basically they'd better be excluded
@@ -123,7 +123,9 @@ def measurements(dbs=inputs()) -> Iterable[Measurement]:
                 p = Measurement(
                     dt=dt,
                     temp=temp,
-                    # TODO use pressure and humidity as well
+                    pressure=pres,
+                    humidity=hum,
+                    dewpoint=dewp,
                 )
                 yield p
         logger.debug('%s: new %d/%d', f, new, tot)

my/bluemaestro/check.py

@@ -5,12 +5,12 @@ from datetime import timedelta, datetime
 from my.bluemaestro import measurements, logger
 
 # TODO move this to backup checker?
-def main():
+def main() -> None:
     temps = list(measurements())
     latest = temps[:-2]
 
-    prev, _ = latest[-2]
+    prev = latest[-2].dt
-    last, _ = latest[-1]
+    last = latest[-1].dt
 
     POINTS_STORED = 6000
     FREQ_SEC = 60

tests/bluemaestro.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
 from pathlib import Path
 
+from more_itertools import one
+
+import pytest # type: ignore
+
 
 def test() -> None:
     from my.bluemaestro import measurements
@@ -20,7 +24,20 @@ def test() -> None:
     assert len(res) < 6000
 
 
-import pytest # type: ignore
+@pytest.mark.skip(reason='todo add old database to the testdata')
+def test_old_db() -> None:
+    from my.bluemaestro import measurements
+    res = list(measurements())
+
+    r1 = one(x for x in res if x.dt.strftime('%Y%m%d %H:%M:%S') == '20181003 09:07:00')
+    r2 = one(x for x in res if x.dt.strftime('%Y%m%d %H:%M:%S') == '20181003 09:19:00')
+
+    assert r1.temp == 16.8
+    assert r2.temp == 18.3
+    assert r1.pressure == 1025.8
+    assert r2.pressure == 1009.9
+
+
 @pytest.fixture(autouse=True)
 def prepare():
     testdata = Path(__file__).absolute().parent.parent / 'testdata'