From ee8ca0573d73765f298a83961eb2eb1583173f48 Mon Sep 17 00:00:00 2001
From: Chris Berkhout <chrisberkhout@gmail.com>
Date: Sun, 15 Sep 2024 12:46:43 +0200
Subject: [PATCH] yahoo: add back null handling, improve timestamp handling.

Thanks @arkn98!
---
 src/pricehist/sources/yahoo.py        | 15 +++++++++++----
 tests/pricehist/sources/test_yahoo.py | 14 ++++++++++++++
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py
index ce5cf7d..65857bb 100644
--- a/src/pricehist/sources/yahoo.py
+++ b/src/pricehist/sources/yahoo.py
@@ -72,6 +72,7 @@ class Yahoo(BaseSource):
 
         data = self._data(series)
         quote = data["chart"]["result"][0]["meta"]["currency"]
+        offset = data["chart"]["result"][0]["meta"]["gmtoffset"]
 
         timestamps = data["chart"]["result"][0]["timestamp"]
         adjclose_data = data["chart"]["result"][0]["indicators"]["adjclose"][0]
@@ -79,19 +80,25 @@ class Yahoo(BaseSource):
         amounts = {**adjclose_data, **rest_data}
 
         prices = [
-            Price(ts, amount)
+            Price(date, amount)
             for i in range(len(timestamps))
-            if (ts := datetime.fromtimestamp(timestamps[i]).strftime("%Y-%m-%d"))
-            <= series.end
+            if (date := self._date_from_ts(timestamps[i], offset)) <= series.end
             if (amount := self._amount(amounts, series.type, i)) is not None
         ]
 
         return dataclasses.replace(series, quote=quote, prices=prices)
 
+    def _date_from_ts(self, ts, offset) -> str:
+        return (
+            datetime.fromtimestamp(ts - offset)
+            .replace(tzinfo=timezone.utc)
+            .strftime("%Y-%m-%d")
+        )
+
     def _amount(self, amounts, type, i):
         if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null":
             return sum([Decimal(amounts["high"][i]), Decimal(amounts["low"][i])]) / 2
-        elif amounts[type] != "null":
+        elif amounts[type] != "null" and amounts[type][i] is not None:
             return Decimal(amounts[type][i])
         else:
             return None
diff --git a/tests/pricehist/sources/test_yahoo.py b/tests/pricehist/sources/test_yahoo.py
index 7e8d055..d490d86 100644
--- a/tests/pricehist/sources/test_yahoo.py
+++ b/tests/pricehist/sources/test_yahoo.py
@@ -54,6 +54,13 @@ def long_ok(requests_mock):
     yield requests_mock
 
 
+@pytest.fixture
+def with_null_ok(requests_mock):
+    json = (Path(os.path.splitext(__file__)[0]) / "inrx-with-null.json").read_text()
+    requests_mock.add(responses.GET, url("INR=X"), body=json, status=200)
+    yield requests_mock
+
+
 def test_normalizesymbol(src):
     assert src.normalizesymbol("tsla") == "TSLA"
 
@@ -157,6 +164,13 @@ def test_fetch_from_before_start(src, type, long_ok):
     assert len(series.prices) > 9
 
 
+def test_fetch_skips_dates_with_nulls(src, type, with_null_ok):
+    series = src.fetch(Series("INR=X", "", type, "2017-07-10", "2017-07-12"))
+    assert series.prices[0] == Price("2017-07-10", Decimal("64.61170196533203125"))
+    assert series.prices[1] == Price("2017-07-12", Decimal("64.52559661865234375"))
+    assert len(series.prices) == 2
+
+
 def test_fetch_to_future(src, type, recent_ok):
     series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2100-01-08"))
     assert len(series.prices) > 0