Merge branch 'fix-yahoo-dates-with-nulls' into 'master'

Fix handling of Yahoo date rows with nulls

See merge request chrisberkhout/pricehist!1

src/pricehist/sources/yahoo.py

@@ -84,8 +84,10 @@ class Yahoo(BaseSource):
     def _amount(self, row, type):
         if type == "mid" and row["high"] != "null" and row["low"] != "null":
             return sum([Decimal(row["high"]), Decimal(row["low"])]) / 2
-        else:
+        elif row[type] != "null":
             return Decimal(row[type])
+        else:
+            return None
 
     def _data(self, series) -> (dict, csv.DictReader):
         base_url = "https://query1.finance.yahoo.com/v7/finance"

tests/pricehist/sources/test_yahoo.py

@@ -64,6 +64,13 @@ def long_ok(requests_mock):
     yield requests_mock
 
 
+@pytest.fixture
+def date_with_nulls_ok(requests_mock):
+    json = (Path(os.path.splitext(__file__)[0]) / "ibm-date-with-nulls.csv").read_text()
+    requests_mock.add(responses.GET, history_url("IBM"), body=json, status=200)
+    yield requests_mock
+
+
 def test_normalizesymbol(src):
     assert src.normalizesymbol("tsla") == "TSLA"
 
@@ -163,6 +170,13 @@ def test_fetch_from_before_start(src, type, spark_ok, long_ok):
     assert len(series.prices) > 9
 
 
+def test_fetch_skips_dates_with_nulls(src, type, spark_ok, date_with_nulls_ok):
+    series = src.fetch(Series("IBM", "", type, "2021-01-05", "2021-01-07"))
+    assert series.prices[0] == Price("2021-01-05", Decimal("123.101204"))
+    assert series.prices[1] == Price("2021-01-07", Decimal("125.882545"))
+    assert len(series.prices) == 2
+
+
 def test_fetch_to_future(src, type, spark_ok, recent_ok):
     series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2100-01-08"))
     assert len(series.prices) > 0

tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv

@@ -0,0 +1,4 @@
+Date,Open,High,Low,Close,Adj Close,Volume
+2021-01-05,125.010002,126.680000,124.610001,126.139999,123.101204,6114600
+2021-01-06,null,null,null,null,null,null
+2021-01-07,130.039993,130.460007,128.259995,128.990005,125.882545,4507400

tests/pricehist/test_cli.py

@@ -54,7 +54,7 @@ def test_cli_no_args_shows_usage(capfd):
     cli.cli(w("pricehist"))
     out, err = capfd.readouterr()
     assert "usage: pricehist" in out
-    assert "optional arguments:" in out
+    assert "optional arguments:" in out or "options:" in out
     assert "commands:" in out
 
 
@@ -64,7 +64,7 @@ def test_cli_help_shows_usage_and_exits(capfd):
     assert e.value.code == 0
     out, err = capfd.readouterr()
     assert "usage: pricehist" in out
-    assert "optional arguments:" in out
+    assert "optional arguments:" in out or "options:" in out
     assert "commands:" in out