From b6f4c175303e6470b24ded7a4112b81abc40ed7c Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 14 Sep 2024 22:49:45 +0200 Subject: [PATCH 1/9] Skip coindesk live test. --- tests/live.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/live.sh b/tests/live.sh index fc24c08..cb2dbeb 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -128,7 +128,7 @@ date,base,quote,amount,source,type 2021-01-07,BTC,USD,39713.5079,coindeskbpi,close 2021-01-08,BTC,USD,40519.4486,coindeskbpi,close END -run_test "$name" "$cmd" "$expected" +skip_test "$name" "$cmd" "$expected" name="CoinMarketCap" cmd="pricehist fetch coinmarketcap BTC/EUR -s 2021-01-04 -e 2021-01-08" From ee8ca0573d73765f298a83961eb2eb1583173f48 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 12:46:43 +0200 Subject: [PATCH 2/9] yahoo: add back null handling, improve timestamp handling. Thanks @arkn98! --- src/pricehist/sources/yahoo.py | 15 +++++++++++---- tests/pricehist/sources/test_yahoo.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index ce5cf7d..65857bb 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -72,6 +72,7 @@ class Yahoo(BaseSource): data = self._data(series) quote = data["chart"]["result"][0]["meta"]["currency"] + offset = data["chart"]["result"][0]["meta"]["gmtoffset"] timestamps = data["chart"]["result"][0]["timestamp"] adjclose_data = data["chart"]["result"][0]["indicators"]["adjclose"][0] @@ -79,19 +80,25 @@ class Yahoo(BaseSource): amounts = {**adjclose_data, **rest_data} prices = [ - Price(ts, amount) + Price(date, amount) for i in range(len(timestamps)) - if (ts := datetime.fromtimestamp(timestamps[i]).strftime("%Y-%m-%d")) - <= series.end + if (date := self._date_from_ts(timestamps[i], offset)) <= series.end if (amount := self._amount(amounts, series.type, i)) is not None ] return dataclasses.replace(series, quote=quote, prices=prices) + def _date_from_ts(self, ts, offset) -> str: + return ( + datetime.fromtimestamp(ts - offset) + .replace(tzinfo=timezone.utc) + .strftime("%Y-%m-%d") + ) + def _amount(self, amounts, type, i): if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": return sum([Decimal(amounts["high"][i]), Decimal(amounts["low"][i])]) / 2 - elif amounts[type] != "null": + elif amounts[type] != "null" and amounts[type][i] is not None: return Decimal(amounts[type][i]) else: return None diff --git a/tests/pricehist/sources/test_yahoo.py b/tests/pricehist/sources/test_yahoo.py index 7e8d055..d490d86 100644 --- a/tests/pricehist/sources/test_yahoo.py +++ b/tests/pricehist/sources/test_yahoo.py @@ -54,6 +54,13 @@ def long_ok(requests_mock): yield requests_mock +@pytest.fixture +def with_null_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "inrx-with-null.json").read_text() + requests_mock.add(responses.GET, url("INR=X"), body=json, status=200) + yield requests_mock + + def test_normalizesymbol(src): assert src.normalizesymbol("tsla") == "TSLA" @@ -157,6 +164,13 @@ def test_fetch_from_before_start(src, type, long_ok): assert len(series.prices) > 9 +def test_fetch_skips_dates_with_nulls(src, type, with_null_ok): + series = src.fetch(Series("INR=X", "", type, "2017-07-10", "2017-07-12")) + assert series.prices[0] == Price("2017-07-10", Decimal("64.61170196533203125")) + assert series.prices[1] == Price("2017-07-12", Decimal("64.52559661865234375")) + assert len(series.prices) == 2 + + def test_fetch_to_future(src, type, recent_ok): series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2100-01-08")) assert len(series.prices) > 0 From 77b2776e55d0c9a6e4265bff94665f1838cba7c0 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 12:59:39 +0200 Subject: [PATCH 3/9] yahoo: More graceful handling of responses with meta but no timestamps. --- src/pricehist/sources/yahoo.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 65857bb..169bf71 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -146,11 +146,10 @@ class Yahoo(BaseSource): raise exceptions.InvalidPair( series.base, series.quote, self, "Symbol not found." ) - if code == 400 and "Data doesn't exist" in text: + elif code == 400 and "Data doesn't exist" in text: raise exceptions.BadResponse( "No data for the given interval. Try requesting a larger interval." ) - elif code == 404 and "Timestamp data missing" in text: raise exceptions.BadResponse( "Data missing. The given interval may be for a gap in the data " @@ -169,4 +168,10 @@ class Yahoo(BaseSource): "The data couldn't be parsed. " ) from e + if "timestamp" not in data["chart"]["result"][0]: + raise exceptions.BadResponse( + "No data for the given interval. " + "There may be a problem with the symbol or the interval." + ) + return data From 1164724ffb8cb2b614af06d2a1d4634cf0a2c886 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 13:01:11 +0200 Subject: [PATCH 4/9] Version 1.4.11. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d5fc989..98bb405 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.10" +version = "1.4.11" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 738cf25..e42b3cf 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.10" +__version__ = "1.4.11" From c78154df3a265da9c26a149d78bff1989fe046fd Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 13:07:50 +0200 Subject: [PATCH 5/9] Add missing file. --- .../sources/test_yahoo/inrx-with-null.json | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tests/pricehist/sources/test_yahoo/inrx-with-null.json diff --git a/tests/pricehist/sources/test_yahoo/inrx-with-null.json b/tests/pricehist/sources/test_yahoo/inrx-with-null.json new file mode 100644 index 0000000..5ae762e --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/inrx-with-null.json @@ -0,0 +1,119 @@ +{ + "chart": { + "result": [ + { + "meta": { + "currency": "INR", + "symbol": "INR=X", + "exchangeName": "CCY", + "fullExchangeName": "CCY", + "instrumentType": "CURRENCY", + "firstTradeDate": 1070236800, + "regularMarketTime": 1726284616, + "hasPrePostMarketData": false, + "gmtoffset": 3600, + "timezone": "BST", + "exchangeTimezoneName": "Europe/London", + "regularMarketPrice": 83.89, + "fiftyTwoWeekHigh": 83.89, + "fiftyTwoWeekLow": 83.89, + "regularMarketDayHigh": 83.89, + "regularMarketDayLow": 83.89, + "regularMarketVolume": 0, + "longName": "USD/INR", + "shortName": "USD/INR", + "chartPreviousClose": 64.6117, + "priceHint": 4, + "currentTradingPeriod": { + "pre": { + "timezone": "BST", + "start": 1726182000, + "end": 1726182000, + "gmtoffset": 3600 + }, + "regular": { + "timezone": "BST", + "start": 1726182000, + "end": 1726268340, + "gmtoffset": 3600 + }, + "post": { + "timezone": "BST", + "start": 1726268340, + "end": 1726268340, + "gmtoffset": 3600 + } + }, + "dataGranularity": "1d", + "range": "", + "validRanges": [ + "1d", + "5d", + "1mo", + "3mo", + "6mo", + "1y", + "2y", + "5y", + "10y", + "ytd", + "max" + ] + }, + "timestamp": [ + 1499641200, + 1499727600, + 1499814000, + 1499900400 + ], + "indicators": { + "quote": [ + { + "open": [ + 64.6155014038086, + null, + 64.55549621582031, + 64.46800231933594 + ], + "volume": [ + 0, + null, + 0, + 0 + ], + "low": [ + 64.41000366210938, + null, + 64.3499984741211, + 64.33999633789062 + ], + "close": [ + 64.61170196533203, + null, + 64.52559661865234, + 64.36499786376953 + ], + "high": [ + 64.6155014038086, + null, + 64.56999969482422, + 64.48419952392578 + ] + } + ], + "adjclose": [ + { + "adjclose": [ + 64.61170196533203, + null, + 64.52559661865234, + 64.36499786376953 + ] + } + ] + } + } + ], + "error": null + } +} From dffe6f8e89678751478382b8549b520f43296e9e Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 13:15:18 +0200 Subject: [PATCH 6/9] Timezone handling tweak. --- src/pricehist/sources/yahoo.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 169bf71..81dd0f7 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -89,11 +89,7 @@ class Yahoo(BaseSource): return dataclasses.replace(series, quote=quote, prices=prices) def _date_from_ts(self, ts, offset) -> str: - return ( - datetime.fromtimestamp(ts - offset) - .replace(tzinfo=timezone.utc) - .strftime("%Y-%m-%d") - ) + return datetime.fromtimestamp(ts - offset).strftime("%Y-%m-%d") def _amount(self, amounts, type, i): if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": From 53f39a26ef9e28565826456e65d119b6f4209d6a Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 11:35:56 +0000 Subject: [PATCH 7/9] More time correction. --- src/pricehist/sources/yahoo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 81dd0f7..25d92fe 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -82,14 +82,14 @@ class Yahoo(BaseSource): prices = [ Price(date, amount) for i in range(len(timestamps)) - if (date := self._date_from_ts(timestamps[i], offset)) <= series.end + if (date := self._ts_to_date(timestamps[i] + offset)) <= series.end if (amount := self._amount(amounts, series.type, i)) is not None ] return dataclasses.replace(series, quote=quote, prices=prices) - def _date_from_ts(self, ts, offset) -> str: - return datetime.fromtimestamp(ts - offset).strftime("%Y-%m-%d") + def _ts_to_date(self, ts) -> str: + return datetime.fromtimestamp(ts, tz=timezone.utc).date().isoformat() def _amount(self, amounts, type, i): if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": From ab507b189cdb30f4df7ead380bc0151740f249b3 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 12:16:36 +0000 Subject: [PATCH 8/9] Update live test. --- tests/live.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/live.sh b/tests/live.sh index cb2dbeb..157cdad 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -75,10 +75,10 @@ name="Alpha Vantage physical currency" cmd="pricehist fetch alphavantage AUD/EUR -s 2021-01-11 -e 2021-01-14" read -r -d '' expected < Date: Sun, 15 Sep 2024 12:17:10 +0000 Subject: [PATCH 9/9] Version 1.4.12. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98bb405..5d555aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.11" +version = "1.4.12" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index e42b3cf..2736991 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.11" +__version__ = "1.4.12"