From e8dec0bf64310ebf9e33af19c455ee3c72631c55 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 3 Aug 2024 17:15:17 +0200 Subject: [PATCH 01/14] Update Alpha Vantage rate limit handling. --- src/pricehist/sources/alphavantage.py | 4 ++-- tests/pricehist/sources/test_alphavantage.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index e60f0f6..c42d490 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -337,8 +337,8 @@ class AlphaVantage(BaseSource): def _raise_for_generic_errors(self, data): if type(data) is dict: - if "Note" in data and "call frequency" in data["Note"]: - raise exceptions.RateLimit(data["Note"]) + if "Information" in data and "daily rate limits" in data["Information"]: + raise exceptions.RateLimit(data["Information"]) if ( "Information" in data and "unlock" in data["Information"] diff --git a/tests/pricehist/sources/test_alphavantage.py b/tests/pricehist/sources/test_alphavantage.py index 8ff70cc..40febb0 100644 --- a/tests/pricehist/sources/test_alphavantage.py +++ b/tests/pricehist/sources/test_alphavantage.py @@ -59,11 +59,11 @@ digital_url = re.compile( ) rate_limit_json = ( - '{ "Note": "' - "Thank you for using Alpha Vantage! Our standard API call frequency is 5 " - "calls per minute and 500 calls per day. Please visit " - "https://www.alphavantage.co/premium/ if you would like to target a higher " - "API call frequency." + '{ "Information": "' + "Thank you for using Alpha Vantage! Our standard API rate limit is 25 " + "requests per day. Please subscribe to any of the premium plans at " + "https://www.alphavantage.co/premium/ to instantly remove all daily rate " + "limits." '" }' ) From b7d0d739ab9841ef05e0793cac9041b0843640ff Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 3 Aug 2024 17:19:36 +0200 Subject: [PATCH 02/14] Version 1.4.9. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9cf089a..d1a7ce1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.8" +version = "1.4.9" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 4963389..1ad354e 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.8" +__version__ = "1.4.9" From 51e297b75252b44e4c8ba3f2bffbeb734c820b8f Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 3 Aug 2024 17:23:53 +0200 Subject: [PATCH 03/14] Update alphavantage source notes regarding API rate limit. --- src/pricehist/sources/alphavantage.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index c42d490..40d5b98 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -56,10 +56,8 @@ class AlphaVantage(BaseSource): "Beware that digital currencies quoted in non-USD currencies may " "be converted from USD data at one recent exchange rate rather " "than using historical rates.\n" - "Alpha Vantage's standard API call frequency limits is 5 calls per " - "minute and 500 per day, so you may need to pause between successive " - "commands. Note that retrieving prices for one stock consumes two " - "API calls." + "Alpha Vantage's standard API rate limit is 25 requests per day. " + "Note that retrieving prices for one stock consumes two API calls." ) def _stock_symbols_message(self): From 59574e91566d80f970a90009064cf967662d0043 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 14 Sep 2024 22:22:35 +0200 Subject: [PATCH 04/14] Fix yahoo source. --- src/pricehist/sources/yahoo.py | 103 +++----- tests/live.sh | 35 ++- tests/pricehist/sources/test_yahoo.py | 181 ++++--------- .../test_yahoo/ibm-date-with-nulls.csv | 4 - .../sources/test_yahoo/ibm-long-partial.csv | 11 - .../sources/test_yahoo/ibm-long-partial.json | 249 ++++++++++++++++++ .../sources/test_yahoo/tsla-recent.csv | 6 - .../sources/test_yahoo/tsla-recent.json | 126 +++++++++ .../sources/test_yahoo/tsla-spark.json | 77 ------ 9 files changed, 475 insertions(+), 317 deletions(-) delete mode 100644 tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv delete mode 100644 tests/pricehist/sources/test_yahoo/ibm-long-partial.csv create mode 100644 tests/pricehist/sources/test_yahoo/ibm-long-partial.json delete mode 100644 tests/pricehist/sources/test_yahoo/tsla-recent.csv create mode 100644 tests/pricehist/sources/test_yahoo/tsla-recent.json delete mode 100644 tests/pricehist/sources/test_yahoo/tsla-spark.json diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 6997301..ce5cf7d 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -1,4 +1,3 @@ -import csv import dataclasses import json import logging @@ -71,63 +70,36 @@ class Yahoo(BaseSource): series.base, series.quote, self, "Don't specify the quote currency." ) - quote, history = self._data(series) + data = self._data(series) + quote = data["chart"]["result"][0]["meta"]["currency"] + + timestamps = data["chart"]["result"][0]["timestamp"] + adjclose_data = data["chart"]["result"][0]["indicators"]["adjclose"][0] + rest_data = data["chart"]["result"][0]["indicators"]["quote"][0] + amounts = {**adjclose_data, **rest_data} prices = [ - Price(row["date"], amount) - for row in history - if (amount := self._amount(row, series.type)) + Price(ts, amount) + for i in range(len(timestamps)) + if (ts := datetime.fromtimestamp(timestamps[i]).strftime("%Y-%m-%d")) + <= series.end + if (amount := self._amount(amounts, series.type, i)) is not None ] return dataclasses.replace(series, quote=quote, prices=prices) - def _amount(self, row, type): - if type == "mid" and row["high"] != "null" and row["low"] != "null": - return sum([Decimal(row["high"]), Decimal(row["low"])]) / 2 - elif row[type] != "null": - return Decimal(row[type]) + def _amount(self, amounts, type, i): + if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": + return sum([Decimal(amounts["high"][i]), Decimal(amounts["low"][i])]) / 2 + elif amounts[type] != "null": + return Decimal(amounts[type][i]) else: return None - def _data(self, series) -> (dict, csv.DictReader): - base_url = "https://query1.finance.yahoo.com/v7/finance" + def _data(self, series) -> dict: + base_url = "https://query1.finance.yahoo.com/v8/finance/chart" headers = {"User-Agent": f"pricehist/{__version__}"} - - spark_url = f"{base_url}/spark" - spark_params = { - "symbols": series.base, - "range": "1d", - "interval": "1d", - "indicators": "close", - "includeTimestamps": "false", - "includePrePost": "false", - } - try: - spark_response = self.log_curl( - requests.get(spark_url, params=spark_params, headers=headers) - ) - except Exception as e: - raise exceptions.RequestError(str(e)) from e - - code = spark_response.status_code - text = spark_response.text - if code == 404 and "No data found for spark symbols" in text: - raise exceptions.InvalidPair( - series.base, series.quote, self, "Symbol not found." - ) - - try: - spark_response.raise_for_status() - except Exception as e: - raise exceptions.BadResponse(str(e)) from e - - try: - spark = json.loads(spark_response.content) - quote = spark["spark"]["result"][0]["response"][0]["meta"]["currency"] - except Exception as e: - raise exceptions.ResponseParsingError( - "The spark data couldn't be parsed. " - ) from e + url = f"{base_url}/{series.base}" start_ts = int( datetime.strptime(series.start, "%Y-%m-%d") @@ -142,24 +114,26 @@ class Yahoo(BaseSource): 24 * 60 * 60 ) # some symbols require padding on the end timestamp - history_url = f"{base_url}/download/{series.base}" - history_params = { + params = { + "symbol": series.base, "period1": start_ts, "period2": end_ts, "interval": "1d", - "events": "history", + "events": "capitalGain%7Cdiv%7Csplit", "includeAdjustedClose": "true", + "formatted": "true", + "userYfid": "true", + "lang": "en-US", + "region": "US", } try: - history_response = self.log_curl( - requests.get(history_url, params=history_params, headers=headers) - ) + response = self.log_curl(requests.get(url, params=params, headers=headers)) except Exception as e: raise exceptions.RequestError(str(e)) from e - code = history_response.status_code - text = history_response.text + code = response.status_code + text = response.text if code == 404 and "No data found, symbol may be delisted" in text: raise exceptions.InvalidPair( @@ -177,20 +151,15 @@ class Yahoo(BaseSource): ) try: - history_response.raise_for_status() + response.raise_for_status() except Exception as e: raise exceptions.BadResponse(str(e)) from e try: - history_lines = history_response.content.decode("utf-8").splitlines() - history_lines[0] = history_lines[0].lower().replace(" ", "") - history = csv.DictReader(history_lines, delimiter=",") + data = json.loads(response.content) except Exception as e: - raise exceptions.ResponseParsingError(str(e)) from e + raise exceptions.ResponseParsingError( + "The data couldn't be parsed. " + ) from e - if history_lines[0] != "date,open,high,low,close,adjclose,volume": - raise exceptions.ResponseParsingError("Unexpected CSV format") - - requested_history = [row for row in history if row["date"] <= series.end] - - return (quote, requested_history) + return data diff --git a/tests/live.sh b/tests/live.sh index dd5f626..fc24c08 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -116,21 +116,20 @@ date,base,quote,amount,source,type 2021-01-07,BTC,EUR,31208.49,coinbasepro,mid 2021-01-08,BTC,EUR,32019,coinbasepro,mid END -run_test "$name" "$cmd" "$expected" - -name="CoinDesk Bitcoin Price Index" -cmd="pricehist fetch coindesk BTC/EUR -s 2021-01-04 -e 2021-01-08" -read -r -d '' expected < 9 -def test_fetch_skips_dates_with_nulls(src, type, spark_ok, date_with_nulls_ok): - series = src.fetch(Series("IBM", "", type, "2021-01-05", "2021-01-07")) - assert series.prices[0] == Price("2021-01-05", Decimal("123.101204")) - assert series.prices[1] == Price("2021-01-07", Decimal("125.882545")) - assert len(series.prices) == 2 - - -def test_fetch_to_future(src, type, spark_ok, recent_ok): +def test_fetch_to_future(src, type, recent_ok): series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2100-01-08")) assert len(series.prices) > 0 -def test_fetch_no_data_in_past(src, type, spark_ok, requests_mock): +def test_fetch_no_data_in_past(src, type, requests_mock): requests_mock.add( responses.GET, - history_url("TSLA"), + url("TSLA"), status=400, body=( "400 Bad Request: Data doesn't exist for " @@ -203,10 +177,10 @@ def test_fetch_no_data_in_past(src, type, spark_ok, requests_mock): assert "No data for the given interval" in str(e.value) -def test_fetch_no_data_in_future(src, type, spark_ok, requests_mock): +def test_fetch_no_data_in_future(src, type, requests_mock): requests_mock.add( responses.GET, - history_url("TSLA"), + url("TSLA"), status=400, body=( "400 Bad Request: Data doesn't exist for " @@ -218,10 +192,10 @@ def test_fetch_no_data_in_future(src, type, spark_ok, requests_mock): assert "No data for the given interval" in str(e.value) -def test_fetch_no_data_on_weekend(src, type, spark_ok, requests_mock): +def test_fetch_no_data_on_weekend(src, type, requests_mock): requests_mock.add( responses.GET, - history_url("TSLA"), + url("TSLA"), status=404, body="404 Not Found: Timestamp data missing.", ) @@ -233,30 +207,7 @@ def test_fetch_no_data_on_weekend(src, type, spark_ok, requests_mock): def test_fetch_bad_sym(src, type, requests_mock): requests_mock.add( responses.GET, - spark_url, - status=404, - body="""{ - "spark": { - "result": null, - "error": { - "code": "Not Found", - "description": "No data found for spark symbols" - } - } - }""", - ) - with pytest.raises(exceptions.InvalidPair) as e: - src.fetch(Series("NOTABASE", "", type, "2021-01-04", "2021-01-08")) - assert "Symbol not found" in str(e.value) - - -def test_fetch_bad_sym_history(src, type, spark_ok, requests_mock): - # In practice the spark history requests should succeed or fail together. - # This extra test ensures that a failure of the the history part is handled - # correctly even if the spark part succeeds. - requests_mock.add( - responses.GET, - history_url("NOTABASE"), + url("NOTABASE"), status=404, body="404 Not Found: No data found, symbol may be delisted", ) @@ -271,61 +222,23 @@ def test_fetch_giving_quote(src, type): assert "quote currency" in str(e.value) -def test_fetch_spark_network_issue(src, type, requests_mock): +def test_fetch_network_issue(src, type, requests_mock): body = requests.exceptions.ConnectionError("Network issue") - requests_mock.add(responses.GET, spark_url, body=body) + requests_mock.add(responses.GET, url("TSLA"), body=body) with pytest.raises(exceptions.RequestError) as e: src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) assert "Network issue" in str(e.value) -def test_fetch_spark_bad_status(src, type, requests_mock): - requests_mock.add(responses.GET, spark_url, status=500, body="Some other reason") +def test_fetch_bad_status(src, type, requests_mock): + requests_mock.add(responses.GET, url("TSLA"), status=500, body="Some other reason") with pytest.raises(exceptions.BadResponse) as e: src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) assert "Internal Server Error" in str(e.value) -def test_fetch_spark_parsing_error(src, type, requests_mock): - requests_mock.add(responses.GET, spark_url, body="NOT JSON") - with pytest.raises(exceptions.ResponseParsingError) as e: - src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) - assert "spark data couldn't be parsed" in str(e.value) - - -def test_fetch_spark_unexpected_json(src, type, requests_mock): - requests_mock.add(responses.GET, spark_url, body='{"notdata": []}') - with pytest.raises(exceptions.ResponseParsingError) as e: - src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) - assert "spark data couldn't be parsed" in str(e.value) - - -def test_fetch_history_network_issue(src, type, spark_ok, requests_mock): - body = requests.exceptions.ConnectionError("Network issue") - requests_mock.add(responses.GET, history_url("TSLA"), body=body) - with pytest.raises(exceptions.RequestError) as e: - src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) - assert "Network issue" in str(e.value) - - -def test_fetch_history_bad_status(src, type, spark_ok, requests_mock): - requests_mock.add( - responses.GET, history_url("TSLA"), status=500, body="Some other reason" - ) - with pytest.raises(exceptions.BadResponse) as e: - src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) - assert "Internal Server Error" in str(e.value) - - -def test_fetch_history_parsing_error(src, type, spark_ok, requests_mock): - requests_mock.add(responses.GET, history_url("TSLA"), body="") +def test_fetch_parsing_error(src, type, requests_mock): + requests_mock.add(responses.GET, url("TSLA"), body="") with pytest.raises(exceptions.ResponseParsingError) as e: src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) assert "error occurred while parsing data from the source" in str(e.value) - - -def test_fetch_history_unexpected_csv_format(src, type, spark_ok, requests_mock): - requests_mock.add(responses.GET, history_url("TSLA"), body="BAD HEADER\nBAD DATA") - with pytest.raises(exceptions.ResponseParsingError) as e: - src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) - assert "Unexpected CSV format" in str(e.value) diff --git a/tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv b/tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv deleted file mode 100644 index 601b395..0000000 --- a/tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv +++ /dev/null @@ -1,4 +0,0 @@ -Date,Open,High,Low,Close,Adj Close,Volume -2021-01-05,125.010002,126.680000,124.610001,126.139999,123.101204,6114600 -2021-01-06,null,null,null,null,null,null -2021-01-07,130.039993,130.460007,128.259995,128.990005,125.882545,4507400 diff --git a/tests/pricehist/sources/test_yahoo/ibm-long-partial.csv b/tests/pricehist/sources/test_yahoo/ibm-long-partial.csv deleted file mode 100644 index 98149ad..0000000 --- a/tests/pricehist/sources/test_yahoo/ibm-long-partial.csv +++ /dev/null @@ -1,11 +0,0 @@ -Date,Open,High,Low,Close,Adj Close,Volume -1962-01-02,7.713333,7.713333,7.626667,7.626667,1.837710,390000 -1962-01-03,7.626667,7.693333,7.626667,7.693333,1.853774,292500 -1962-01-04,7.693333,7.693333,7.613333,7.616667,1.835299,262500 -1962-01-05,7.606667,7.606667,7.453333,7.466667,1.799155,367500 -1962-01-08,7.460000,7.460000,7.266667,7.326667,1.765422,547500 -2021-01-04,125.849998,125.919998,123.040001,123.940002,120.954201,5179200 -2021-01-05,125.010002,126.680000,124.610001,126.139999,123.101204,6114600 -2021-01-06,126.900002,131.880005,126.720001,129.289993,126.175316,7956700 -2021-01-07,130.039993,130.460007,128.259995,128.990005,125.882545,4507400 -2021-01-08,128.570007,129.320007,126.980003,128.529999,125.433624,4676200 diff --git a/tests/pricehist/sources/test_yahoo/ibm-long-partial.json b/tests/pricehist/sources/test_yahoo/ibm-long-partial.json new file mode 100644 index 0000000..df98efa --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/ibm-long-partial.json @@ -0,0 +1,249 @@ +{ + "chart": { + "result": [ + { + "meta": { + "currency": "USD", + "symbol": "IBM", + "exchangeName": "NYQ", + "fullExchangeName": "NYSE", + "instrumentType": "EQUITY", + "firstTradeDate": -252322200, + "regularMarketTime": 1726257602, + "hasPrePostMarketData": true, + "gmtoffset": -14400, + "timezone": "EDT", + "exchangeTimezoneName": "America/New_York", + "regularMarketPrice": 214.79, + "fiftyTwoWeekHigh": 216.08, + "fiftyTwoWeekLow": 212.13, + "regularMarketDayHigh": 216.08, + "regularMarketDayLow": 212.13, + "regularMarketVolume": 4553547, + "longName": "International Business Machines Corporation", + "shortName": "International Business Machines", + "chartPreviousClose": 7.291, + "priceHint": 2, + "currentTradingPeriod": { + "pre": { + "timezone": "EDT", + "end": 1726234200, + "start": 1726214400, + "gmtoffset": -14400 + }, + "regular": { + "timezone": "EDT", + "end": 1726257600, + "start": 1726234200, + "gmtoffset": -14400 + }, + "post": { + "timezone": "EDT", + "end": 1726272000, + "start": 1726257600, + "gmtoffset": -14400 + } + }, + "dataGranularity": "1d", + "range": "", + "validRanges": [ + "1d", + "5d", + "1mo", + "3mo", + "6mo", + "1y", + "2y", + "5y", + "10y", + "ytd", + "max" + ] + }, + "timestamp": [ + -252322200, + -252235800, + -252149400, + -252063000, + -251803800, + 1609770600, + 1609857000, + 1609943400, + 1610029800, + 1610116200 + ], + "events": { + "dividends": { + "-249298200": { + "amount": 0.000956, + "date": -249298200 + }, + "-241439400": { + "amount": 0.000956, + "date": -241439400 + }, + "-233577000": { + "amount": 0.000956, + "date": -233577000 + }, + "-225797400": { + "amount": 0.000956, + "date": -225797400 + }, + "-217848600": { + "amount": 0.001275, + "date": -217848600 + }, + "1573137000": { + "amount": 1.548757, + "date": 1573137000 + }, + "1581085800": { + "amount": 1.548757, + "date": 1581085800 + }, + "1588858200": { + "amount": 1.558317, + "date": 1588858200 + }, + "1596807000": { + "amount": 1.558317, + "date": 1596807000 + }, + "1604932200": { + "amount": 1.558317, + "date": 1604932200 + } + }, + "splits": { + "-177417000": { + "date": -177417000, + "numerator": 5.0, + "denominator": 4.0, + "splitRatio": "5:4" + }, + "-114345000": { + "date": -114345000, + "numerator": 3.0, + "denominator": 2.0, + "splitRatio": "3:2" + }, + "-53343000": { + "date": -53343000, + "numerator": 2.0, + "denominator": 1.0, + "splitRatio": "2:1" + }, + "107530200": { + "date": 107530200, + "numerator": 5.0, + "denominator": 4.0, + "splitRatio": "5:4" + }, + "297091800": { + "date": 297091800, + "numerator": 4.0, + "denominator": 1.0, + "splitRatio": "4:1" + }, + "864826200": { + "date": 864826200, + "numerator": 2.0, + "denominator": 1.0, + "splitRatio": "2:1" + }, + "927811800": { + "date": 927811800, + "numerator": 2.0, + "denominator": 1.0, + "splitRatio": "2:1" + } + } + }, + "indicators": { + "quote": [ + { + "close": [ + 7.2912678718566895, + 7.3550028800964355, + 7.281707763671875, + 7.138305187225342, + 7.00446081161499, + 118.48948669433594, + 120.59273529052734, + 123.60420989990234, + 123.31739807128906, + 122.87763214111328 + ], + "low": [ + 7.2912678718566895, + 7.2912678718566895, + 7.2785210609436035, + 7.125557899475098, + 6.9471001625061035, + 117.62906646728516, + 119.13002014160156, + 121.14722442626953, + 122.61949920654297, + 121.39579010009766 + ], + "open": [ + 7.374124050140381, + 7.2912678718566895, + 7.3550028800964355, + 7.272148132324219, + 7.131930828094482, + 120.31549072265625, + 119.5124282836914, + 121.3193130493164, + 124.32122039794922, + 122.9158706665039 + ], + "high": [ + 7.374124050140381, + 7.3550028800964355, + 7.3550028800964355, + 7.272148132324219, + 7.131930828094482, + 120.38240814208984, + 121.1089859008789, + 126.08030700683594, + 124.7227554321289, + 123.63288879394531 + ], + "volume": [ + 407940, + 305955, + 274575, + 384405, + 572685, + 5417443, + 6395872, + 8322708, + 4714740, + 4891305 + ] + } + ], + "adjclose": [ + { + "adjclose": [ + 1.5133211612701416, + 1.5265485048294067, + 1.5113375186920166, + 1.4815733432769775, + 1.4537923336029053, + 99.60364532470703, + 101.37164306640625, + 103.90313720703125, + 103.66202545166016, + 103.29237365722656 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/tests/pricehist/sources/test_yahoo/tsla-recent.csv b/tests/pricehist/sources/test_yahoo/tsla-recent.csv deleted file mode 100644 index 48b5692..0000000 --- a/tests/pricehist/sources/test_yahoo/tsla-recent.csv +++ /dev/null @@ -1,6 +0,0 @@ -Date,Open,High,Low,Close,Adj Close,Volume -2021-01-04,719.460022,744.489990,717.190002,729.770020,729.770020,48638200 -2021-01-05,723.659973,740.840027,719.200012,735.109985,735.109985,32245200 -2021-01-06,758.489990,774.000000,749.099976,755.979980,755.979980,44700000 -2021-01-07,777.630005,816.989990,775.200012,816.039978,816.039978,51498900 -2021-01-08,856.000000,884.489990,838.390015,880.020020,880.020020,75055500 \ No newline at end of file diff --git a/tests/pricehist/sources/test_yahoo/tsla-recent.json b/tests/pricehist/sources/test_yahoo/tsla-recent.json new file mode 100644 index 0000000..3f35daa --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/tsla-recent.json @@ -0,0 +1,126 @@ +{ + "chart": { + "result": [ + { + "meta": { + "currency": "USD", + "symbol": "TSLA", + "exchangeName": "NMS", + "fullExchangeName": "NasdaqGS", + "instrumentType": "EQUITY", + "firstTradeDate": 1277818200, + "regularMarketTime": 1726257600, + "hasPrePostMarketData": true, + "gmtoffset": -14400, + "timezone": "EDT", + "exchangeTimezoneName": "America/New_York", + "regularMarketPrice": 230.29, + "fiftyTwoWeekHigh": 232.664, + "fiftyTwoWeekLow": 226.32, + "regularMarketDayHigh": 232.664, + "regularMarketDayLow": 226.32, + "regularMarketVolume": 59096538, + "longName": "Tesla, Inc.", + "shortName": "Tesla, Inc.", + "chartPreviousClose": 235.223, + "priceHint": 2, + "currentTradingPeriod": { + "pre": { + "timezone": "EDT", + "start": 1726214400, + "end": 1726234200, + "gmtoffset": -14400 + }, + "regular": { + "timezone": "EDT", + "start": 1726234200, + "end": 1726257600, + "gmtoffset": -14400 + }, + "post": { + "timezone": "EDT", + "start": 1726257600, + "end": 1726272000, + "gmtoffset": -14400 + } + }, + "dataGranularity": "1d", + "range": "", + "validRanges": [ + "1d", + "5d", + "1mo", + "3mo", + "6mo", + "1y", + "2y", + "5y", + "10y", + "ytd", + "max" + ] + }, + "timestamp": [ + 1609770600, + 1609857000, + 1609943400, + 1610029800, + 1610116200 + ], + "indicators": { + "quote": [ + { + "open": [ + 239.82000732421875, + 241.22000122070312, + 252.8300018310547, + 259.2099914550781, + 285.3333435058594 + ], + "close": [ + 243.2566680908203, + 245.0366668701172, + 251.9933319091797, + 272.0133361816406, + 293.3399963378906 + ], + "high": [ + 248.163330078125, + 246.94667053222656, + 258.0, + 272.3299865722656, + 294.8299865722656 + ], + "low": [ + 239.06333923339844, + 239.73333740234375, + 249.6999969482422, + 258.3999938964844, + 279.46331787109375 + ], + "volume": [ + 145914600, + 96735600, + 134100000, + 154496700, + 225166500 + ] + } + ], + "adjclose": [ + { + "adjclose": [ + 243.2566680908203, + 245.0366668701172, + 251.9933319091797, + 272.0133361816406, + 293.3399963378906 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/tests/pricehist/sources/test_yahoo/tsla-spark.json b/tests/pricehist/sources/test_yahoo/tsla-spark.json deleted file mode 100644 index 53e7585..0000000 --- a/tests/pricehist/sources/test_yahoo/tsla-spark.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "spark": { - "result": [ - { - "symbol": "TSLA", - "response": [ - { - "meta": { - "currency": "USD", - "symbol": "TSLA", - "exchangeName": "NMS", - "instrumentType": "EQUITY", - "firstTradeDate": 1277818200, - "regularMarketTime": 1626465603, - "gmtoffset": -14400, - "timezone": "EDT", - "exchangeTimezoneName": "America/New_York", - "regularMarketPrice": 644.22, - "chartPreviousClose": 650.6, - "priceHint": 2, - "currentTradingPeriod": { - "pre": { - "timezone": "EDT", - "start": 1626422400, - "end": 1626442200, - "gmtoffset": -14400 - }, - "regular": { - "timezone": "EDT", - "start": 1626442200, - "end": 1626465600, - "gmtoffset": -14400 - }, - "post": { - "timezone": "EDT", - "start": 1626465600, - "end": 1626480000, - "gmtoffset": -14400 - } - }, - "dataGranularity": "1d", - "range": "1d", - "validRanges": [ - "1d", - "5d", - "1mo", - "3mo", - "6mo", - "1y", - "2y", - "5y", - "10y", - "ytd", - "max" - ] - }, - "timestamp": [ - 1626442200, - 1626465603 - ], - "indicators": { - "quote": [ - { - "close": [ - 644.22, - 644.22 - ] - } - ] - } - } - ] - } - ], - "error": null - } -} From 5e75759b0fa8c14bc7ab88e5fd7d93e0cf3b08a2 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 14 Sep 2024 22:24:46 +0200 Subject: [PATCH 05/14] Version 1.4.10. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d1a7ce1..d5fc989 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.9" +version = "1.4.10" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 1ad354e..738cf25 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.9" +__version__ = "1.4.10" From b6f4c175303e6470b24ded7a4112b81abc40ed7c Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 14 Sep 2024 22:49:45 +0200 Subject: [PATCH 06/14] Skip coindesk live test. --- tests/live.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/live.sh b/tests/live.sh index fc24c08..cb2dbeb 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -128,7 +128,7 @@ date,base,quote,amount,source,type 2021-01-07,BTC,USD,39713.5079,coindeskbpi,close 2021-01-08,BTC,USD,40519.4486,coindeskbpi,close END -run_test "$name" "$cmd" "$expected" +skip_test "$name" "$cmd" "$expected" name="CoinMarketCap" cmd="pricehist fetch coinmarketcap BTC/EUR -s 2021-01-04 -e 2021-01-08" From ee8ca0573d73765f298a83961eb2eb1583173f48 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 12:46:43 +0200 Subject: [PATCH 07/14] yahoo: add back null handling, improve timestamp handling. Thanks @arkn98! --- src/pricehist/sources/yahoo.py | 15 +++++++++++---- tests/pricehist/sources/test_yahoo.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index ce5cf7d..65857bb 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -72,6 +72,7 @@ class Yahoo(BaseSource): data = self._data(series) quote = data["chart"]["result"][0]["meta"]["currency"] + offset = data["chart"]["result"][0]["meta"]["gmtoffset"] timestamps = data["chart"]["result"][0]["timestamp"] adjclose_data = data["chart"]["result"][0]["indicators"]["adjclose"][0] @@ -79,19 +80,25 @@ class Yahoo(BaseSource): amounts = {**adjclose_data, **rest_data} prices = [ - Price(ts, amount) + Price(date, amount) for i in range(len(timestamps)) - if (ts := datetime.fromtimestamp(timestamps[i]).strftime("%Y-%m-%d")) - <= series.end + if (date := self._date_from_ts(timestamps[i], offset)) <= series.end if (amount := self._amount(amounts, series.type, i)) is not None ] return dataclasses.replace(series, quote=quote, prices=prices) + def _date_from_ts(self, ts, offset) -> str: + return ( + datetime.fromtimestamp(ts - offset) + .replace(tzinfo=timezone.utc) + .strftime("%Y-%m-%d") + ) + def _amount(self, amounts, type, i): if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": return sum([Decimal(amounts["high"][i]), Decimal(amounts["low"][i])]) / 2 - elif amounts[type] != "null": + elif amounts[type] != "null" and amounts[type][i] is not None: return Decimal(amounts[type][i]) else: return None diff --git a/tests/pricehist/sources/test_yahoo.py b/tests/pricehist/sources/test_yahoo.py index 7e8d055..d490d86 100644 --- a/tests/pricehist/sources/test_yahoo.py +++ b/tests/pricehist/sources/test_yahoo.py @@ -54,6 +54,13 @@ def long_ok(requests_mock): yield requests_mock +@pytest.fixture +def with_null_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "inrx-with-null.json").read_text() + requests_mock.add(responses.GET, url("INR=X"), body=json, status=200) + yield requests_mock + + def test_normalizesymbol(src): assert src.normalizesymbol("tsla") == "TSLA" @@ -157,6 +164,13 @@ def test_fetch_from_before_start(src, type, long_ok): assert len(series.prices) > 9 +def test_fetch_skips_dates_with_nulls(src, type, with_null_ok): + series = src.fetch(Series("INR=X", "", type, "2017-07-10", "2017-07-12")) + assert series.prices[0] == Price("2017-07-10", Decimal("64.61170196533203125")) + assert series.prices[1] == Price("2017-07-12", Decimal("64.52559661865234375")) + assert len(series.prices) == 2 + + def test_fetch_to_future(src, type, recent_ok): series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2100-01-08")) assert len(series.prices) > 0 From 77b2776e55d0c9a6e4265bff94665f1838cba7c0 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 12:59:39 +0200 Subject: [PATCH 08/14] yahoo: More graceful handling of responses with meta but no timestamps. --- src/pricehist/sources/yahoo.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 65857bb..169bf71 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -146,11 +146,10 @@ class Yahoo(BaseSource): raise exceptions.InvalidPair( series.base, series.quote, self, "Symbol not found." ) - if code == 400 and "Data doesn't exist" in text: + elif code == 400 and "Data doesn't exist" in text: raise exceptions.BadResponse( "No data for the given interval. Try requesting a larger interval." ) - elif code == 404 and "Timestamp data missing" in text: raise exceptions.BadResponse( "Data missing. The given interval may be for a gap in the data " @@ -169,4 +168,10 @@ class Yahoo(BaseSource): "The data couldn't be parsed. " ) from e + if "timestamp" not in data["chart"]["result"][0]: + raise exceptions.BadResponse( + "No data for the given interval. " + "There may be a problem with the symbol or the interval." + ) + return data From 1164724ffb8cb2b614af06d2a1d4634cf0a2c886 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 13:01:11 +0200 Subject: [PATCH 09/14] Version 1.4.11. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d5fc989..98bb405 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.10" +version = "1.4.11" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 738cf25..e42b3cf 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.10" +__version__ = "1.4.11" From c78154df3a265da9c26a149d78bff1989fe046fd Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 13:07:50 +0200 Subject: [PATCH 10/14] Add missing file. --- .../sources/test_yahoo/inrx-with-null.json | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tests/pricehist/sources/test_yahoo/inrx-with-null.json diff --git a/tests/pricehist/sources/test_yahoo/inrx-with-null.json b/tests/pricehist/sources/test_yahoo/inrx-with-null.json new file mode 100644 index 0000000..5ae762e --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/inrx-with-null.json @@ -0,0 +1,119 @@ +{ + "chart": { + "result": [ + { + "meta": { + "currency": "INR", + "symbol": "INR=X", + "exchangeName": "CCY", + "fullExchangeName": "CCY", + "instrumentType": "CURRENCY", + "firstTradeDate": 1070236800, + "regularMarketTime": 1726284616, + "hasPrePostMarketData": false, + "gmtoffset": 3600, + "timezone": "BST", + "exchangeTimezoneName": "Europe/London", + "regularMarketPrice": 83.89, + "fiftyTwoWeekHigh": 83.89, + "fiftyTwoWeekLow": 83.89, + "regularMarketDayHigh": 83.89, + "regularMarketDayLow": 83.89, + "regularMarketVolume": 0, + "longName": "USD/INR", + "shortName": "USD/INR", + "chartPreviousClose": 64.6117, + "priceHint": 4, + "currentTradingPeriod": { + "pre": { + "timezone": "BST", + "start": 1726182000, + "end": 1726182000, + "gmtoffset": 3600 + }, + "regular": { + "timezone": "BST", + "start": 1726182000, + "end": 1726268340, + "gmtoffset": 3600 + }, + "post": { + "timezone": "BST", + "start": 1726268340, + "end": 1726268340, + "gmtoffset": 3600 + } + }, + "dataGranularity": "1d", + "range": "", + "validRanges": [ + "1d", + "5d", + "1mo", + "3mo", + "6mo", + "1y", + "2y", + "5y", + "10y", + "ytd", + "max" + ] + }, + "timestamp": [ + 1499641200, + 1499727600, + 1499814000, + 1499900400 + ], + "indicators": { + "quote": [ + { + "open": [ + 64.6155014038086, + null, + 64.55549621582031, + 64.46800231933594 + ], + "volume": [ + 0, + null, + 0, + 0 + ], + "low": [ + 64.41000366210938, + null, + 64.3499984741211, + 64.33999633789062 + ], + "close": [ + 64.61170196533203, + null, + 64.52559661865234, + 64.36499786376953 + ], + "high": [ + 64.6155014038086, + null, + 64.56999969482422, + 64.48419952392578 + ] + } + ], + "adjclose": [ + { + "adjclose": [ + 64.61170196533203, + null, + 64.52559661865234, + 64.36499786376953 + ] + } + ] + } + } + ], + "error": null + } +} From dffe6f8e89678751478382b8549b520f43296e9e Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 13:15:18 +0200 Subject: [PATCH 11/14] Timezone handling tweak. --- src/pricehist/sources/yahoo.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 169bf71..81dd0f7 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -89,11 +89,7 @@ class Yahoo(BaseSource): return dataclasses.replace(series, quote=quote, prices=prices) def _date_from_ts(self, ts, offset) -> str: - return ( - datetime.fromtimestamp(ts - offset) - .replace(tzinfo=timezone.utc) - .strftime("%Y-%m-%d") - ) + return datetime.fromtimestamp(ts - offset).strftime("%Y-%m-%d") def _amount(self, amounts, type, i): if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": From 53f39a26ef9e28565826456e65d119b6f4209d6a Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 11:35:56 +0000 Subject: [PATCH 12/14] More time correction. --- src/pricehist/sources/yahoo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 81dd0f7..25d92fe 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -82,14 +82,14 @@ class Yahoo(BaseSource): prices = [ Price(date, amount) for i in range(len(timestamps)) - if (date := self._date_from_ts(timestamps[i], offset)) <= series.end + if (date := self._ts_to_date(timestamps[i] + offset)) <= series.end if (amount := self._amount(amounts, series.type, i)) is not None ] return dataclasses.replace(series, quote=quote, prices=prices) - def _date_from_ts(self, ts, offset) -> str: - return datetime.fromtimestamp(ts - offset).strftime("%Y-%m-%d") + def _ts_to_date(self, ts) -> str: + return datetime.fromtimestamp(ts, tz=timezone.utc).date().isoformat() def _amount(self, amounts, type, i): if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": From ab507b189cdb30f4df7ead380bc0151740f249b3 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 12:16:36 +0000 Subject: [PATCH 13/14] Update live test. --- tests/live.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/live.sh b/tests/live.sh index cb2dbeb..157cdad 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -75,10 +75,10 @@ name="Alpha Vantage physical currency" cmd="pricehist fetch alphavantage AUD/EUR -s 2021-01-11 -e 2021-01-14" read -r -d '' expected < Date: Sun, 15 Sep 2024 12:17:10 +0000 Subject: [PATCH 14/14] Version 1.4.12. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98bb405..5d555aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.11" +version = "1.4.12" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index e42b3cf..2736991 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.11" +__version__ = "1.4.12"