From d506b8502efe1b4802221af3dabe07f342c538d8 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Wed, 14 Jul 2021 16:22:35 +0200 Subject: [PATCH 001/149] Fix UNIX timestamp logic to avoid timezone issues for yahoo and coinmarketcap. --- src/pricehist/sources/coinmarketcap.py | 15 ++++++++++++--- src/pricehist/sources/yahoo.py | 14 +++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/pricehist/sources/coinmarketcap.py b/src/pricehist/sources/coinmarketcap.py index 0d04b62..2d6692b 100644 --- a/src/pricehist/sources/coinmarketcap.py +++ b/src/pricehist/sources/coinmarketcap.py @@ -1,6 +1,6 @@ import dataclasses import json -from datetime import datetime +from datetime import datetime, timezone from decimal import Decimal import requests @@ -77,10 +77,19 @@ class CoinMarketCap(BaseSource): params["convert"] = series.quote params["time_start"] = int( - int(datetime.strptime(series.start, "%Y-%m-%d").timestamp()) + int( + datetime.strptime(series.start, "%Y-%m-%d") + .replace(tzinfo=timezone.utc) + .timestamp() + ) ) params["time_end"] = ( - int(datetime.strptime(series.end, "%Y-%m-%d").timestamp()) + 24 * 60 * 60 + int( + datetime.strptime(series.end, "%Y-%m-%d") + .replace(tzinfo=timezone.utc) + .timestamp() + ) + + 24 * 60 * 60 ) # round up to include the last day response = self.log_curl(requests.get(url, params=params)) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index c0b4d15..0a61cb5 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -2,7 +2,7 @@ import csv import dataclasses import json import logging -from datetime import datetime +from datetime import datetime, timezone from decimal import Decimal import requests @@ -102,8 +102,16 @@ class Yahoo(BaseSource): ) spark = json.loads(spark_response.content) - start_ts = int(datetime.strptime(series.start, "%Y-%m-%d").timestamp()) - end_ts = int(datetime.strptime(series.end, "%Y-%m-%d").timestamp()) + ( + start_ts = int( + datetime.strptime(series.start, "%Y-%m-%d") + .replace(tzinfo=timezone.utc) + .timestamp() + ) + end_ts = int( + datetime.strptime(series.end, "%Y-%m-%d") + .replace(tzinfo=timezone.utc) + .timestamp() + ) + ( 24 * 60 * 60 ) # round up to include the last day From fadeee4870dbce7faca88f29151e305df2f9694b Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Wed, 14 Jul 2021 16:24:32 +0200 Subject: [PATCH 002/149] Version 0.1.4. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bf385df..ceae949 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "0.1.3" +version = "0.1.4" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index ae73625..bbab024 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "0.1.3" +__version__ = "0.1.4" From 73c5f3a1c6dadb26423a625396305ad8a44e2524 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Wed, 14 Jul 2021 16:58:30 +0200 Subject: [PATCH 003/149] Tidy tests for coindesk, ecb. --- tests/pricehist/sources/test_coindesk.py | 141 ++++++----------------- tests/pricehist/sources/test_ecb.py | 69 ++++------- 2 files changed, 63 insertions(+), 147 deletions(-) diff --git a/tests/pricehist/sources/test_coindesk.py b/tests/pricehist/sources/test_coindesk.py index 98d3d6f..6987c83 100644 --- a/tests/pricehist/sources/test_coindesk.py +++ b/tests/pricehist/sources/test_coindesk.py @@ -35,6 +35,11 @@ def currencies_url(): return "https://api.coindesk.com/v1/bpi/supported-currencies.json" +@pytest.fixture +def fetch_url(): + return "https://api.coindesk.com/v1/bpi/historical/close.json" + + @pytest.fixture def currencies_json(): dir = Path(os.path.splitext(__file__)[0]) @@ -43,54 +48,29 @@ def currencies_json(): @pytest.fixture def currencies_response_ok(requests_mock, currencies_url, currencies_json): - requests_mock.add( - responses.GET, - currencies_url, - body=currencies_json, - status=200, - ) + requests_mock.add(responses.GET, currencies_url, body=currencies_json, status=200) yield requests_mock @pytest.fixture -def recent_json(): - dir = Path(os.path.splitext(__file__)[0]) - return (dir / "recent.json").read_text() - - -@pytest.fixture -def recent_response_ok(requests_mock, recent_json): - requests_mock.add( - responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", - body=recent_json, - status=200, - ) +def recent_response_ok(requests_mock, fetch_url): + json = (Path(os.path.splitext(__file__)[0]) / "recent.json").read_text() + requests_mock.add(responses.GET, fetch_url, body=json, status=200) yield requests_mock @pytest.fixture -def all_json(): - dir = Path(os.path.splitext(__file__)[0]) - return (dir / "all-partial.json").read_text() - - -@pytest.fixture -def all_response_ok(requests_mock, all_json): - requests_mock.add( - responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", - body=all_json, - status=200, - ) +def all_response_ok(requests_mock, fetch_url): + json = (Path(os.path.splitext(__file__)[0]) / "all-partial.json").read_text() + requests_mock.add(responses.GET, fetch_url, body=json, status=200) yield requests_mock @pytest.fixture -def not_found_response(requests_mock): +def not_found_response(requests_mock, fetch_url): requests_mock.add( responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", + fetch_url, status=404, body="Sorry, that currency was not found", ) @@ -139,12 +119,7 @@ def test_symbols_requests_logged(src, currencies_response_ok, caplog): def test_symbols_not_found(src, requests_mock, currencies_url): - requests_mock.add( - responses.GET, - currencies_url, - body="[]", - status=200, - ) + requests_mock.add(responses.GET, currencies_url, body="[]", status=200) with pytest.raises(exceptions.ResponseParsingError) as e: src.symbols() assert "data not found" in str(e.value) @@ -162,22 +137,14 @@ def test_symbols_network_issue(src, requests_mock, currencies_url): def test_symbols_bad_status(src, requests_mock, currencies_url): - requests_mock.add( - responses.GET, - currencies_url, - status=500, - ) + requests_mock.add(responses.GET, currencies_url, status=500) with pytest.raises(exceptions.BadResponse) as e: src.symbols() assert "Server Error" in str(e.value) def test_symbols_parsing_error(src, requests_mock, currencies_url): - requests_mock.add( - responses.GET, - currencies_url, - body="NOT JSON", - ) + requests_mock.add(responses.GET, currencies_url, body="NOT JSON") with pytest.raises(exceptions.ResponseParsingError) as e: src.symbols() assert "while parsing data" in str(e.value) @@ -209,13 +176,9 @@ def test_fetch_long_hist_from_start(src, type, all_response_ok): assert len(series.prices) > 13 -def test_fetch_from_before_start(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", - status=404, - body="Sorry, the CoinDesk BPI only covers data from 2010-07-17 onwards.", - ) +def test_fetch_from_before_start(src, type, requests_mock, fetch_url): + body = "Sorry, the CoinDesk BPI only covers data from 2010-07-17 onwards." + requests_mock.add(responses.GET, fetch_url, status=404, body=body) with pytest.raises(exceptions.BadResponse) as e: src.fetch(Series("BTC", "AUD", type, "2010-01-01", "2010-07-24")) assert "only covers data from" in str(e.value) @@ -226,47 +189,31 @@ def test_fetch_to_future(src, type, all_response_ok): assert len(series.prices) > 0 -def test_wrong_dates_order(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", - status=404, - body="Sorry, but your specified end date is before your start date.", - ) +def test_wrong_dates_order(src, type, requests_mock, fetch_url): + body = "Sorry, but your specified end date is before your start date." + requests_mock.add(responses.GET, fetch_url, status=404, body=body) with pytest.raises(exceptions.BadResponse) as e: src.fetch(Series("BTC", "AUD", type, "2021-01-07", "2021-01-01")) assert "End date is before start date." in str(e.value) -def test_fetch_in_future(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", - status=404, - body="Sorry, but your specified end date is before your start date.", - ) +def test_fetch_in_future(src, type, requests_mock, fetch_url): + body = "Sorry, but your specified end date is before your start date." + requests_mock.add(responses.GET, fetch_url, status=404, body=body) with pytest.raises(exceptions.BadResponse) as e: src.fetch(Series("BTC", "AUD", type, "2030-01-01", "2030-01-07")) assert "start date must be in the past" in str(e.value) -def test_fetch_empty(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", - body="{}", - ) +def test_fetch_empty(src, type, requests_mock, fetch_url): + requests_mock.add(responses.GET, fetch_url, body="{}") series = src.fetch(Series("BTC", "AUD", type, "2010-07-17", "2010-07-17")) assert len(series.prices) == 0 -def test_fetch_known_pair_no_data(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", - status=500, - body="No results returned from database", - ) +def test_fetch_known_pair_no_data(src, type, requests_mock, fetch_url): + body = "No results returned from database" + requests_mock.add(responses.GET, fetch_url, status=500, body=body) with pytest.raises(exceptions.BadResponse) as e: src.fetch(Series("BTC", "CUP", type, "2010-07-17", "2010-07-23")) assert "No results returned from database" in str(e.value) @@ -292,35 +239,23 @@ def test_fetch_unknown_pair(src, type): src.fetch(Series("ABC", "XZY", type, "2021-01-01", "2021-01-07")) -def test_fetch_network_issue(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", - body=requests.exceptions.ConnectionError("Network issue"), - ) +def test_fetch_network_issue(src, type, requests_mock, fetch_url): + body = requests.exceptions.ConnectionError("Network issue") + requests_mock.add(responses.GET, fetch_url, body=body) with pytest.raises(exceptions.RequestError) as e: src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) assert "Network issue" in str(e.value) -def test_fetch_bad_status(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", - status=500, - body="Some other reason", - ) +def test_fetch_bad_status(src, type, requests_mock, fetch_url): + requests_mock.add(responses.GET, fetch_url, status=500, body="Some other reason") with pytest.raises(exceptions.BadResponse) as e: src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) assert "Internal Server Error" in str(e.value) -def test_fetch_parsing_error(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://api.coindesk.com/v1/bpi/historical/close.json", - body="NOT JSON", - ) +def test_fetch_parsing_error(src, type, requests_mock, fetch_url): + requests_mock.add(responses.GET, fetch_url, body="NOT JSON") with pytest.raises(exceptions.ResponseParsingError) as e: src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) assert "while parsing data" in str(e.value) diff --git a/tests/pricehist/sources/test_ecb.py b/tests/pricehist/sources/test_ecb.py index c68db11..2d747fe 100644 --- a/tests/pricehist/sources/test_ecb.py +++ b/tests/pricehist/sources/test_ecb.py @@ -25,15 +25,19 @@ def type(src): @pytest.fixture -def xml(): - dir = Path(os.path.splitext(__file__)[0]) - return (dir / "eurofxref-hist-partial.xml").read_text() +def url(): + return "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml" @pytest.fixture -def empty_xml(): +def url_90d(): + return "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml" + + +@pytest.fixture +def xml(): dir = Path(os.path.splitext(__file__)[0]) - return (dir / "eurofxref-hist-empty.xml").read_text() + return (dir / "eurofxref-hist-partial.xml").read_text() @pytest.fixture @@ -43,35 +47,23 @@ def requests_mock(): @pytest.fixture -def response_ok(requests_mock, xml): - requests_mock.add( - responses.GET, - "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml", - body=xml, - status=200, - ) +def response_ok(requests_mock, url, xml): + requests_mock.add(responses.GET, url, body=xml, status=200) yield requests_mock @pytest.fixture -def response_ok_90d(requests_mock, xml): - requests_mock.add( - responses.GET, - "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml", - body=xml, - status=200, - ) +def response_ok_90d(requests_mock, url_90d, xml): + requests_mock.add(responses.GET, url_90d, body=xml, status=200) yield requests_mock @pytest.fixture -def response_empty_xml(requests_mock, empty_xml): - requests_mock.add( - responses.GET, - "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml", - body=empty_xml, - status=200, - ) +def response_empty_xml(requests_mock, url): + empty_xml = ( + Path(os.path.splitext(__file__)[0]) / "eurofxref-hist-empty.xml" + ).read_text() + requests_mock.add(responses.GET, url, body=empty_xml, status=200) yield requests_mock @@ -198,34 +190,23 @@ def test_fetch_unknown_pair(src, type): src.fetch(Series("ABC", "XZY", type, "2021-01-04", "2021-01-08")) -def test_fetch_network_issue(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml", - body=requests.exceptions.ConnectionError("Network issue"), - ) +def test_fetch_network_issue(src, type, requests_mock, url): + err = requests.exceptions.ConnectionError("Network issue") + requests_mock.add(responses.GET, url, body=err) with pytest.raises(exceptions.RequestError) as e: src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) assert "Network issue" in str(e.value) -def test_fetch_bad_status(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml", - status=500, - ) +def test_fetch_bad_status(src, type, requests_mock, url): + requests_mock.add(responses.GET, url, status=500) with pytest.raises(exceptions.BadResponse) as e: src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) assert "Server Error" in str(e.value) -def test_fetch_parsing_error(src, type, requests_mock): - requests_mock.add( - responses.GET, - "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml", - body="NOT XML", - ) +def test_fetch_parsing_error(src, type, requests_mock, url): + requests_mock.add(responses.GET, url, body="NOT XML") with pytest.raises(exceptions.ResponseParsingError) as e: src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) assert "while parsing data" in str(e.value) From b00bca39039b0eab283b0d5499b3a31a89ce1e9d Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 16 Jul 2021 11:52:57 +0200 Subject: [PATCH 004/149] Allow --verbose or -vvv for any subcommand as well as at the top level. --- src/pricehist/cli.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/pricehist/cli.py b/src/pricehist/cli.py index c5777b1..d845bf0 100644 --- a/src/pricehist/cli.py +++ b/src/pricehist/cli.py @@ -130,6 +130,7 @@ def build_parser(): ) parser.add_argument( + "-vvv", "--verbose", action="store_true", help="show all log messages", @@ -137,11 +138,17 @@ def build_parser(): subparsers = parser.add_subparsers(title="commands", dest="command") - subparsers.add_parser( + sources_parser = subparsers.add_parser( "sources", help="list sources", formatter_class=formatter, ) + sources_parser.add_argument( + "-vvv", + "--verbose", + action="store_true", + help="show all log messages", + ) source_parser = subparsers.add_parser( "source", @@ -156,6 +163,12 @@ def build_parser(): choices=sources.by_id.keys(), help="the source identifier", ) + source_parser.add_argument( + "-vvv", + "--verbose", + action="store_true", + help="show all log messages", + ) source_list_or_search = source_parser.add_mutually_exclusive_group(required=False) source_list_or_search.add_argument( @@ -177,7 +190,7 @@ def build_parser(): usage=( # Set usage manually to have positional arguments before options # and show allowed values where appropriate - "pricehist fetch SOURCE PAIR [-h] " + "pricehist fetch SOURCE PAIR [-h] [-vvv] " "[-t TYPE] [-s DATE | -sx DATE] [-e DATE | -ex DATE] " f"[-o {'|'.join(outputs.by_type.keys())}] " "[--invert] [--quantize INT] " @@ -201,6 +214,12 @@ def build_parser(): type=valid_pair, help="pair, usually BASE/QUOTE, e.g. BTC/USD", ) + fetch_parser.add_argument( + "-vvv", + "--verbose", + action="store_true", + help="show all log messages", + ) fetch_parser.add_argument( "-t", "--type", From 881f3b2acfa6d642fb2d915f4e4a5d26d12ea09e Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 17 Jul 2021 18:40:21 +0200 Subject: [PATCH 005/149] Minor rewording of error message. --- src/pricehist/sources/coindesk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pricehist/sources/coindesk.py b/src/pricehist/sources/coindesk.py index a7e751e..0170a99 100644 --- a/src/pricehist/sources/coindesk.py +++ b/src/pricehist/sources/coindesk.py @@ -105,7 +105,7 @@ class CoinDesk(BaseSource): raise exceptions.BadResponse( "No results returned from database. This can happen when data " "for a valid quote currency (e.g. CUP) doesn't go all the way " - "back to the start date, or potentially for other reasons." + "back to the start date, and potentially for other reasons." ) else: try: From d1704615df3e9fc6ec71c6df1e44fdcc58f3aede Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 17 Jul 2021 18:39:59 +0200 Subject: [PATCH 006/149] Improve coinmarketcap error handling and add tests. --- src/pricehist/exceptions.py | 10 +- src/pricehist/sources/coinmarketcap.py | 130 +++++- tests/pricehist/sources/test_coinmarketcap.py | 430 ++++++++++++++++++ .../test_coinmarketcap/crypto-partial.json | 34 ++ .../test_coinmarketcap/fiat-partial.json | 30 ++ .../long-btc-aud-partial.json | 255 +++++++++++ .../test_coinmarketcap/recent-btc-aud.json | 136 ++++++ .../test_coinmarketcap/recent-btc-id2782.json | 136 ++++++ .../test_coinmarketcap/recent-id1-aud.json | 136 ++++++ .../test_coinmarketcap/recent-id1-id2782.json | 136 ++++++ 10 files changed, 1406 insertions(+), 27 deletions(-) create mode 100644 tests/pricehist/sources/test_coinmarketcap.py create mode 100644 tests/pricehist/sources/test_coinmarketcap/crypto-partial.json create mode 100644 tests/pricehist/sources/test_coinmarketcap/fiat-partial.json create mode 100644 tests/pricehist/sources/test_coinmarketcap/long-btc-aud-partial.json create mode 100644 tests/pricehist/sources/test_coinmarketcap/recent-btc-aud.json create mode 100644 tests/pricehist/sources/test_coinmarketcap/recent-btc-id2782.json create mode 100644 tests/pricehist/sources/test_coinmarketcap/recent-id1-aud.json create mode 100644 tests/pricehist/sources/test_coinmarketcap/recent-id1-id2782.json diff --git a/src/pricehist/exceptions.py b/src/pricehist/exceptions.py index e393b97..1e0f0eb 100644 --- a/src/pricehist/exceptions.py +++ b/src/pricehist/exceptions.py @@ -5,17 +5,19 @@ class SourceError(Exception): class InvalidPair(SourceError, ValueError): """An invalid pair was requested.""" - def __init__(self, base, quote, source): + def __init__(self, base, quote, source, message=None): self.base = base self.quote = quote self.source = source pair = "/".join([base, quote]) - message = ( - f"Invalid pair '{pair}'. " + insert = message + " " if message else "" + + full_message = ( + f"Invalid pair '{pair}'. {insert}" f"Run 'pricehist source {source.id()} --symbols' " f"for information about valid pairs." ) - super(InvalidPair, self).__init__(message) + super(InvalidPair, self).__init__(full_message) class InvalidType(SourceError, ValueError): diff --git a/src/pricehist/sources/coinmarketcap.py b/src/pricehist/sources/coinmarketcap.py index 2d6692b..313c493 100644 --- a/src/pricehist/sources/coinmarketcap.py +++ b/src/pricehist/sources/coinmarketcap.py @@ -5,6 +5,7 @@ from decimal import Decimal import requests +from pricehist import exceptions from pricehist.price import Price from .basesource import BaseSource @@ -47,10 +48,13 @@ class CoinMarketCap(BaseSource): return list(zip(ids, descriptions)) def fetch(self, series): + if series.base == "ID=" or not series.quote or series.quote == "ID=": + raise exceptions.InvalidPair(series.base, series.quote, self) + data = self._data(series) prices = [] - for item in data["data"]["quotes"]: + for item in data.get("quotes", []): d = item["time_open"][0:10] amount = self._amount(next(iter(item["quote"].values())), series.type) prices.append(Price(d, amount)) @@ -82,19 +86,73 @@ class CoinMarketCap(BaseSource): .replace(tzinfo=timezone.utc) .timestamp() ) + - 24 * 60 * 60 + # Start one period earlier since the start is exclusive. ) - params["time_end"] = ( - int( - datetime.strptime(series.end, "%Y-%m-%d") - .replace(tzinfo=timezone.utc) - .timestamp() + params["time_end"] = int( + datetime.strptime(series.end, "%Y-%m-%d") + .replace(tzinfo=timezone.utc) + .timestamp() + ) # Don't round up since it's inclusive of the period covering the end time. + + try: + response = self.log_curl(requests.get(url, params=params)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + code = response.status_code + text = response.text + + if code == 400 and "No items found." in text: + raise exceptions.InvalidPair( + series.base, series.quote, self, "Bad base ID." ) - + 24 * 60 * 60 - ) # round up to include the last day - response = self.log_curl(requests.get(url, params=params)) + elif code == 400 and 'Invalid value for \\"convert_id\\"' in text: + raise exceptions.InvalidPair( + series.base, series.quote, self, "Bad quote ID." + ) - return json.loads(response.content) + elif code == 400 and 'Invalid value for \\"convert\\"' in text: + raise exceptions.InvalidPair( + series.base, series.quote, self, "Bad quote symbol." + ) + + elif code == 400 and "must be older than" in text: + if series.start <= series.end: + raise exceptions.BadResponse("The start date must be in the past.") + else: + raise exceptions.BadResponse( + "The start date must preceed or match the end date." + ) + + elif ( + code == 400 + and "must be a valid ISO 8601 timestamp or unix time" in text + and series.start < "2001-09-11" + ): + raise exceptions.BadResponse("The start date can't preceed 2001-09-11.") + + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + parsed = json.loads(response.content) + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if type(parsed) != dict or "data" not in parsed: + raise exceptions.ResponseParsingError("Unexpected content.") + + elif len(parsed["data"]) == 0: + raise exceptions.ResponseParsingError( + "The data section was empty. This can happen when the quote " + "currency symbol can't be found, and potentially for other reasons." + ) + + return parsed["data"] def _amount(self, data, type): if type in ["mid"]: @@ -105,26 +163,52 @@ class CoinMarketCap(BaseSource): return Decimal(str(data[type])) def _output_pair(self, base, quote, data): - data_base = data["data"]["symbol"] - data_quote = next(iter(data["data"]["quotes"][0]["quote"].keys())) + data_base = data["symbol"] - lookup_quote = False + data_quote = None + if len(data["quotes"]) > 0: + data_quote = next(iter(data["quotes"][0]["quote"].keys())) + + lookup_quote = None if quote.startswith("ID="): symbols = {i["id"]: (i["symbol"] or i["code"]) for i in self._symbol_data()} lookup_quote = symbols[int(quote[3:])] output_base = data_base - output_quote = lookup_quote or data_quote + output_quote = lookup_quote or data_quote or quote return (output_base, output_quote) def _symbol_data(self): - fiat_url = "https://web-api.coinmarketcap.com/v1/fiat/map?include_metals=true" - fiat_res = self.log_curl(requests.get(fiat_url)) - fiat = json.loads(fiat_res.content) - crypto_url = ( - "https://web-api.coinmarketcap.com/v1/cryptocurrency/map?sort=cmc_rank" - ) - crypto_res = self.log_curl(requests.get(crypto_url)) - crypto = json.loads(crypto_res.content) - return crypto["data"] + fiat["data"] + base_url = "https://web-api.coinmarketcap.com/v1/" + fiat_url = f"{base_url}fiat/map?include_metals=true" + crypto_url = f"{base_url}cryptocurrency/map?sort=cmc_rank" + + fiat = self._get_json_data(fiat_url) + crypto = self._get_json_data(crypto_url) + + return crypto + fiat + + def _get_json_data(self, url, params={}): + try: + response = self.log_curl(requests.get(url, params=params)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + parsed = json.loads(response.content) + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if type(parsed) != dict or "data" not in parsed: + raise exceptions.ResponseParsingError("Unexpected content.") + + elif len(parsed["data"]) == 0: + raise exceptions.ResponseParsingError("Empty data section.") + + return parsed["data"] diff --git a/tests/pricehist/sources/test_coinmarketcap.py b/tests/pricehist/sources/test_coinmarketcap.py new file mode 100644 index 0000000..6ae175c --- /dev/null +++ b/tests/pricehist/sources/test_coinmarketcap.py @@ -0,0 +1,430 @@ +import logging +import os +from datetime import datetime, timezone +from decimal import Decimal +from pathlib import Path + +import pytest +import requests +import responses + +from pricehist import exceptions +from pricehist.price import Price +from pricehist.series import Series +from pricehist.sources.coinmarketcap import CoinMarketCap + + +def timestamp(date): + return int( + datetime.strptime(date, "%Y-%m-%d").replace(tzinfo=timezone.utc).timestamp() + ) + + +@pytest.fixture +def src(): + return CoinMarketCap() + + +@pytest.fixture +def type(src): + return src.types()[0] + + +@pytest.fixture +def requests_mock(): + with responses.RequestsMock() as mock: + yield mock + + +crypto_url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/map?sort=cmc_rank" +fiat_url = "https://web-api.coinmarketcap.com/v1/fiat/map?include_metals=true" +fetch_url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical" + + +@pytest.fixture +def crypto_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "crypto-partial.json").read_text() + requests_mock.add(responses.GET, crypto_url, body=json, status=200) + yield requests_mock + + +@pytest.fixture +def fiat_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "fiat-partial.json").read_text() + requests_mock.add(responses.GET, fiat_url, body=json, status=200) + yield requests_mock + + +@pytest.fixture +def recent_id_id_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "recent-id1-id2782.json").read_text() + requests_mock.add(responses.GET, fetch_url, body=json, status=200) + yield requests_mock + + +@pytest.fixture +def recent_id_sym_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "recent-id1-aud.json").read_text() + requests_mock.add(responses.GET, fetch_url, body=json, status=200) + yield requests_mock + + +@pytest.fixture +def recent_sym_id_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "recent-btc-id2782.json").read_text() + requests_mock.add(responses.GET, fetch_url, body=json, status=200) + yield requests_mock + + +@pytest.fixture +def recent_sym_sym_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "recent-btc-aud.json").read_text() + requests_mock.add(responses.GET, fetch_url, body=json, status=200) + yield requests_mock + + +@pytest.fixture +def long_sym_sym_ok(requests_mock): + json = ( + Path(os.path.splitext(__file__)[0]) / "long-btc-aud-partial.json" + ).read_text() + requests_mock.add(responses.GET, fetch_url, body=json, status=200) + yield requests_mock + + +def test_normalizesymbol(src): + assert src.normalizesymbol("btc") == "BTC" + assert src.normalizesymbol("id=1") == "ID=1" + + +def test_metadata(src): + assert isinstance(src.id(), str) + assert len(src.id()) > 0 + + assert isinstance(src.name(), str) + assert len(src.name()) > 0 + + assert isinstance(src.description(), str) + assert len(src.description()) > 0 + + assert isinstance(src.source_url(), str) + assert src.source_url().startswith("http") + + assert datetime.strptime(src.start(), "%Y-%m-%d") + + assert isinstance(src.types(), list) + assert len(src.types()) > 0 + assert isinstance(src.types()[0], str) + assert len(src.types()[0]) > 0 + + assert isinstance(src.notes(), str) + + +def test_symbols(src, crypto_ok, fiat_ok): + syms = src.symbols() + assert ("id=1", "BTC Bitcoin") in syms + assert ("id=2782", "AUD Australian Dollar") in syms + assert len(syms) > 2 + + +def test_symbols_requests_logged(src, crypto_ok, fiat_ok, caplog): + with caplog.at_level(logging.DEBUG): + src.symbols() + logged_requests = 0 + for r in caplog.records: + if r.levelname == "DEBUG" and " curl " in r.message: + logged_requests += 1 + assert logged_requests == 2 + + +def test_symbols_fiat_not_found(src, requests_mock): + requests_mock.add(responses.GET, fiat_url, body="{}", status=200) + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "Unexpected content" in str(e.value) + + +def test_symbols_fiat_network_issue(src, requests_mock): + requests_mock.add( + responses.GET, + fiat_url, + body=requests.exceptions.ConnectionError("Network issue"), + ) + with pytest.raises(exceptions.RequestError) as e: + src.symbols() + assert "Network issue" in str(e.value) + + +def test_symbols_fiat_bad_status(src, requests_mock): + requests_mock.add(responses.GET, fiat_url, status=500) + with pytest.raises(exceptions.BadResponse) as e: + src.symbols() + assert "Server Error" in str(e.value) + + +def test_symbols_fiat_parsing_error(src, requests_mock): + requests_mock.add(responses.GET, fiat_url, body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "while parsing data" in str(e.value) + + +def test_symbols_crypto_not_found(src, requests_mock, fiat_ok): + requests_mock.add(responses.GET, crypto_url, body="{}", status=200) + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "Unexpected content" in str(e.value) + + +def test_symbols_crypto_network_issue(src, requests_mock, fiat_ok): + requests_mock.add( + responses.GET, + crypto_url, + body=requests.exceptions.ConnectionError("Network issue"), + ) + with pytest.raises(exceptions.RequestError) as e: + src.symbols() + assert "Network issue" in str(e.value) + + +def test_symbols_crypto_bad_status(src, requests_mock, fiat_ok): + requests_mock.add(responses.GET, crypto_url, status=500) + with pytest.raises(exceptions.BadResponse) as e: + src.symbols() + assert "Server Error" in str(e.value) + + +def test_symbols_crypto_parsing_error(src, requests_mock, fiat_ok): + requests_mock.add(responses.GET, crypto_url, body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "while parsing data" in str(e.value) + + +def test_fetch_known_pair_id_id(src, type, recent_id_id_ok, crypto_ok, fiat_ok): + series = src.fetch(Series("ID=1", "ID=2782", type, "2021-01-01", "2021-01-07")) + req = recent_id_id_ok.calls[0].request + assert req.params["id"] == "1" + assert req.params["convert_id"] == "2782" + assert (series.base, series.quote) == ("BTC", "AUD") + assert len(series.prices) == 7 + + +def test_fetch_known_pair_id_sym(src, type, recent_id_sym_ok): + series = src.fetch(Series("ID=1", "AUD", type, "2021-01-01", "2021-01-07")) + req = recent_id_sym_ok.calls[0].request + assert req.params["id"] == "1" + assert req.params["convert"] == "AUD" + assert (series.base, series.quote) == ("BTC", "AUD") + assert len(series.prices) == 7 + + +def test_fetch_known_pair_sym_id(src, type, recent_sym_id_ok, crypto_ok, fiat_ok): + series = src.fetch(Series("BTC", "ID=2782", type, "2021-01-01", "2021-01-07")) + req = recent_sym_id_ok.calls[0].request + assert req.params["symbol"] == "BTC" + assert req.params["convert_id"] == "2782" + assert (series.base, series.quote) == ("BTC", "AUD") + assert len(series.prices) == 7 + + +def test_fetch_known_pair_sym_sym(src, type, recent_sym_sym_ok): + series = src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) + req = recent_sym_sym_ok.calls[0].request + assert req.params["symbol"] == "BTC" + assert req.params["convert"] == "AUD" + assert len(series.prices) == 7 + + +def test_fetch_requests_and_receives_correct_times( + src, type, recent_id_id_ok, crypto_ok, fiat_ok +): + series = src.fetch(Series("ID=1", "ID=2782", type, "2021-01-01", "2021-01-07")) + req = recent_id_id_ok.calls[0].request + assert req.params["time_start"] == str(timestamp("2020-12-31")) # back one period + assert req.params["time_end"] == str(timestamp("2021-01-07")) + assert series.prices[0] == Price("2021-01-01", Decimal("37914.350602379853")) + assert series.prices[-1] == Price("2021-01-07", Decimal("49370.064689585612")) + + +def test_fetch_requests_logged(src, type, recent_sym_sym_ok, caplog): + with caplog.at_level(logging.DEBUG): + src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) + assert any( + ["DEBUG" == r.levelname and " curl " in r.message for r in caplog.records] + ) + + +def test_fetch_types_all_available(src, recent_sym_sym_ok): + mid = src.fetch(Series("BTC", "AUD", "mid", "2021-01-01", "2021-01-07")) + opn = src.fetch(Series("BTC", "AUD", "open", "2021-01-01", "2021-01-07")) + hgh = src.fetch(Series("BTC", "AUD", "high", "2021-01-01", "2021-01-07")) + low = src.fetch(Series("BTC", "AUD", "low", "2021-01-01", "2021-01-07")) + cls = src.fetch(Series("BTC", "AUD", "close", "2021-01-01", "2021-01-07")) + assert mid.prices[0].amount == Decimal("37914.350602379853") + assert opn.prices[0].amount == Decimal("37658.83948707033") + assert hgh.prices[0].amount == Decimal("38417.9137031205") + assert low.prices[0].amount == Decimal("37410.787501639206") + assert cls.prices[0].amount == Decimal("38181.99133300758") + + +def test_fetch_type_mid_is_mean_of_low_and_high(src, recent_sym_sym_ok): + mid = src.fetch(Series("BTC", "AUD", "mid", "2021-01-01", "2021-01-07")).prices + low = src.fetch(Series("BTC", "AUD", "low", "2021-01-01", "2021-01-07")).prices + hgh = src.fetch(Series("BTC", "AUD", "high", "2021-01-01", "2021-01-07")).prices + assert all( + [ + mid[i].amount == (sum([low[i].amount, hgh[i].amount]) / 2) + for i in range(0, 7) + ] + ) + + +def test_fetch_long_hist_from_start(src, type, long_sym_sym_ok): + series = src.fetch(Series("BTC", "AUD", type, src.start(), "2021-01-07")) + assert series.prices[0] == Price("2013-04-28", Decimal("130.45956234123247")) + assert series.prices[-1] == Price("2021-01-07", Decimal("49370.064689585612")) + assert len(series.prices) > 13 + + +def test_fetch_from_before_start(src, type, requests_mock): + requests_mock.add( + responses.GET, + fetch_url, + status=400, + body="""{ "status": { "error_code": 400, "error_message": + "\\"time_start\\" must be a valid ISO 8601 timestamp or unix time value", + } }""", + ) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("BTC", "AUD", type, "2001-09-10", "2001-10-01")) + assert "start date can't preceed" in str(e.value) + + +def test_fetch_to_future(src, type, recent_sym_sym_ok): + series = src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2100-01-01")) + assert len(series.prices) > 0 + + +def test_fetch_in_future(src, type, requests_mock): + requests_mock.add( + responses.GET, + fetch_url, + status=400, + body="""{ + "status": { + "error_code": 400, + "error_message": "\\"time_start\\" must be older than \\"time_end\\"." + } + }""", + ) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("BTC", "AUD", type, "2030-01-01", "2030-01-07")) + assert "start date must be in the past" in str(e.value) + + +def test_fetch_empty(src, type, requests_mock): + requests_mock.add( + responses.GET, + fetch_url, + body="""{ + "status": { + "error_code": 0, + "error_message": null + }, + "data": { + "id": 1, + "name": "Bitcoin", + "symbol": "BTC", + "quotes": [] + } + }""", + ) + series = src.fetch(Series("BTC", "AUD", type, "2010-01-01", "2010-01-07")) + assert len(series.prices) == 0 + + +def test_fetch_bad_base_sym(src, type, requests_mock): + requests_mock.add(responses.GET, fetch_url, body='{"data":{}}') + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("NOTABASE", "USD", type, "2021-01-01", "2021-01-07")) + assert "quote currency symbol can't be found" in str(e.value) + assert "other reasons" in str(e.value) + + +def test_fetch_bad_quote_sym(src, type, requests_mock): + requests_mock.add( + responses.GET, + fetch_url, + status=400, + body="""{ + "status": { + "error_code": 400, + "error_message": "Invalid value for \\"convert\\": \\"NOTAQUOTE\\"" + } + }""", + ) + with pytest.raises(exceptions.InvalidPair) as e: + src.fetch(Series("BTC", "NOTAQUOTE", type, "2021-01-01", "2021-01-07")) + assert "Bad quote symbol" in str(e.value) + + +def test_fetch_bad_base_id(src, type, requests_mock): + requests_mock.add( + responses.GET, + fetch_url, + status=400, + body="""{ + "status": { + "error_code": 400, + "error_message": "No items found." + } + }""", + ) + with pytest.raises(exceptions.InvalidPair) as e: + src.fetch(Series("ID=20000", "USD", type, "2021-01-01", "2021-01-07")) + assert "Bad base ID" in str(e.value) + + +def test_fetch_bad_quote_id(src, type, requests_mock): + requests_mock.add( + responses.GET, + fetch_url, + status=400, + body="""{ + "status": { + "error_code": 400, + "error_message": "Invalid value for \\"convert_id\\": \\"20000\\"" + } + }""", + ) + with pytest.raises(exceptions.InvalidPair) as e: + src.fetch(Series("BTC", "ID=20000", type, "2021-01-01", "2021-01-07")) + assert "Bad quote ID" in str(e.value) + + +def test_fetch_no_quote(src, type): + with pytest.raises(exceptions.InvalidPair): + src.fetch(Series("BTC", "", type, "2021-01-01", "2021-01-07")) + + +def test_fetch_network_issue(src, type, requests_mock): + body = requests.exceptions.ConnectionError("Network issue") + requests_mock.add(responses.GET, fetch_url, body=body) + with pytest.raises(exceptions.RequestError) as e: + src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) + assert "Network issue" in str(e.value) + + +def test_fetch_bad_status(src, type, requests_mock): + requests_mock.add(responses.GET, fetch_url, status=500, body="Some other reason") + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) + assert "Internal Server Error" in str(e.value) + + +def test_fetch_parsing_error(src, type, requests_mock): + requests_mock.add(responses.GET, fetch_url, body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) + assert "while parsing data" in str(e.value) diff --git a/tests/pricehist/sources/test_coinmarketcap/crypto-partial.json b/tests/pricehist/sources/test_coinmarketcap/crypto-partial.json new file mode 100644 index 0000000..c1a27b5 --- /dev/null +++ b/tests/pricehist/sources/test_coinmarketcap/crypto-partial.json @@ -0,0 +1,34 @@ +{ + "status": { + "timestamp": "2021-07-16T10:08:28.938Z", + "error_code": 0, + "error_message": null, + "elapsed": 18, + "credit_count": 0, + "notice": null + }, + "data": [ + { + "id": 1, + "name": "Bitcoin", + "symbol": "BTC", + "slug": "bitcoin", + "rank": 1, + "is_active": 1, + "first_historical_data": "2013-04-28T18:47:21.000Z", + "last_historical_data": "2021-07-16T09:59:03.000Z", + "platform": null + }, + { + "id": 1027, + "name": "Ethereum", + "symbol": "ETH", + "slug": "ethereum", + "rank": 2, + "is_active": 1, + "first_historical_data": "2015-08-07T14:49:30.000Z", + "last_historical_data": "2021-07-16T09:59:04.000Z", + "platform": null + } + ] +} diff --git a/tests/pricehist/sources/test_coinmarketcap/fiat-partial.json b/tests/pricehist/sources/test_coinmarketcap/fiat-partial.json new file mode 100644 index 0000000..781824b --- /dev/null +++ b/tests/pricehist/sources/test_coinmarketcap/fiat-partial.json @@ -0,0 +1,30 @@ +{ + "status": { + "timestamp": "2021-07-16T10:08:13.272Z", + "error_code": 0, + "error_message": null, + "elapsed": 1, + "credit_count": 0, + "notice": null + }, + "data": [ + { + "id": 2781, + "name": "United States Dollar", + "sign": "$", + "symbol": "USD" + }, + { + "id": 2782, + "name": "Australian Dollar", + "sign": "$", + "symbol": "AUD" + }, + { + "id": 3575, + "name": "Gold Troy Ounce", + "symbol": "", + "code": "XAU" + } + ] +} diff --git a/tests/pricehist/sources/test_coinmarketcap/long-btc-aud-partial.json b/tests/pricehist/sources/test_coinmarketcap/long-btc-aud-partial.json new file mode 100644 index 0000000..0b11696 --- /dev/null +++ b/tests/pricehist/sources/test_coinmarketcap/long-btc-aud-partial.json @@ -0,0 +1,255 @@ +{ + "status": { + "timestamp": "2021-07-17T16:16:11.926Z", + "error_code": 0, + "error_message": null, + "elapsed": 2262, + "credit_count": 0, + "notice": null + }, + "data": { + "id": 1, + "name": "Bitcoin", + "symbol": "BTC", + "quotes": [ + { + "time_open": "2013-04-28T00:00:00.000Z", + "time_close": "2013-04-28T23:59:59.999Z", + "time_high": "2013-04-28T18:50:02.000Z", + "time_low": "2013-04-28T20:15:02.000Z", + "quote": { + "AUD": { + "open": null, + "high": 132.39216797540558, + "low": 128.52695670705936, + "close": 130.52908647526473, + "volume": 0, + "market_cap": 1447740447.626921, + "timestamp": "2013-04-28T23:59:00.000Z" + } + } + }, + { + "time_open": "2013-04-29T00:00:00.000Z", + "time_close": "2013-04-29T23:59:59.999Z", + "time_high": "2013-04-29T13:15:01.000Z", + "time_low": "2013-04-29T05:20:01.000Z", + "quote": { + "AUD": { + "open": 130.75666236543535, + "high": 142.67970067891736, + "low": 129.9456943366951, + "close": 139.77370978254794, + "volume": 0, + "market_cap": 1550883729.329852, + "timestamp": "2013-04-29T23:59:00.000Z" + } + } + }, + { + "time_open": "2013-04-30T00:00:00.000Z", + "time_close": "2013-04-30T23:59:59.999Z", + "time_high": "2013-04-30T08:25:02.000Z", + "time_low": "2013-04-30T18:55:01.000Z", + "quote": { + "AUD": { + "open": 139.2515230635335, + "high": 141.93391873626476, + "low": 129.37940647790543, + "close": 134.06635802469137, + "volume": 0, + "market_cap": 1488052782.6003087, + "timestamp": "2013-04-30T23:59:00.000Z" + } + } + }, + { + "time_open": "2013-05-01T00:00:00.000Z", + "time_close": "2013-05-01T23:59:59.999Z", + "time_high": "2013-05-01T00:15:01.000Z", + "time_low": "2013-05-01T19:55:01.000Z", + "quote": { + "AUD": { + "open": 134.06635802469137, + "high": 134.88573849160971, + "low": 104.93911468163968, + "close": 113.79243056489595, + "volume": 0, + "market_cap": 1263451603.6864119, + "timestamp": "2013-05-01T23:59:00.000Z" + } + } + }, + { + "time_open": "2013-05-02T00:00:00.000Z", + "time_close": "2013-05-02T23:59:59.999Z", + "time_high": "2013-05-02T14:25:01.000Z", + "time_low": "2013-05-02T14:30:02.000Z", + "quote": { + "AUD": { + "open": 113.19910247390133, + "high": 122.60835462135991, + "low": 90.08385249759387, + "close": 102.63388848353591, + "volume": 0, + "market_cap": 1139905858.2089553, + "timestamp": "2013-05-02T23:59:00.000Z" + } + } + }, + { + "time_open": "2013-05-03T00:00:00.000Z", + "time_close": "2013-05-03T23:59:59.999Z", + "time_high": "2013-05-03T05:30:02.000Z", + "time_low": "2013-05-03T03:05:01.000Z", + "quote": { + "AUD": { + "open": 103.64842454394694, + "high": 105.43929629649027, + "low": 77.03544845551335, + "close": 94.77409346519293, + "volume": 0, + "market_cap": 1052933070.3412836, + "timestamp": "2013-05-03T23:59:00.000Z" + } + } + }, + { + "time_open": "2013-05-04T00:00:00.000Z", + "time_close": "2013-05-04T23:59:59.999Z", + "time_high": "2013-05-04T07:15:01.000Z", + "time_low": "2013-05-04T06:50:01.000Z", + "quote": { + "AUD": { + "open": 95.11343656595025, + "high": 111.49893348846227, + "low": 89.68392476245879, + "close": 109.07504363001745, + "volume": 0, + "market_cap": 1212251854.2757416, + "timestamp": "2013-05-04T23:59:00.000Z" + } + } + }, + { + "time_open": "2021-01-01T00:00:00.000Z", + "time_close": "2021-01-01T23:59:59.999Z", + "time_high": "2021-01-01T12:38:43.000Z", + "time_low": "2021-01-01T00:16:43.000Z", + "quote": { + "AUD": { + "open": 37658.83948707033, + "high": 38417.9137031205, + "low": 37410.787501639206, + "close": 38181.99133300758, + "volume": 52943282221.028366, + "market_cap": 709720173049.5383, + "timestamp": "2021-01-01T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-02T00:00:00.000Z", + "time_close": "2021-01-02T23:59:59.999Z", + "time_high": "2021-01-02T19:49:42.000Z", + "time_low": "2021-01-02T00:31:44.000Z", + "quote": { + "AUD": { + "open": 38184.98611600682, + "high": 43096.681197423015, + "low": 37814.17187096531, + "close": 41760.62923079505, + "volume": 88214867181.97835, + "market_cap": 776278147177.8037, + "timestamp": "2021-01-02T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-03T00:00:00.000Z", + "time_close": "2021-01-03T23:59:59.999Z", + "time_high": "2021-01-03T07:47:38.000Z", + "time_low": "2021-01-03T00:20:45.000Z", + "quote": { + "AUD": { + "open": 41763.41015117659, + "high": 44985.93247585023, + "low": 41663.204350601605, + "close": 42511.10646879765, + "volume": 102011582370.28117, + "market_cap": 790270288834.0249, + "timestamp": "2021-01-03T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-04T00:00:00.000Z", + "time_close": "2021-01-04T23:59:59.999Z", + "time_high": "2021-01-04T04:07:42.000Z", + "time_low": "2021-01-04T10:19:42.000Z", + "quote": { + "AUD": { + "open": 42548.61349648768, + "high": 43360.96165147421, + "low": 37133.98436952697, + "close": 41686.38761359174, + "volume": 105824510346.65779, + "market_cap": 774984045201.7122, + "timestamp": "2021-01-04T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-05T00:00:00.000Z", + "time_close": "2021-01-05T23:59:59.999Z", + "time_high": "2021-01-05T22:44:35.000Z", + "time_low": "2021-01-05T06:16:41.000Z", + "quote": { + "AUD": { + "open": 41693.07321807638, + "high": 44403.79487147647, + "low": 39221.81167941294, + "close": 43790.067253370056, + "volume": 87016490203.50436, + "market_cap": 814135603090.2502, + "timestamp": "2021-01-05T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-06T00:00:00.000Z", + "time_close": "2021-01-06T23:59:59.999Z", + "time_high": "2021-01-06T23:57:36.000Z", + "time_low": "2021-01-06T00:25:38.000Z", + "quote": { + "AUD": { + "open": 43817.35864984641, + "high": 47186.65232598287, + "low": 43152.60281764236, + "close": 47115.85365360005, + "volume": 96330948324.8061, + "market_cap": 876019742889.9551, + "timestamp": "2021-01-06T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-07T00:00:00.000Z", + "time_close": "2021-01-07T23:59:59.999Z", + "time_high": "2021-01-07T18:17:42.000Z", + "time_low": "2021-01-07T08:25:51.000Z", + "quote": { + "AUD": { + "open": 47128.02139328098, + "high": 51833.478207775144, + "low": 46906.65117139608, + "close": 50686.90986207153, + "volume": 109124136558.20264, + "market_cap": 942469208700.134, + "timestamp": "2021-01-07T23:59:06.000Z" + } + } + } + ] + } +} diff --git a/tests/pricehist/sources/test_coinmarketcap/recent-btc-aud.json b/tests/pricehist/sources/test_coinmarketcap/recent-btc-aud.json new file mode 100644 index 0000000..342b824 --- /dev/null +++ b/tests/pricehist/sources/test_coinmarketcap/recent-btc-aud.json @@ -0,0 +1,136 @@ +{ + "status": { + "timestamp": "2021-07-16T10:42:32.013Z", + "error_code": 0, + "error_message": null, + "elapsed": 20, + "credit_count": 0, + "notice": null + }, + "data": { + "id": 1, + "name": "Bitcoin", + "symbol": "BTC", + "quotes": [ + { + "time_open": "2021-01-01T00:00:00.000Z", + "time_close": "2021-01-01T23:59:59.999Z", + "time_high": "2021-01-01T12:38:43.000Z", + "time_low": "2021-01-01T00:16:43.000Z", + "quote": { + "AUD": { + "open": 37658.83948707033, + "high": 38417.9137031205, + "low": 37410.787501639206, + "close": 38181.99133300758, + "volume": 52943282221.028366, + "market_cap": 709720173049.5383, + "timestamp": "2021-01-01T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-02T00:00:00.000Z", + "time_close": "2021-01-02T23:59:59.999Z", + "time_high": "2021-01-02T19:49:42.000Z", + "time_low": "2021-01-02T00:31:44.000Z", + "quote": { + "AUD": { + "open": 38184.98611600682, + "high": 43096.681197423015, + "low": 37814.17187096531, + "close": 41760.62923079505, + "volume": 88214867181.97835, + "market_cap": 776278147177.8037, + "timestamp": "2021-01-02T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-03T00:00:00.000Z", + "time_close": "2021-01-03T23:59:59.999Z", + "time_high": "2021-01-03T07:47:38.000Z", + "time_low": "2021-01-03T00:20:45.000Z", + "quote": { + "AUD": { + "open": 41763.41015117659, + "high": 44985.93247585023, + "low": 41663.204350601605, + "close": 42511.10646879765, + "volume": 102011582370.28117, + "market_cap": 790270288834.0249, + "timestamp": "2021-01-03T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-04T00:00:00.000Z", + "time_close": "2021-01-04T23:59:59.999Z", + "time_high": "2021-01-04T04:07:42.000Z", + "time_low": "2021-01-04T10:19:42.000Z", + "quote": { + "AUD": { + "open": 42548.61349648768, + "high": 43360.96165147421, + "low": 37133.98436952697, + "close": 41686.38761359174, + "volume": 105824510346.65779, + "market_cap": 774984045201.7122, + "timestamp": "2021-01-04T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-05T00:00:00.000Z", + "time_close": "2021-01-05T23:59:59.999Z", + "time_high": "2021-01-05T22:44:35.000Z", + "time_low": "2021-01-05T06:16:41.000Z", + "quote": { + "AUD": { + "open": 41693.07321807638, + "high": 44403.79487147647, + "low": 39221.81167941294, + "close": 43790.067253370056, + "volume": 87016490203.50436, + "market_cap": 814135603090.2502, + "timestamp": "2021-01-05T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-06T00:00:00.000Z", + "time_close": "2021-01-06T23:59:59.999Z", + "time_high": "2021-01-06T23:57:36.000Z", + "time_low": "2021-01-06T00:25:38.000Z", + "quote": { + "AUD": { + "open": 43817.35864984641, + "high": 47186.65232598287, + "low": 43152.60281764236, + "close": 47115.85365360005, + "volume": 96330948324.8061, + "market_cap": 876019742889.9551, + "timestamp": "2021-01-06T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-07T00:00:00.000Z", + "time_close": "2021-01-07T23:59:59.999Z", + "time_high": "2021-01-07T18:17:42.000Z", + "time_low": "2021-01-07T08:25:51.000Z", + "quote": { + "AUD": { + "open": 47128.02139328098, + "high": 51833.478207775144, + "low": 46906.65117139608, + "close": 50686.90986207153, + "volume": 109124136558.20264, + "market_cap": 942469208700.134, + "timestamp": "2021-01-07T23:59:06.000Z" + } + } + } + ] + } +} diff --git a/tests/pricehist/sources/test_coinmarketcap/recent-btc-id2782.json b/tests/pricehist/sources/test_coinmarketcap/recent-btc-id2782.json new file mode 100644 index 0000000..8614b07 --- /dev/null +++ b/tests/pricehist/sources/test_coinmarketcap/recent-btc-id2782.json @@ -0,0 +1,136 @@ +{ + "status": { + "timestamp": "2021-07-16T10:42:27.169Z", + "error_code": 0, + "error_message": null, + "elapsed": 19, + "credit_count": 0, + "notice": null + }, + "data": { + "id": 1, + "name": "Bitcoin", + "symbol": "BTC", + "quotes": [ + { + "time_open": "2021-01-01T00:00:00.000Z", + "time_close": "2021-01-01T23:59:59.999Z", + "time_high": "2021-01-01T12:38:43.000Z", + "time_low": "2021-01-01T00:16:43.000Z", + "quote": { + "2782": { + "open": 37658.83948707033, + "high": 38417.9137031205, + "low": 37410.787501639206, + "close": 38181.99133300758, + "volume": 52943282221.028366, + "market_cap": 709720173049.5383, + "timestamp": "2021-01-01T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-02T00:00:00.000Z", + "time_close": "2021-01-02T23:59:59.999Z", + "time_high": "2021-01-02T19:49:42.000Z", + "time_low": "2021-01-02T00:31:44.000Z", + "quote": { + "2782": { + "open": 38184.98611600682, + "high": 43096.681197423015, + "low": 37814.17187096531, + "close": 41760.62923079505, + "volume": 88214867181.97835, + "market_cap": 776278147177.8037, + "timestamp": "2021-01-02T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-03T00:00:00.000Z", + "time_close": "2021-01-03T23:59:59.999Z", + "time_high": "2021-01-03T07:47:38.000Z", + "time_low": "2021-01-03T00:20:45.000Z", + "quote": { + "2782": { + "open": 41763.41015117659, + "high": 44985.93247585023, + "low": 41663.204350601605, + "close": 42511.10646879765, + "volume": 102011582370.28117, + "market_cap": 790270288834.0249, + "timestamp": "2021-01-03T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-04T00:00:00.000Z", + "time_close": "2021-01-04T23:59:59.999Z", + "time_high": "2021-01-04T04:07:42.000Z", + "time_low": "2021-01-04T10:19:42.000Z", + "quote": { + "2782": { + "open": 42548.61349648768, + "high": 43360.96165147421, + "low": 37133.98436952697, + "close": 41686.38761359174, + "volume": 105824510346.65779, + "market_cap": 774984045201.7122, + "timestamp": "2021-01-04T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-05T00:00:00.000Z", + "time_close": "2021-01-05T23:59:59.999Z", + "time_high": "2021-01-05T22:44:35.000Z", + "time_low": "2021-01-05T06:16:41.000Z", + "quote": { + "2782": { + "open": 41693.07321807638, + "high": 44403.79487147647, + "low": 39221.81167941294, + "close": 43790.067253370056, + "volume": 87016490203.50436, + "market_cap": 814135603090.2502, + "timestamp": "2021-01-05T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-06T00:00:00.000Z", + "time_close": "2021-01-06T23:59:59.999Z", + "time_high": "2021-01-06T23:57:36.000Z", + "time_low": "2021-01-06T00:25:38.000Z", + "quote": { + "2782": { + "open": 43817.35864984641, + "high": 47186.65232598287, + "low": 43152.60281764236, + "close": 47115.85365360005, + "volume": 96330948324.8061, + "market_cap": 876019742889.9551, + "timestamp": "2021-01-06T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-07T00:00:00.000Z", + "time_close": "2021-01-07T23:59:59.999Z", + "time_high": "2021-01-07T18:17:42.000Z", + "time_low": "2021-01-07T08:25:51.000Z", + "quote": { + "2782": { + "open": 47128.02139328098, + "high": 51833.478207775144, + "low": 46906.65117139608, + "close": 50686.90986207153, + "volume": 109124136558.20264, + "market_cap": 942469208700.134, + "timestamp": "2021-01-07T23:59:06.000Z" + } + } + } + ] + } +} diff --git a/tests/pricehist/sources/test_coinmarketcap/recent-id1-aud.json b/tests/pricehist/sources/test_coinmarketcap/recent-id1-aud.json new file mode 100644 index 0000000..8b70b6a --- /dev/null +++ b/tests/pricehist/sources/test_coinmarketcap/recent-id1-aud.json @@ -0,0 +1,136 @@ +{ + "status": { + "timestamp": "2021-07-16T10:42:24.612Z", + "error_code": 0, + "error_message": null, + "elapsed": 57, + "credit_count": 0, + "notice": null + }, + "data": { + "id": 1, + "name": "Bitcoin", + "symbol": "BTC", + "quotes": [ + { + "time_open": "2021-01-01T00:00:00.000Z", + "time_close": "2021-01-01T23:59:59.999Z", + "time_high": "2021-01-01T12:38:43.000Z", + "time_low": "2021-01-01T00:16:43.000Z", + "quote": { + "AUD": { + "open": 37658.83948707033, + "high": 38417.9137031205, + "low": 37410.787501639206, + "close": 38181.99133300758, + "volume": 52943282221.028366, + "market_cap": 709720173049.5383, + "timestamp": "2021-01-01T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-02T00:00:00.000Z", + "time_close": "2021-01-02T23:59:59.999Z", + "time_high": "2021-01-02T19:49:42.000Z", + "time_low": "2021-01-02T00:31:44.000Z", + "quote": { + "AUD": { + "open": 38184.98611600682, + "high": 43096.681197423015, + "low": 37814.17187096531, + "close": 41760.62923079505, + "volume": 88214867181.97835, + "market_cap": 776278147177.8037, + "timestamp": "2021-01-02T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-03T00:00:00.000Z", + "time_close": "2021-01-03T23:59:59.999Z", + "time_high": "2021-01-03T07:47:38.000Z", + "time_low": "2021-01-03T00:20:45.000Z", + "quote": { + "AUD": { + "open": 41763.41015117659, + "high": 44985.93247585023, + "low": 41663.204350601605, + "close": 42511.10646879765, + "volume": 102011582370.28117, + "market_cap": 790270288834.0249, + "timestamp": "2021-01-03T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-04T00:00:00.000Z", + "time_close": "2021-01-04T23:59:59.999Z", + "time_high": "2021-01-04T04:07:42.000Z", + "time_low": "2021-01-04T10:19:42.000Z", + "quote": { + "AUD": { + "open": 42548.61349648768, + "high": 43360.96165147421, + "low": 37133.98436952697, + "close": 41686.38761359174, + "volume": 105824510346.65779, + "market_cap": 774984045201.7122, + "timestamp": "2021-01-04T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-05T00:00:00.000Z", + "time_close": "2021-01-05T23:59:59.999Z", + "time_high": "2021-01-05T22:44:35.000Z", + "time_low": "2021-01-05T06:16:41.000Z", + "quote": { + "AUD": { + "open": 41693.07321807638, + "high": 44403.79487147647, + "low": 39221.81167941294, + "close": 43790.067253370056, + "volume": 87016490203.50436, + "market_cap": 814135603090.2502, + "timestamp": "2021-01-05T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-06T00:00:00.000Z", + "time_close": "2021-01-06T23:59:59.999Z", + "time_high": "2021-01-06T23:57:36.000Z", + "time_low": "2021-01-06T00:25:38.000Z", + "quote": { + "AUD": { + "open": 43817.35864984641, + "high": 47186.65232598287, + "low": 43152.60281764236, + "close": 47115.85365360005, + "volume": 96330948324.8061, + "market_cap": 876019742889.9551, + "timestamp": "2021-01-06T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-07T00:00:00.000Z", + "time_close": "2021-01-07T23:59:59.999Z", + "time_high": "2021-01-07T18:17:42.000Z", + "time_low": "2021-01-07T08:25:51.000Z", + "quote": { + "AUD": { + "open": 47128.02139328098, + "high": 51833.478207775144, + "low": 46906.65117139608, + "close": 50686.90986207153, + "volume": 109124136558.20264, + "market_cap": 942469208700.134, + "timestamp": "2021-01-07T23:59:06.000Z" + } + } + } + ] + } +} diff --git a/tests/pricehist/sources/test_coinmarketcap/recent-id1-id2782.json b/tests/pricehist/sources/test_coinmarketcap/recent-id1-id2782.json new file mode 100644 index 0000000..d172453 --- /dev/null +++ b/tests/pricehist/sources/test_coinmarketcap/recent-id1-id2782.json @@ -0,0 +1,136 @@ +{ + "status": { + "timestamp": "2021-07-16T10:42:21.065Z", + "error_code": 0, + "error_message": null, + "elapsed": 17, + "credit_count": 0, + "notice": null + }, + "data": { + "id": 1, + "name": "Bitcoin", + "symbol": "BTC", + "quotes": [ + { + "time_open": "2021-01-01T00:00:00.000Z", + "time_close": "2021-01-01T23:59:59.999Z", + "time_high": "2021-01-01T12:38:43.000Z", + "time_low": "2021-01-01T00:16:43.000Z", + "quote": { + "2782": { + "open": 37658.83948707033, + "high": 38417.9137031205, + "low": 37410.787501639206, + "close": 38181.99133300758, + "volume": 52943282221.028366, + "market_cap": 709720173049.5383, + "timestamp": "2021-01-01T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-02T00:00:00.000Z", + "time_close": "2021-01-02T23:59:59.999Z", + "time_high": "2021-01-02T19:49:42.000Z", + "time_low": "2021-01-02T00:31:44.000Z", + "quote": { + "2782": { + "open": 38184.98611600682, + "high": 43096.681197423015, + "low": 37814.17187096531, + "close": 41760.62923079505, + "volume": 88214867181.97835, + "market_cap": 776278147177.8037, + "timestamp": "2021-01-02T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-03T00:00:00.000Z", + "time_close": "2021-01-03T23:59:59.999Z", + "time_high": "2021-01-03T07:47:38.000Z", + "time_low": "2021-01-03T00:20:45.000Z", + "quote": { + "2782": { + "open": 41763.41015117659, + "high": 44985.93247585023, + "low": 41663.204350601605, + "close": 42511.10646879765, + "volume": 102011582370.28117, + "market_cap": 790270288834.0249, + "timestamp": "2021-01-03T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-04T00:00:00.000Z", + "time_close": "2021-01-04T23:59:59.999Z", + "time_high": "2021-01-04T04:07:42.000Z", + "time_low": "2021-01-04T10:19:42.000Z", + "quote": { + "2782": { + "open": 42548.61349648768, + "high": 43360.96165147421, + "low": 37133.98436952697, + "close": 41686.38761359174, + "volume": 105824510346.65779, + "market_cap": 774984045201.7122, + "timestamp": "2021-01-04T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-05T00:00:00.000Z", + "time_close": "2021-01-05T23:59:59.999Z", + "time_high": "2021-01-05T22:44:35.000Z", + "time_low": "2021-01-05T06:16:41.000Z", + "quote": { + "2782": { + "open": 41693.07321807638, + "high": 44403.79487147647, + "low": 39221.81167941294, + "close": 43790.067253370056, + "volume": 87016490203.50436, + "market_cap": 814135603090.2502, + "timestamp": "2021-01-05T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-06T00:00:00.000Z", + "time_close": "2021-01-06T23:59:59.999Z", + "time_high": "2021-01-06T23:57:36.000Z", + "time_low": "2021-01-06T00:25:38.000Z", + "quote": { + "2782": { + "open": 43817.35864984641, + "high": 47186.65232598287, + "low": 43152.60281764236, + "close": 47115.85365360005, + "volume": 96330948324.8061, + "market_cap": 876019742889.9551, + "timestamp": "2021-01-06T23:59:06.000Z" + } + } + }, + { + "time_open": "2021-01-07T00:00:00.000Z", + "time_close": "2021-01-07T23:59:59.999Z", + "time_high": "2021-01-07T18:17:42.000Z", + "time_low": "2021-01-07T08:25:51.000Z", + "quote": { + "2782": { + "open": 47128.02139328098, + "high": 51833.478207775144, + "low": 46906.65117139608, + "close": 50686.90986207153, + "volume": 109124136558.20264, + "market_cap": 942469208700.134, + "timestamp": "2021-01-07T23:59:06.000Z" + } + } + } + ] + } +} From b868602caefc94e44abf29d73831af8aaea880c4 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 17 Jul 2021 18:50:01 +0200 Subject: [PATCH 007/149] Extra coinmarketcap test cases for coverage. --- tests/pricehist/sources/test_coinmarketcap.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/pricehist/sources/test_coinmarketcap.py b/tests/pricehist/sources/test_coinmarketcap.py index 6ae175c..ba4cc0f 100644 --- a/tests/pricehist/sources/test_coinmarketcap.py +++ b/tests/pricehist/sources/test_coinmarketcap.py @@ -201,6 +201,13 @@ def test_symbols_crypto_parsing_error(src, requests_mock, fiat_ok): assert "while parsing data" in str(e.value) +def test_symbols_no_data(src, type, requests_mock): + requests_mock.add(responses.GET, fiat_url, body='{"data": []}') + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "Empty data section" in str(e.value) + + def test_fetch_known_pair_id_id(src, type, recent_id_id_ok, crypto_ok, fiat_ok): series = src.fetch(Series("ID=1", "ID=2782", type, "2021-01-01", "2021-01-07")) req = recent_id_id_ok.calls[0].request @@ -323,6 +330,23 @@ def test_fetch_in_future(src, type, requests_mock): assert "start date must be in the past" in str(e.value) +def test_fetch_reversed_dates(src, type, requests_mock): + requests_mock.add( + responses.GET, + fetch_url, + status=400, + body="""{ + "status": { + "error_code": 400, + "error_message": "\\"time_start\\" must be older than \\"time_end\\"." + } + }""", + ) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("BTC", "AUD", type, "2021-01-07", "2021-01-01")) + assert "start date must preceed or match the end" in str(e.value) + + def test_fetch_empty(src, type, requests_mock): requests_mock.add( responses.GET, @@ -428,3 +452,10 @@ def test_fetch_parsing_error(src, type, requests_mock): with pytest.raises(exceptions.ResponseParsingError) as e: src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) assert "while parsing data" in str(e.value) + + +def test_fetch_unexpected_json(src, type, requests_mock): + requests_mock.add(responses.GET, fetch_url, body='{"notdata": []}') + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) + assert "Unexpected content" in str(e.value) From 591bb33cd3fe16b94f129ff520613c98d8863534 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 17 Jul 2021 18:56:00 +0200 Subject: [PATCH 008/149] Version 0.1.5. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ceae949..fb5a54f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "0.1.4" +version = "0.1.5" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index bbab024..1276d02 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "0.1.4" +__version__ = "0.1.5" From c4afbb5ec0796d8d1edc41f2abbc642d66a86cc4 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 17 Jul 2021 19:02:23 +0200 Subject: [PATCH 009/149] Update README for --verbose/-vvv changes. --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 49857ee..c349dda 100644 --- a/README.md +++ b/README.md @@ -35,14 +35,14 @@ Show usage information: pricehist -h ``` ``` -usage: pricehist [-h] [--version] [--verbose] {sources,source,fetch} ... +usage: pricehist [-h] [--version] [-vvv] {sources,source,fetch} ... Fetch historical price data optional arguments: -h, --help show this help message and exit --version show version information - --verbose show all log messages + -vvv, --verbose show all log messages commands: {sources,source,fetch} @@ -57,7 +57,7 @@ Show usage information for the `fetch` command: pricehist fetch -h ``` ``` -usage: pricehist fetch SOURCE PAIR [-h] [-t TYPE] [-s DATE | -sx DATE] [-e DATE | -ex DATE] +usage: pricehist fetch SOURCE PAIR [-h] [-vvv] [-t TYPE] [-s DATE | -sx DATE] [-e DATE | -ex DATE] [-o beancount|csv|gnucash-sql|ledger] [--invert] [--quantize INT] [--fmt-base SYM] [--fmt-quote SYM] [--fmt-time TIME] [--fmt-decimal CHAR] [--fmt-thousands CHAR] [--fmt-symbol rightspace|right|leftspace|left] [--fmt-datesep CHAR] [--fmt-csvdelim CHAR] @@ -68,6 +68,7 @@ positional arguments: optional arguments: -h, --help show this help message and exit + -vvv, --verbose show all log messages -t TYPE, --type TYPE price type, e.g. close -s DATE, --start DATE start date, inclusive (default: source start) -sx DATE, --startx DATE start date, exclusive From cdd78f04454f6122603119ddd85f08c7e91d18c7 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 18 Jul 2021 18:10:14 +0200 Subject: [PATCH 010/149] Fix formatting of base-only pairs in exception messages. --- src/pricehist/exceptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pricehist/exceptions.py b/src/pricehist/exceptions.py index 1e0f0eb..78d66b7 100644 --- a/src/pricehist/exceptions.py +++ b/src/pricehist/exceptions.py @@ -9,7 +9,7 @@ class InvalidPair(SourceError, ValueError): self.base = base self.quote = quote self.source = source - pair = "/".join([base, quote]) + pair = "/".join([s for s in [base, quote] if s]) insert = message + " " if message else "" full_message = ( @@ -25,7 +25,7 @@ class InvalidType(SourceError, ValueError): def __init__(self, type, base, quote, source): self.type = type - self.pair = "/".join([base, quote]) + self.pair = "/".join([s for s in [base, quote] if s]) message = ( f"Invalid price type '{type}' for pair '{self.pair}'. " f"Run 'pricehist source {source.id()} " From 46db6e9a6faf56499e7677cc02645cf4d4e03ba2 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 18 Jul 2021 18:09:42 +0200 Subject: [PATCH 011/149] Improve yahoo error handling and add tests. --- src/pricehist/sources/yahoo.py | 103 ++++-- tests/pricehist/sources/test_yahoo.py | 311 ++++++++++++++++++ .../sources/test_yahoo/ibm-long-partial.csv | 11 + .../sources/test_yahoo/tsla-recent.csv | 6 + .../sources/test_yahoo/tsla-spark.json | 77 +++++ 5 files changed, 486 insertions(+), 22 deletions(-) create mode 100644 tests/pricehist/sources/test_yahoo.py create mode 100644 tests/pricehist/sources/test_yahoo/ibm-long-partial.csv create mode 100644 tests/pricehist/sources/test_yahoo/tsla-recent.csv create mode 100644 tests/pricehist/sources/test_yahoo/tsla-spark.json diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 0a61cb5..ae3179a 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -7,7 +7,7 @@ from decimal import Decimal import requests -from pricehist import __version__ +from pricehist import __version__, exceptions from pricehist.price import Price from .basesource import BaseSource @@ -30,7 +30,10 @@ class Yahoo(BaseSource): return "https://finance.yahoo.com/" def start(self): - return "1970-01-01" + # The "Download historical data in Yahoo Finance" page says + # "Historical prices usually don't go back earlier than 1970", but + # several do. Examples going back to 1962-01-02 include ED and IBM. + return "1962-01-02" def types(self): return ["adjclose", "open", "high", "low", "close", "mid"] @@ -55,7 +58,7 @@ class Yahoo(BaseSource): return ( "Find the symbol of interest on https://finance.yahoo.com/ and use " "that as the PAIR in your pricehist command. Prices for each symbol " - "are given in its native currency." + "are quoted in its native currency." ) def symbols(self): @@ -63,10 +66,12 @@ class Yahoo(BaseSource): return [] def fetch(self, series): - # TODO fail if quote isn't empty - yahoo symbols don't have a slash - spark, history = self._data(series) + if series.quote: + raise exceptions.InvalidPair( + series.base, series.quote, self, "Don't specify the quote currency." + ) - output_quote = spark["spark"]["result"][0]["response"][0]["meta"]["currency"] + quote, history = self._data(series) prices = [ Price(row["date"], amount) @@ -74,15 +79,13 @@ class Yahoo(BaseSource): if (amount := self._amount(row, series.type)) ] - return dataclasses.replace(series, quote=output_quote, prices=prices) + return dataclasses.replace(series, quote=quote, prices=prices) def _amount(self, row, type): - if type != "mid" and row[type] != "null": - return Decimal(row[type]) - elif type == "mid" and row["high"] != "null" and row["low"] != "null": + if type == "mid" and row["high"] != "null" and row["low"] != "null": return sum([Decimal(row["high"]), Decimal(row["low"])]) / 2 else: - return None + return Decimal(row[type]) def _data(self, series) -> (dict, csv.DictReader): base_url = "https://query1.finance.yahoo.com/v7/finance" @@ -97,10 +100,32 @@ class Yahoo(BaseSource): "includeTimestamps": "false", "includePrePost": "false", } - spark_response = self.log_curl( - requests.get(spark_url, params=spark_params, headers=headers) - ) - spark = json.loads(spark_response.content) + try: + spark_response = self.log_curl( + requests.get(spark_url, params=spark_params, headers=headers) + ) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + code = spark_response.status_code + text = spark_response.text + if code == 404 and "No data found for spark symbols" in text: + raise exceptions.InvalidPair( + series.base, series.quote, self, "Symbol not found." + ) + + try: + spark_response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + spark = json.loads(spark_response.content) + quote = spark["spark"]["result"][0]["response"][0]["meta"]["currency"] + except Exception as e: + raise exceptions.ResponseParsingError( + "The spark data couldn't be parsed. " + ) from e start_ts = int( datetime.strptime(series.start, "%Y-%m-%d") @@ -123,11 +148,45 @@ class Yahoo(BaseSource): "events": "history", "includeAdjustedClose": "true", } - history_response = self.log_curl( - requests.get(history_url, params=history_params, headers=headers) - ) - history_lines = history_response.content.decode("utf-8").splitlines() - history_lines[0] = history_lines[0].lower().replace(" ", "") - history = csv.DictReader(history_lines, delimiter=",") - return (spark, history) + try: + history_response = self.log_curl( + requests.get(history_url, params=history_params, headers=headers) + ) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + code = history_response.status_code + text = history_response.text + + if code == 404 and "No data found, symbol may be delisted" in text: + raise exceptions.InvalidPair( + series.base, series.quote, self, "Symbol not found." + ) + if code == 400 and "Data doesn't exist" in text: + raise exceptions.BadResponse( + "No data for the given interval. Try requesting a larger interval." + ) + + elif code == 404 and "Timestamp data missing" in text: + raise exceptions.BadResponse( + "Data missing. The given interval may be for a gap in the data " + "such as a weekend or holiday. Try requesting a larger interval." + ) + + try: + history_response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + history_lines = history_response.content.decode("utf-8").splitlines() + history_lines[0] = history_lines[0].lower().replace(" ", "") + history = csv.DictReader(history_lines, delimiter=",") + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if history_lines[0] != "date,open,high,low,close,adjclose,volume": + raise exceptions.ResponseParsingError("Unexpected CSV format") + + return (quote, history) diff --git a/tests/pricehist/sources/test_yahoo.py b/tests/pricehist/sources/test_yahoo.py new file mode 100644 index 0000000..322e9a5 --- /dev/null +++ b/tests/pricehist/sources/test_yahoo.py @@ -0,0 +1,311 @@ +import logging +import os +from datetime import datetime, timezone +from decimal import Decimal +from pathlib import Path + +import pytest +import requests +import responses + +from pricehist import exceptions +from pricehist.price import Price +from pricehist.series import Series +from pricehist.sources.yahoo import Yahoo + + +def timestamp(date): + return int( + datetime.strptime(date, "%Y-%m-%d").replace(tzinfo=timezone.utc).timestamp() + ) + + +@pytest.fixture +def src(): + return Yahoo() + + +@pytest.fixture +def type(src): + return src.types()[0] + + +@pytest.fixture +def requests_mock(): + with responses.RequestsMock() as mock: + yield mock + + +spark_url = "https://query1.finance.yahoo.com/v7/finance/spark" + + +def history_url(base): + return f"https://query1.finance.yahoo.com/v7/finance/download/{base}" + + +@pytest.fixture +def spark_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "tsla-spark.json").read_text() + requests_mock.add(responses.GET, spark_url, body=json, status=200) + yield requests_mock + + +@pytest.fixture +def recent_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "tsla-recent.csv").read_text() + requests_mock.add(responses.GET, history_url("TSLA"), body=json, status=200) + yield requests_mock + + +@pytest.fixture +def long_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "ibm-long-partial.csv").read_text() + requests_mock.add(responses.GET, history_url("IBM"), body=json, status=200) + yield requests_mock + + +def test_normalizesymbol(src): + assert src.normalizesymbol("tsla") == "TSLA" + + +def test_metadata(src): + assert isinstance(src.id(), str) + assert len(src.id()) > 0 + + assert isinstance(src.name(), str) + assert len(src.name()) > 0 + + assert isinstance(src.description(), str) + assert len(src.description()) > 0 + + assert isinstance(src.source_url(), str) + assert src.source_url().startswith("http") + + assert datetime.strptime(src.start(), "%Y-%m-%d") + + assert isinstance(src.types(), list) + assert len(src.types()) > 0 + assert isinstance(src.types()[0], str) + assert len(src.types()[0]) > 0 + + assert isinstance(src.notes(), str) + + +def test_symbols(src, caplog): + with caplog.at_level(logging.INFO): + symbols = src.symbols() + assert symbols == [] + assert any(["Find the symbol of interest on" in r.message for r in caplog.records]) + + +def test_fetch_known(src, type, spark_ok, recent_ok): + series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + spark_req = recent_ok.calls[0].request + hist_req = recent_ok.calls[1].request + assert spark_req.params["symbols"] == "TSLA" + assert hist_req.params["events"] == "history" + assert hist_req.params["includeAdjustedClose"] == "true" + assert (series.base, series.quote) == ("TSLA", "USD") + assert len(series.prices) == 5 + + +def test_fetch_requests_and_receives_correct_times(src, type, spark_ok, recent_ok): + series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + hist_req = recent_ok.calls[1].request + assert hist_req.params["period1"] == str(timestamp("2021-01-04")) + assert hist_req.params["period2"] == str(timestamp("2021-01-09")) # rounded up one + assert hist_req.params["interval"] == "1d" + assert series.prices[0] == Price("2021-01-04", Decimal("729.770020")) + assert series.prices[-1] == Price("2021-01-08", Decimal("880.020020")) + + +def test_fetch_requests_logged(src, type, spark_ok, recent_ok, caplog): + with caplog.at_level(logging.DEBUG): + src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + logged_requests = 0 + for r in caplog.records: + if r.levelname == "DEBUG" and " curl " in r.message: + logged_requests += 1 + assert logged_requests == 2 + + +def test_fetch_types_all_available(src, spark_ok, recent_ok): + adj = src.fetch(Series("TSLA", "", "adjclose", "2021-01-04", "2021-01-08")) + opn = src.fetch(Series("TSLA", "", "open", "2021-01-04", "2021-01-08")) + hgh = src.fetch(Series("TSLA", "", "high", "2021-01-04", "2021-01-08")) + low = src.fetch(Series("TSLA", "", "low", "2021-01-04", "2021-01-08")) + cls = src.fetch(Series("TSLA", "", "close", "2021-01-04", "2021-01-08")) + mid = src.fetch(Series("TSLA", "", "mid", "2021-01-04", "2021-01-08")) + assert adj.prices[0].amount == Decimal("729.770020") + assert opn.prices[0].amount == Decimal("719.460022") + assert hgh.prices[0].amount == Decimal("744.489990") + assert low.prices[0].amount == Decimal("717.190002") + assert cls.prices[0].amount == Decimal("729.770020") + assert mid.prices[0].amount == Decimal("730.839996") + + +def test_fetch_type_mid_is_mean_of_low_and_high(src, spark_ok, recent_ok): + mid = src.fetch(Series("TSLA", "", "mid", "2021-01-04", "2021-01-08")).prices + hgh = src.fetch(Series("TSLA", "", "high", "2021-01-04", "2021-01-08")).prices + low = src.fetch(Series("TSLA", "", "low", "2021-01-04", "2021-01-08")).prices + assert all( + [ + mid[i].amount == (sum([low[i].amount, hgh[i].amount]) / 2) + for i in range(0, 5) + ] + ) + + +def test_fetch_from_before_start(src, type, spark_ok, long_ok): + series = src.fetch(Series("IBM", "", type, "1900-01-01", "2021-01-08")) + assert series.prices[0] == Price("1962-01-02", Decimal("1.837710")) + assert series.prices[-1] == Price("2021-01-08", Decimal("125.433624")) + assert len(series.prices) > 9 + + +def test_fetch_to_future(src, type, spark_ok, recent_ok): + series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2100-01-08")) + assert len(series.prices) > 0 + + +def test_fetch_no_data_in_past(src, type, spark_ok, requests_mock): + requests_mock.add( + responses.GET, + history_url("TSLA"), + status=400, + body=( + "400 Bad Request: Data doesn't exist for " + "startDate = 1262304000, endDate = 1262995200" + ), + ) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("TSLA", "", type, "2010-01-04", "2010-01-08")) + assert "No data for the given interval" in str(e.value) + + +def test_fetch_no_data_in_future(src, type, spark_ok, requests_mock): + requests_mock.add( + responses.GET, + history_url("TSLA"), + status=400, + body=( + "400 Bad Request: Data doesn't exist for " + "startDate = 1893715200, endDate = 1894147200" + ), + ) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("TSLA", "", type, "2030-01-04", "2030-01-08")) + assert "No data for the given interval" in str(e.value) + + +def test_fetch_no_data_on_weekend(src, type, spark_ok, requests_mock): + requests_mock.add( + responses.GET, + history_url("TSLA"), + status=404, + body="404 Not Found: Timestamp data missing.", + ) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("TSLA", "", type, "2021-01-09", "2021-01-10")) + assert "may be for a gap in the data" in str(e.value) + + +def test_fetch_bad_sym(src, type, requests_mock): + requests_mock.add( + responses.GET, + spark_url, + status=404, + body="""{ + "spark": { + "result": null, + "error": { + "code": "Not Found", + "description": "No data found for spark symbols" + } + } + }""", + ) + with pytest.raises(exceptions.InvalidPair) as e: + src.fetch(Series("NOTABASE", "", type, "2021-01-04", "2021-01-08")) + assert "Symbol not found" in str(e.value) + + +def test_fetch_bad_sym_history(src, type, spark_ok, requests_mock): + # In practice the spark history requests should succeed or fail together. + # This extra test ensures that a failure of the the history part is handled + # correctly even if the spark part succeeds. + requests_mock.add( + responses.GET, + history_url("NOTABASE"), + status=404, + body="404 Not Found: No data found, symbol may be delisted", + ) + with pytest.raises(exceptions.InvalidPair) as e: + src.fetch(Series("NOTABASE", "", type, "2021-01-04", "2021-01-08")) + assert "Symbol not found" in str(e.value) + + +def test_fetch_giving_quote(src, type): + with pytest.raises(exceptions.InvalidPair) as e: + src.fetch(Series("TSLA", "USD", type, "2021-01-04", "2021-01-08")) + assert "quote currency" in str(e.value) + + +def test_fetch_spark_network_issue(src, type, requests_mock): + body = requests.exceptions.ConnectionError("Network issue") + requests_mock.add(responses.GET, spark_url, body=body) + with pytest.raises(exceptions.RequestError) as e: + src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + assert "Network issue" in str(e.value) + + +def test_fetch_spark_bad_status(src, type, requests_mock): + requests_mock.add(responses.GET, spark_url, status=500, body="Some other reason") + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + assert "Internal Server Error" in str(e.value) + + +def test_fetch_spark_parsing_error(src, type, requests_mock): + requests_mock.add(responses.GET, spark_url, body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + assert "spark data couldn't be parsed" in str(e.value) + + +def test_fetch_spark_unexpected_json(src, type, requests_mock): + requests_mock.add(responses.GET, spark_url, body='{"notdata": []}') + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + assert "spark data couldn't be parsed" in str(e.value) + + +def test_fetch_history_network_issue(src, type, spark_ok, requests_mock): + body = requests.exceptions.ConnectionError("Network issue") + requests_mock.add(responses.GET, history_url("TSLA"), body=body) + with pytest.raises(exceptions.RequestError) as e: + src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + assert "Network issue" in str(e.value) + + +def test_fetch_history_bad_status(src, type, spark_ok, requests_mock): + requests_mock.add( + responses.GET, history_url("TSLA"), status=500, body="Some other reason" + ) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + assert "Internal Server Error" in str(e.value) + + +def test_fetch_history_parsing_error(src, type, spark_ok, requests_mock): + requests_mock.add(responses.GET, history_url("TSLA"), body="") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + assert "error occurred while parsing data from the source" in str(e.value) + + +def test_fetch_history_unexpected_csv_format(src, type, spark_ok, requests_mock): + requests_mock.add(responses.GET, history_url("TSLA"), body="BAD HEADER\nBAD DATA") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) + assert "Unexpected CSV format" in str(e.value) diff --git a/tests/pricehist/sources/test_yahoo/ibm-long-partial.csv b/tests/pricehist/sources/test_yahoo/ibm-long-partial.csv new file mode 100644 index 0000000..98149ad --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/ibm-long-partial.csv @@ -0,0 +1,11 @@ +Date,Open,High,Low,Close,Adj Close,Volume +1962-01-02,7.713333,7.713333,7.626667,7.626667,1.837710,390000 +1962-01-03,7.626667,7.693333,7.626667,7.693333,1.853774,292500 +1962-01-04,7.693333,7.693333,7.613333,7.616667,1.835299,262500 +1962-01-05,7.606667,7.606667,7.453333,7.466667,1.799155,367500 +1962-01-08,7.460000,7.460000,7.266667,7.326667,1.765422,547500 +2021-01-04,125.849998,125.919998,123.040001,123.940002,120.954201,5179200 +2021-01-05,125.010002,126.680000,124.610001,126.139999,123.101204,6114600 +2021-01-06,126.900002,131.880005,126.720001,129.289993,126.175316,7956700 +2021-01-07,130.039993,130.460007,128.259995,128.990005,125.882545,4507400 +2021-01-08,128.570007,129.320007,126.980003,128.529999,125.433624,4676200 diff --git a/tests/pricehist/sources/test_yahoo/tsla-recent.csv b/tests/pricehist/sources/test_yahoo/tsla-recent.csv new file mode 100644 index 0000000..48b5692 --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/tsla-recent.csv @@ -0,0 +1,6 @@ +Date,Open,High,Low,Close,Adj Close,Volume +2021-01-04,719.460022,744.489990,717.190002,729.770020,729.770020,48638200 +2021-01-05,723.659973,740.840027,719.200012,735.109985,735.109985,32245200 +2021-01-06,758.489990,774.000000,749.099976,755.979980,755.979980,44700000 +2021-01-07,777.630005,816.989990,775.200012,816.039978,816.039978,51498900 +2021-01-08,856.000000,884.489990,838.390015,880.020020,880.020020,75055500 \ No newline at end of file diff --git a/tests/pricehist/sources/test_yahoo/tsla-spark.json b/tests/pricehist/sources/test_yahoo/tsla-spark.json new file mode 100644 index 0000000..53e7585 --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/tsla-spark.json @@ -0,0 +1,77 @@ +{ + "spark": { + "result": [ + { + "symbol": "TSLA", + "response": [ + { + "meta": { + "currency": "USD", + "symbol": "TSLA", + "exchangeName": "NMS", + "instrumentType": "EQUITY", + "firstTradeDate": 1277818200, + "regularMarketTime": 1626465603, + "gmtoffset": -14400, + "timezone": "EDT", + "exchangeTimezoneName": "America/New_York", + "regularMarketPrice": 644.22, + "chartPreviousClose": 650.6, + "priceHint": 2, + "currentTradingPeriod": { + "pre": { + "timezone": "EDT", + "start": 1626422400, + "end": 1626442200, + "gmtoffset": -14400 + }, + "regular": { + "timezone": "EDT", + "start": 1626442200, + "end": 1626465600, + "gmtoffset": -14400 + }, + "post": { + "timezone": "EDT", + "start": 1626465600, + "end": 1626480000, + "gmtoffset": -14400 + } + }, + "dataGranularity": "1d", + "range": "1d", + "validRanges": [ + "1d", + "5d", + "1mo", + "3mo", + "6mo", + "1y", + "2y", + "5y", + "10y", + "ytd", + "max" + ] + }, + "timestamp": [ + 1626442200, + 1626465603 + ], + "indicators": { + "quote": [ + { + "close": [ + 644.22, + 644.22 + ] + } + ] + } + } + ] + } + ], + "error": null + } +} From b0834575ed43a20d48fc53fb52dbb00a9e8cf520 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 18 Jul 2021 18:13:43 +0200 Subject: [PATCH 012/149] Version 0.1.6. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fb5a54f..eab3ab4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "0.1.5" +version = "0.1.6" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 1276d02..0a8da88 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "0.1.5" +__version__ = "0.1.6" From 81291cbf2b1d2e0c2f67060188c574c9add24760 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 11:00:29 +0000 Subject: [PATCH 013/149] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..e05a7a4 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,23 @@ +image: python:latest + +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/poetry" + +cache: + paths: + - .cache/pip + - .cache/poetry + +before_script: + - python -V + - pip install poetry + - poetry install + +test: + script: + - poetry run isort --check + - poetry run black --check + - poetry run pytest + - poetry run coverage run --source=pricehist -m pytest + - poetry run coverage report \ No newline at end of file From d3931310a2181cd92eed5f3e9835218ff1a77d86 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 12:31:44 +0200 Subject: [PATCH 015/149] Improve alphavantage error handling and add tests. --- src/pricehist/exceptions.py | 26 +- src/pricehist/sources/alphavantage.py | 212 ++++-- tests/pricehist/sources/test_alphavantage.py | 632 ++++++++++++++++++ .../test_alphavantage/btc-aud-partial.json | 97 +++ .../test_alphavantage/digital-partial.csv | 3 + .../test_alphavantage/eur-aud-partial.json | 60 ++ .../test_alphavantage/ibm-partial.json | 81 +++ .../test_alphavantage/physical-partial.csv | 4 + .../sources/test_alphavantage/search-ibm.json | 114 ++++ 9 files changed, 1185 insertions(+), 44 deletions(-) create mode 100644 tests/pricehist/sources/test_alphavantage.py create mode 100644 tests/pricehist/sources/test_alphavantage/btc-aud-partial.json create mode 100644 tests/pricehist/sources/test_alphavantage/digital-partial.csv create mode 100644 tests/pricehist/sources/test_alphavantage/eur-aud-partial.json create mode 100644 tests/pricehist/sources/test_alphavantage/ibm-partial.json create mode 100644 tests/pricehist/sources/test_alphavantage/physical-partial.csv create mode 100644 tests/pricehist/sources/test_alphavantage/search-ibm.json diff --git a/src/pricehist/exceptions.py b/src/pricehist/exceptions.py index 78d66b7..d7fb6f4 100644 --- a/src/pricehist/exceptions.py +++ b/src/pricehist/exceptions.py @@ -28,10 +28,32 @@ class InvalidType(SourceError, ValueError): self.pair = "/".join([s for s in [base, quote] if s]) message = ( f"Invalid price type '{type}' for pair '{self.pair}'. " - f"Run 'pricehist source {source.id()} " + f"Run 'pricehist source {source.id()}' " f"for information about valid types." ) - super(InvalidPair, self).__init__(message) + super(InvalidType, self).__init__(message) + + +class CredentialsError(SourceError): + """Access credentials are unavailable or invalid.""" + + def __init__(self, keys, source): + self.keys = keys + self.source = source + message = ( + f"Access credentials for source '{source.id()}' are unavailable " + f"""or invalid. Set the environment variables '{"', '".join(keys)}' """ + f"correctly. Run 'pricehist source {source.id()}' for more " + f"information about credentials." + ) + super(CredentialsError, self).__init__(message) + + +class RateLimit(SourceError): + """Source request rate limit reached.""" + + def __init__(self, message): + super(RateLimit, self).__init__(f"{self.__doc__} {message}") class RequestError(SourceError): diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index 7b4db38..a74eeda 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -8,6 +8,7 @@ from decimal import Decimal import requests +from pricehist import exceptions from pricehist.price import Price from .basesource import BaseSource @@ -85,20 +86,38 @@ class AlphaVantage(BaseSource): output_quote = series.quote if series.quote == "": - output_quote = self._stock_currency(output_base) - data = self._stock_data(series) + output_quote, data = self._stock_data(series) else: if series.type == "adjclose": - logging.critical( - "The 'adjclose' price type is only available for stocks. " - "Use 'close' instead." + raise exceptions.InvalidType( + series.type, series.base, series.quote, self ) - exit(1) - elif series.base in [s for s, n in self._physical_symbols()]: + + physical_symbols = [s for s, n in self._physical_symbols()] + + if series.quote not in physical_symbols: + raise exceptions.InvalidPair( + series.base, + series.quote, + self, + "When given, the quote must be a physical currency.", + ) + + if series.base in physical_symbols: data = self._physical_data(series) - else: + + elif series.base in [s for s, n in self._digital_symbols()]: data = self._digital_data(series) + else: + raise exceptions.InvalidPair( + series.base, + series.quote, + self, + "When a quote currency is given, the base must be a known " + "physical or digital currency.", + ) + prices = [ Price(day, amount) for day, entries in data.items() @@ -112,7 +131,7 @@ class AlphaVantage(BaseSource): def _amount(self, day, entries, series): if day < series.start or day > series.end: return None - elif type == "mid": + elif series.type == "mid": return sum([Decimal(entries["high"]), Decimal(entries["low"])]) / 2 else: return Decimal(entries[series.type]) @@ -122,7 +141,7 @@ class AlphaVantage(BaseSource): for match in data["bestMatches"]: if match["1. symbol"] == symbol: return match["8. currency"] - return "Unknown" + return None def _search_data(self, keywords: str): params = { @@ -130,30 +149,87 @@ class AlphaVantage(BaseSource): "keywords": keywords, "apikey": self._apikey(), } - response = self.log_curl(requests.get(self.QUERY_URL, params=params)) - data = json.loads(response.content) + + try: + response = self.log_curl(requests.get(self.QUERY_URL, params=params)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + data = json.loads(response.content) + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if type(data) == dict and "Note" in data and "call frequency" in data["Note"]: + raise exceptions.RateLimit(data["Note"]) + + expected_keys = ["1. symbol", "2. name", "3. type", "4. region", "8. currency"] + if ( + type(data) != dict + or "bestMatches" not in data + or type(data["bestMatches"]) != list + or not all(k in m for k in expected_keys for m in data["bestMatches"]) + ): + raise exceptions.ResponseParsingError("Unexpected content.") + return data def _stock_data(self, series): + output_quote = self._stock_currency(series.base) or "UNKNOWN" + params = { "function": "TIME_SERIES_DAILY_ADJUSTED", "symbol": series.base, "outputsize": self._outputsize(series.start), "apikey": self._apikey(), } - response = self.log_curl(requests.get(self.QUERY_URL, params=params)) - data = json.loads(response.content) - normalized_data = { - day: { - "open": entries["1. open"], - "high": entries["2. high"], - "low": entries["3. low"], - "close": entries["4. close"], - "adjclose": entries["5. adjusted close"], + + try: + response = self.log_curl(requests.get(self.QUERY_URL, params=params)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + data = json.loads(response.content) + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if type(data) == dict and "Note" in data and "call frequency" in data["Note"]: + raise exceptions.RateLimit(data["Note"]) + + if "Error Message" in data: + if output_quote == "UNKNOWN": + raise exceptions.InvalidPair( + series.base, series.quote, self, "Unknown stock symbol." + ) + else: + raise exceptions.BadResponse(data["Error Message"]) + + try: + normalized_data = { + day: { + "open": entries["1. open"], + "high": entries["2. high"], + "low": entries["3. low"], + "close": entries["4. close"], + "adjclose": entries["5. adjusted close"], + } + for day, entries in reversed(data["Time Series (Daily)"].items()) } - for day, entries in reversed(data["Time Series (Daily)"].items()) - } - return normalized_data + except Exception as e: + raise exceptions.ResponseParsingError("Unexpected content.") from e + + return output_quote, normalized_data def _physical_data(self, series): params = { @@ -163,8 +239,28 @@ class AlphaVantage(BaseSource): "outputsize": self._outputsize(series.start), "apikey": self._apikey(), } - response = self.log_curl(requests.get(self.QUERY_URL, params=params)) - data = json.loads(response.content) + + try: + response = self.log_curl(requests.get(self.QUERY_URL, params=params)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + data = json.loads(response.content) + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if type(data) == dict and "Note" in data and "call frequency" in data["Note"]: + raise exceptions.RateLimit(data["Note"]) + + if type(data) != dict or "Time Series FX (Daily)" not in data: + raise exceptions.ResponseParsingError("Unexpected content.") + normalized_data = { day: {k[3:]: v for k, v in entries.items()} for day, entries in reversed(data["Time Series FX (Daily)"].items()) @@ -185,8 +281,28 @@ class AlphaVantage(BaseSource): "market": series.quote, "apikey": self._apikey(), } - response = self.log_curl(requests.get(self.QUERY_URL, params=params)) - data = json.loads(response.content) + + try: + response = self.log_curl(requests.get(self.QUERY_URL, params=params)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + data = json.loads(response.content) + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if type(data) == dict and "Note" in data and "call frequency" in data["Note"]: + raise exceptions.RateLimit(data["Note"]) + + if type(data) != dict or "Time Series (Digital Currency Daily)" not in data: + raise exceptions.ResponseParsingError("Unexpected content.") + normalized_data = { day: { "open": entries[f"1a. open ({series.quote})"], @@ -204,24 +320,36 @@ class AlphaVantage(BaseSource): key_name = "ALPHAVANTAGE_API_KEY" key = os.getenv(key_name) if require and not key: - logging.critical( - f"The environment variable {key_name} is empty. " - "Get a free API key from https://www.alphavantage.co/support/#api-key, " - f'export {key_name}="YOUR_OWN_API_KEY" and retry.' - ) - exit(1) + raise exceptions.CredentialsError([key_name], self) return key def _physical_symbols(self) -> list[(str, str)]: url = "https://www.alphavantage.co/physical_currency_list/" - response = self.log_curl(requests.get(url)) - lines = response.content.decode("utf-8").splitlines() - data = csv.reader(lines[1:], delimiter=",") - return [(s, f"Physical: {n}") for s, n in data] + return self._get_symbols(url, "Physical: ") def _digital_symbols(self) -> list[(str, str)]: url = "https://www.alphavantage.co/digital_currency_list/" - response = self.log_curl(requests.get(url)) - lines = response.content.decode("utf-8").splitlines() - data = csv.reader(lines[1:], delimiter=",") - return [(s, f"Digital: {n}") for s, n in data] + return self._get_symbols(url, "Digital: ") + + def _get_symbols(self, url, prefix) -> list[(str, str)]: + try: + response = self.log_curl(requests.get(url)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + lines = response.content.decode("utf-8").splitlines() + data = csv.reader(lines[1:], delimiter=",") + results = [(s, f"{prefix}{n}") for s, n in data] + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if len(results) == 0: + raise exceptions.ResponseParsingError("Symbols data missing.") + + return results diff --git a/tests/pricehist/sources/test_alphavantage.py b/tests/pricehist/sources/test_alphavantage.py new file mode 100644 index 0000000..b585a7e --- /dev/null +++ b/tests/pricehist/sources/test_alphavantage.py @@ -0,0 +1,632 @@ +import logging +import os +import re +from datetime import datetime, timedelta +from decimal import Decimal +from pathlib import Path + +import pytest +import requests +import responses + +from pricehist import exceptions +from pricehist.price import Price +from pricehist.series import Series +from pricehist.sources.alphavantage import AlphaVantage + +api_key_name = "ALPHAVANTAGE_API_KEY" + + +@pytest.fixture(autouse=True) +def mock_settings_env_vars(monkeypatch): + value = "NOTAREALKEY12345" + if not os.getenv(api_key_name): + monkeypatch.setenv(api_key_name, value, prepend=False) + yield + + +@pytest.fixture +def src(): + return AlphaVantage() + + +@pytest.fixture +def type(src): + return src.types()[0] + + +@pytest.fixture +def requests_mock(): + with responses.RequestsMock() as mock: + yield mock + + +physical_list_url = "https://www.alphavantage.co/physical_currency_list/" +digital_list_url = "https://www.alphavantage.co/digital_currency_list/" + +search_url = re.compile( + r"https://www\.alphavantage\.co/query\?function=SYMBOL_SEARCH.*" +) +stock_url = re.compile( + r"https://www\.alphavantage\.co/query\?function=TIME_SERIES_DAILY_ADJUSTED.*" +) +physical_url = re.compile(r"https://www\.alphavantage\.co/query\?function=FX_DAILY.*") +digital_url = re.compile( + r"https://www\.alphavantage\.co/query\?function=DIGITAL_CURRENCY_DAILY.*" +) + +rate_limit_json = ( + '{ "Note": "' + "Thank you for using Alpha Vantage! Our standard API call frequency is 5 " + "calls per minute and 500 calls per day. Please visit " + "https://www.alphavantage.co/premium/ if you would like to target a higher " + "API call frequency." + '" }' +) + + +@pytest.fixture +def physical_list_ok(requests_mock): + text = (Path(os.path.splitext(__file__)[0]) / "physical-partial.csv").read_text() + requests_mock.add(responses.GET, physical_list_url, body=text, status=200) + yield requests_mock + + +@pytest.fixture +def digital_list_ok(requests_mock): + text = (Path(os.path.splitext(__file__)[0]) / "digital-partial.csv").read_text() + requests_mock.add(responses.GET, digital_list_url, body=text, status=200) + yield requests_mock + + +@pytest.fixture +def search_ok(requests_mock): + text = (Path(os.path.splitext(__file__)[0]) / "search-ibm.json").read_text() + requests_mock.add(responses.GET, search_url, body=text, status=200) + yield requests_mock + + +@pytest.fixture +def search_not_found(requests_mock): + requests_mock.add(responses.GET, search_url, body='{"bestMatches":[]}', status=200) + yield requests_mock + + +@pytest.fixture +def ibm_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "ibm-partial.json").read_text() + requests_mock.add(responses.GET, stock_url, body=json, status=200) + yield requests_mock + + +@pytest.fixture +def euraud_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "eur-aud-partial.json").read_text() + requests_mock.add(responses.GET, physical_url, body=json, status=200) + yield requests_mock + + +@pytest.fixture +def btcaud_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "btc-aud-partial.json").read_text() + requests_mock.add(responses.GET, digital_url, body=json, status=200) + yield requests_mock + + +def test_normalizesymbol(src): + assert src.normalizesymbol("tsla") == "TSLA" + assert src.normalizesymbol("btc") == "BTC" + assert src.normalizesymbol("eur") == "EUR" + + +def test_metadata(src): + assert isinstance(src.id(), str) + assert len(src.id()) > 0 + + assert isinstance(src.name(), str) + assert len(src.name()) > 0 + + assert isinstance(src.description(), str) + assert len(src.description()) > 0 + + assert isinstance(src.source_url(), str) + assert src.source_url().startswith("http") + + assert datetime.strptime(src.start(), "%Y-%m-%d") + + assert isinstance(src.types(), list) + assert len(src.types()) > 0 + assert isinstance(src.types()[0], str) + assert len(src.types()[0]) > 0 + + assert isinstance(src.notes(), str) + + +def test_symbols_stock_message(src, physical_list_ok, digital_list_ok, caplog): + with caplog.at_level(logging.INFO): + src.symbols() + assert any(["Stock symbols can be discovered" in r.message for r in caplog.records]) + + +def test_symbols(src, physical_list_ok, digital_list_ok): + syms = src.symbols() + assert ("BTC", "Digital: Bitcoin") in syms + assert ("AUD", "Physical: Australian Dollar") in syms + assert len(syms) > 2 + + +def test_symbols_digital_network_issue(src, requests_mock): + requests_mock.add( + responses.GET, + digital_list_url, + body=requests.exceptions.ConnectionError("Network issue"), + ) + with pytest.raises(exceptions.RequestError) as e: + src.symbols() + assert "Network issue" in str(e.value) + + +def test_symbols_digital_bad_status(src, requests_mock): + requests_mock.add(responses.GET, digital_list_url, status=500) + with pytest.raises(exceptions.BadResponse) as e: + src.symbols() + assert "Server Error" in str(e.value) + + +def test_symbols_digital_no_data(src, requests_mock): + requests_mock.add(responses.GET, digital_list_url, body="NOT CSV", status=200) + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "Symbols data missing." in str(e.value) + + +def test_symbols_digital_bad_data(src, requests_mock): + requests_mock.add(responses.GET, digital_list_url, body="A,B,C\na,b,c", status=200) + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "too many values" in str(e.value) + + +def test_symbols_physical_network_issue(src, digital_list_ok, requests_mock): + requests_mock.add( + responses.GET, + physical_list_url, + body=requests.exceptions.ConnectionError("Network issue"), + ) + with pytest.raises(exceptions.RequestError) as e: + src.symbols() + assert "Network issue" in str(e.value) + + +def test_symbols_physical_bad_status(src, digital_list_ok, requests_mock): + requests_mock.add(responses.GET, physical_list_url, status=500) + with pytest.raises(exceptions.BadResponse) as e: + src.symbols() + assert "Server Error" in str(e.value) + + +def test_symbols_physical_no_data(src, digital_list_ok, requests_mock): + requests_mock.add(responses.GET, physical_list_url, body="", status=200) + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "Symbols data missing." in str(e.value) + + +def test_search(src, search_ok): + results = src.search("IBM") + req = search_ok.calls[0].request + assert req.params["function"] == "SYMBOL_SEARCH" + assert req.params["keywords"] == "IBM" + assert len(req.params["apikey"]) > 0 + assert len(results) == 10 + for expected in [ + ("IBM", "International Business Machines Corp, Equity, United States, USD"), + ("IBMJ", "iShares iBonds Dec 2021 Term Muni Bond ETF, ETF, United States, USD"), + ("IBMK", "iShares iBonds Dec 2022 Term Muni Bond ETF, ETF, United States, USD"), + ("IBM.DEX", "International Business Machines Corporation, Equity, XETRA, EUR"), + ]: + assert expected in results + + +def test_search_network_issue(src, requests_mock): + requests_mock.add( + responses.GET, + search_url, + body=requests.exceptions.ConnectionError("Network issue"), + ) + with pytest.raises(exceptions.RequestError) as e: + src.search("IBM") + assert "Network issue" in str(e.value) + + +def test_search_bad_status(src, requests_mock): + requests_mock.add(responses.GET, search_url, status=500) + with pytest.raises(exceptions.BadResponse) as e: + src.search("IBM") + assert "Server Error" in str(e.value) + + +def test_search_bad_data(src, requests_mock): + requests_mock.add(responses.GET, search_url, body="NOT JSON", status=200) + with pytest.raises(exceptions.ResponseParsingError) as e: + src.search("IBM") + assert "while parsing data" in str(e.value) + + +def test_search_bad_json(src, requests_mock): + requests_mock.add(responses.GET, search_url, body="{}", status=200) + with pytest.raises(exceptions.ResponseParsingError) as e: + src.search("IBM") + assert "Unexpected content." in str(e.value) + + +def test_search_bad_json_tricky(src, requests_mock): + requests_mock.add( + responses.GET, search_url, body='{"bestMatches": [{}]}', status=200 + ) + with pytest.raises(exceptions.ResponseParsingError) as e: + src.search("IBM") + assert "Unexpected content." in str(e.value) + + +def test_search_rate_limit(src, type, requests_mock): + requests_mock.add(responses.GET, search_url, body=rate_limit_json) + with pytest.raises(exceptions.RateLimit) as e: + src.fetch(Series("IBM", "", type, "2021-01-04", "2021-01-08")) + assert "rate limit" in str(e.value) + + +def test_fetch_stock_known(src, type, search_ok, ibm_ok): + series = src.fetch(Series("IBM", "", type, "2021-01-04", "2021-01-08")) + search_req = search_ok.calls[0].request + stock_req = ibm_ok.calls[1].request + assert search_req.params["function"] == "SYMBOL_SEARCH" + assert search_req.params["keywords"] == "IBM" + assert stock_req.params["function"] == "TIME_SERIES_DAILY_ADJUSTED" + assert stock_req.params["symbol"] == "IBM" + assert stock_req.params["outputsize"] == "full" + assert (series.base, series.quote) == ("IBM", "USD") + assert len(series.prices) == 5 + assert series.prices[0] == Price("2021-01-04", Decimal("123.94")) + assert series.prices[-1] == Price("2021-01-08", Decimal("128.53")) + + +def test_fetch_stock_compact_if_recent(src, type, search_ok, ibm_ok): + today = datetime.now().date() + start = (today - timedelta(days=30)).isoformat() + end = today.isoformat() + src.fetch(Series("IBM", "", type, start, end)) + stock_req = ibm_ok.calls[1].request + assert stock_req.params["outputsize"] == "compact" + + +def test_fetch_stock_requests_logged(src, type, search_ok, ibm_ok, caplog): + with caplog.at_level(logging.DEBUG): + src.fetch(Series("IBM", "", type, "2021-01-04", "2021-01-08")) + logged_requests = 0 + for r in caplog.records: + if r.levelname == "DEBUG" and " curl " in r.message: + logged_requests += 1 + assert logged_requests == 2 + + +def test_fetch_stock_types_all_available(src, search_ok, ibm_ok): + cls = src.fetch(Series("IBM", "", "close", "2021-01-04", "2021-01-08")) + opn = src.fetch(Series("IBM", "", "open", "2021-01-04", "2021-01-08")) + hgh = src.fetch(Series("IBM", "", "high", "2021-01-04", "2021-01-08")) + low = src.fetch(Series("IBM", "", "low", "2021-01-04", "2021-01-08")) + adj = src.fetch(Series("IBM", "", "adjclose", "2021-01-04", "2021-01-08")) + mid = src.fetch(Series("IBM", "", "mid", "2021-01-04", "2021-01-08")) + assert cls.prices[0].amount == Decimal("123.94") + assert opn.prices[0].amount == Decimal("125.85") + assert hgh.prices[0].amount == Decimal("125.9174") + assert low.prices[0].amount == Decimal("123.04") + assert adj.prices[0].amount == Decimal("120.943645029") + assert mid.prices[0].amount == Decimal("124.4787") + + +def test_fetch_stock_type_mid_is_mean_of_low_and_high(src, search_ok, ibm_ok): + hgh = src.fetch(Series("IBM", "", "high", "2021-01-04", "2021-01-08")).prices + low = src.fetch(Series("IBM", "", "low", "2021-01-04", "2021-01-08")).prices + mid = src.fetch(Series("IBM", "", "mid", "2021-01-04", "2021-01-08")).prices + assert all( + [ + mid[i].amount == (sum([low[i].amount, hgh[i].amount]) / 2) + for i in range(0, 5) + ] + ) + + +def test_fetch_stock_bad_sym(src, type, search_not_found, requests_mock): + requests_mock.add( + responses.GET, + stock_url, + status=200, + body="""{ + "Error Message": "Invalid API call. Please retry or..." + }""", + ) + with pytest.raises(exceptions.InvalidPair) as e: + src.fetch(Series("NOTASTOCK", "", type, "2021-01-04", "2021-01-08")) + assert "Unknown stock symbol" in str(e.value) + + +def test_fetch_stock_quote_found_prices_error(src, type, search_ok, requests_mock): + requests_mock.add( + responses.GET, + stock_url, + status=200, + body="""{ + "Error Message": "Invalid API call. Please retry or..." + }""", + ) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("IBM", "", type, "2021-01-04", "2021-01-08")) + assert "bad response" in str(e.value) + + +def test_fetch_stock_network_issue(src, type, search_ok, requests_mock): + body = requests.exceptions.ConnectionError("Network issue") + requests_mock.add(responses.GET, stock_url, body=body) + with pytest.raises(exceptions.RequestError) as e: + src.fetch(Series("IBM", "", type, "2021-01-04", "2021-01-08")) + assert "Network issue" in str(e.value) + + +def test_fetch_stock_bad_status(src, type, search_ok, requests_mock): + requests_mock.add(responses.GET, stock_url, status=500, body="Some other reason") + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("IBM", "", type, "2021-01-04", "2021-01-08")) + assert "Internal Server Error" in str(e.value) + + +def test_fetch_stock_parsing_error(src, type, search_ok, requests_mock): + requests_mock.add(responses.GET, stock_url, body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("IBM", "", type, "2021-01-04", "2021-01-08")) + assert "while parsing data" in str(e.value) + + +def test_fetch_stock_unexpected_json(src, type, search_ok, requests_mock): + requests_mock.add(responses.GET, stock_url, body='{"notdata": []}') + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("IBM", "", type, "2021-01-04", "2021-01-08")) + assert "Unexpected content" in str(e.value) + + +def test_fetch_stock_rate_limit(src, type, search_ok, requests_mock): + requests_mock.add(responses.GET, stock_url, body=rate_limit_json) + with pytest.raises(exceptions.RateLimit) as e: + src.fetch(Series("IBM", "", type, "2021-01-04", "2021-01-08")) + assert "rate limit" in str(e.value) + + +def test_fetch_physical_known(src, type, physical_list_ok, euraud_ok): + series = src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) + req = euraud_ok.calls[1].request + assert req.params["function"] == "FX_DAILY" + assert req.params["from_symbol"] == "EUR" + assert req.params["to_symbol"] == "AUD" + assert req.params["outputsize"] == "full" + assert (series.base, series.quote) == ("EUR", "AUD") + assert len(series.prices) == 5 + assert series.prices[0] == Price("2021-01-04", Decimal("1.59718")) + assert series.prices[-1] == Price("2021-01-08", Decimal("1.57350")) + + +def test_fetch_physical_compact_if_recent(src, type, physical_list_ok, euraud_ok): + today = datetime.now().date() + start = (today - timedelta(days=30)).isoformat() + end = today.isoformat() + src.fetch(Series("EUR", "AUD", type, start, end)) + req = euraud_ok.calls[1].request + assert req.params["outputsize"] == "compact" + + +def test_fetch_physical_requests_logged(src, type, physical_list_ok, euraud_ok, caplog): + with caplog.at_level(logging.DEBUG): + src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) + logged_requests = 0 + for r in caplog.records: + if r.levelname == "DEBUG" and " curl " in r.message: + logged_requests += 1 + assert logged_requests == 2 + + +def test_fetch_physical_types_but_adjclose_available(src, physical_list_ok, euraud_ok): + cls = src.fetch(Series("EUR", "AUD", "close", "2021-01-04", "2021-01-08")) + opn = src.fetch(Series("EUR", "AUD", "open", "2021-01-04", "2021-01-08")) + hgh = src.fetch(Series("EUR", "AUD", "high", "2021-01-04", "2021-01-08")) + low = src.fetch(Series("EUR", "AUD", "low", "2021-01-04", "2021-01-08")) + mid = src.fetch(Series("EUR", "AUD", "mid", "2021-01-04", "2021-01-08")) + assert cls.prices[0].amount == Decimal("1.59718") + assert opn.prices[0].amount == Decimal("1.58741") + assert hgh.prices[0].amount == Decimal("1.60296") + assert low.prices[0].amount == Decimal("1.58550") + assert mid.prices[0].amount == Decimal("1.59423") + + +def test_fetch_physical_adjclose_not_available(src): + with pytest.raises(exceptions.InvalidType) as e: + src.fetch(Series("EUR", "AUD", "adjclose", "2021-01-04", "2021-01-08")) + assert "Invalid price type 'adjclose' for pair 'EUR/AUD'." in str(e) + + +def test_fetch_physical_type_mid_is_mean_of_low_and_high( + src, physical_list_ok, euraud_ok +): + hgh = src.fetch(Series("EUR", "AUD", "high", "2021-01-04", "2021-01-08")).prices + low = src.fetch(Series("EUR", "AUD", "low", "2021-01-04", "2021-01-08")).prices + mid = src.fetch(Series("EUR", "AUD", "mid", "2021-01-04", "2021-01-08")).prices + assert all( + [ + mid[i].amount == (sum([low[i].amount, hgh[i].amount]) / 2) + for i in range(0, 5) + ] + ) + + +def test_fetch_physical_bad_sym(src, type, physical_list_ok, digital_list_ok): + with pytest.raises(exceptions.InvalidPair) as e: + src.fetch(Series("NOTPHYSICAL", "AUD", type, "2021-01-04", "2021-01-08")) + assert "base must be a known physical or digital currency" in str(e.value) + + +def test_fetch_physical_network_issue(src, type, physical_list_ok, requests_mock): + body = requests.exceptions.ConnectionError("Network issue") + requests_mock.add(responses.GET, physical_url, body=body) + with pytest.raises(exceptions.RequestError) as e: + src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) + assert "Network issue" in str(e.value) + + +def test_fetch_physical_bad_status(src, type, physical_list_ok, requests_mock): + requests_mock.add(responses.GET, physical_url, status=500, body="Some other reason") + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) + assert "Internal Server Error" in str(e.value) + + +def test_fetch_physical_parsing_error(src, type, physical_list_ok, requests_mock): + requests_mock.add(responses.GET, physical_url, body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) + assert "while parsing data" in str(e.value) + + +def test_fetch_physical_unexpected_json(src, type, physical_list_ok, requests_mock): + requests_mock.add(responses.GET, physical_url, body='{"notdata": []}') + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) + assert "Unexpected content" in str(e.value) + + +def test_fetch_physical_rate_limit(src, type, physical_list_ok, requests_mock): + requests_mock.add(responses.GET, physical_url, body=rate_limit_json) + with pytest.raises(exceptions.RateLimit) as e: + src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) + assert "rate limit" in str(e.value) + + +def test_fetch_digital_known(src, type, physical_list_ok, digital_list_ok, btcaud_ok): + series = src.fetch(Series("BTC", "AUD", type, "2021-01-04", "2021-01-08")) + req = btcaud_ok.calls[2].request + assert req.params["function"] == "DIGITAL_CURRENCY_DAILY" + assert req.params["symbol"] == "BTC" + assert req.params["market"] == "AUD" + assert (series.base, series.quote) == ("BTC", "AUD") + assert len(series.prices) == 5 + assert series.prices[0] == Price("2021-01-04", Decimal("43406.76014740")) + assert series.prices[-1] == Price("2021-01-08", Decimal("55068.43820140")) + + +def test_fetch_digital_requests_logged( + src, type, physical_list_ok, digital_list_ok, btcaud_ok, caplog +): + with caplog.at_level(logging.DEBUG): + src.fetch(Series("BTC", "AUD", type, "2021-01-04", "2021-01-08")) + logged_requests = 0 + for r in caplog.records: + if r.levelname == "DEBUG" and " curl " in r.message: + logged_requests += 1 + assert logged_requests == 3 + + +def test_fetch_digital_types_but_adjclose_available( + src, physical_list_ok, digital_list_ok, btcaud_ok +): + cls = src.fetch(Series("BTC", "AUD", "close", "2021-01-04", "2021-01-08")) + opn = src.fetch(Series("BTC", "AUD", "open", "2021-01-04", "2021-01-08")) + hgh = src.fetch(Series("BTC", "AUD", "high", "2021-01-04", "2021-01-08")) + low = src.fetch(Series("BTC", "AUD", "low", "2021-01-04", "2021-01-08")) + mid = src.fetch(Series("BTC", "AUD", "mid", "2021-01-04", "2021-01-08")) + assert cls.prices[0].amount == Decimal("43406.76014740") + assert opn.prices[0].amount == Decimal("44779.08784700") + assert hgh.prices[0].amount == Decimal("45593.18400000") + assert low.prices[0].amount == Decimal("38170.72220000") + assert mid.prices[0].amount == Decimal("41881.95310000") + + +def test_fetch_digital_adjclose_not_available(src): + with pytest.raises(exceptions.InvalidType) as e: + src.fetch(Series("BTC", "AUD", "adjclose", "2021-01-04", "2021-01-08")) + assert "Invalid price type 'adjclose' for pair 'BTC/AUD'." in str(e.value) + + +def test_fetch_digital_type_mid_is_mean_of_low_and_high( + src, physical_list_ok, digital_list_ok, btcaud_ok +): + hgh = src.fetch(Series("BTC", "AUD", "high", "2021-01-04", "2021-01-08")).prices + low = src.fetch(Series("BTC", "AUD", "low", "2021-01-04", "2021-01-08")).prices + mid = src.fetch(Series("BTC", "AUD", "mid", "2021-01-04", "2021-01-08")).prices + assert all( + [ + mid[i].amount == (sum([low[i].amount, hgh[i].amount]) / 2) + for i in range(0, 5) + ] + ) + + +def test_fetch_digital_bad_sym(src, type, physical_list_ok, digital_list_ok): + with pytest.raises(exceptions.InvalidPair) as e: + src.fetch(Series("NOTDIGITAL", "AUD", type, "2021-01-04", "2021-01-08")) + assert "base must be a known physical or digital currency" in str(e.value) + + +def test_fetch_digital_network_issue( + src, type, physical_list_ok, digital_list_ok, requests_mock +): + body = requests.exceptions.ConnectionError("Network issue") + requests_mock.add(responses.GET, digital_url, body=body) + with pytest.raises(exceptions.RequestError) as e: + src.fetch(Series("BTC", "AUD", type, "2021-01-04", "2021-01-08")) + assert "Network issue" in str(e.value) + + +def test_fetch_digital_bad_status( + src, type, physical_list_ok, digital_list_ok, requests_mock +): + requests_mock.add(responses.GET, digital_url, status=500, body="Some other reason") + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("BTC", "AUD", type, "2021-01-04", "2021-01-08")) + assert "Internal Server Error" in str(e.value) + + +def test_fetch_digital_parsing_error( + src, type, physical_list_ok, digital_list_ok, requests_mock +): + requests_mock.add(responses.GET, digital_url, body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("BTC", "AUD", type, "2021-01-04", "2021-01-08")) + assert "while parsing data" in str(e.value) + + +def test_fetch_digital_unexpected_json( + src, type, physical_list_ok, digital_list_ok, requests_mock +): + requests_mock.add(responses.GET, digital_url, body='{"notdata": []}') + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("BTC", "AUD", type, "2021-01-04", "2021-01-08")) + assert "Unexpected content" in str(e.value) + + +def test_fetch_digital_rate_limit( + src, type, physical_list_ok, digital_list_ok, requests_mock +): + requests_mock.add(responses.GET, digital_url, body=rate_limit_json) + with pytest.raises(exceptions.RateLimit) as e: + src.fetch(Series("BTC", "AUD", type, "2021-01-04", "2021-01-08")) + assert "rate limit" in str(e.value) + + +def test_fetch_bad_pair_quote_non_physical(src, type, physical_list_ok): + with pytest.raises(exceptions.InvalidPair) as e: + src.fetch(Series("EUR", "BTC", type, "2021-01-04", "2021-01-08")) + assert "quote must be a physical currency" in str(e.value) + + +def test_fetch_api_key_missing(src, type, physical_list_ok, monkeypatch): + monkeypatch.delenv(api_key_name) + with pytest.raises(exceptions.CredentialsError) as e: + src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) + assert "unavailable or invalid" in str(e.value) diff --git a/tests/pricehist/sources/test_alphavantage/btc-aud-partial.json b/tests/pricehist/sources/test_alphavantage/btc-aud-partial.json new file mode 100644 index 0000000..cd6412d --- /dev/null +++ b/tests/pricehist/sources/test_alphavantage/btc-aud-partial.json @@ -0,0 +1,97 @@ +{ + "Meta Data": { + "1. Information": "Daily Prices and Volumes for Digital Currency", + "2. Digital Currency Code": "BTC", + "3. Digital Currency Name": "Bitcoin", + "4. Market Code": "AUD", + "5. Market Name": "Australian Dollar", + "6. Last Refreshed": "2021-07-28 00:00:00", + "7. Time Zone": "UTC" + }, + "Time Series (Digital Currency Daily)": { + "2021-01-09": { + "1a. open (AUD)": "55074.06950240", + "1b. open (USD)": "40586.96000000", + "2a. high (AUD)": "56150.17720000", + "2b. high (USD)": "41380.00000000", + "3a. low (AUD)": "52540.71680000", + "3b. low (USD)": "38720.00000000", + "4a. close (AUD)": "54397.30924680", + "4b. close (USD)": "40088.22000000", + "5. volume": "75785.97967500", + "6. market cap (USD)": "75785.97967500" + }, + "2021-01-08": { + "1a. open (AUD)": "53507.50941120", + "1b. open (USD)": "39432.48000000", + "2a. high (AUD)": "56923.63300000", + "2b. high (USD)": "41950.00000000", + "3a. low (AUD)": "49528.31000000", + "3b. low (USD)": "36500.00000000", + "4a. close (AUD)": "55068.43820140", + "4b. close (USD)": "40582.81000000", + "5. volume": "139789.95749900", + "6. market cap (USD)": "139789.95749900" + }, + "2021-01-07": { + "1a. open (AUD)": "49893.81535840", + "1b. open (USD)": "36769.36000000", + "2a. high (AUD)": "54772.88310000", + "2b. high (USD)": "40365.00000000", + "3a. low (AUD)": "49256.92200000", + "3b. low (USD)": "36300.00000000", + "4a. close (AUD)": "53507.23802320", + "4b. close (USD)": "39432.28000000", + "5. volume": "132825.70043700", + "6. market cap (USD)": "132825.70043700" + }, + "2021-01-06": { + "1a. open (AUD)": "46067.47523820", + "1b. open (USD)": "33949.53000000", + "2a. high (AUD)": "50124.29161740", + "2b. high (USD)": "36939.21000000", + "3a. low (AUD)": "45169.81872000", + "3b. low (USD)": "33288.00000000", + "4a. close (AUD)": "49893.81535840", + "4b. close (USD)": "36769.36000000", + "5. volume": "127139.20131000", + "6. market cap (USD)": "127139.20131000" + }, + "2021-01-05": { + "1a. open (AUD)": "43408.17136500", + "1b. open (USD)": "31989.75000000", + "2a. high (AUD)": "46624.45840000", + "2b. high (USD)": "34360.00000000", + "3a. low (AUD)": "40572.50600000", + "3b. low (USD)": "29900.00000000", + "4a. close (AUD)": "46067.47523820", + "4b. close (USD)": "33949.53000000", + "5. volume": "116049.99703800", + "6. market cap (USD)": "116049.99703800" + }, + "2021-01-04": { + "1a. open (AUD)": "44779.08784700", + "1b. open (USD)": "33000.05000000", + "2a. high (AUD)": "45593.18400000", + "2b. high (USD)": "33600.00000000", + "3a. low (AUD)": "38170.72220000", + "3b. low (USD)": "28130.00000000", + "4a. close (AUD)": "43406.76014740", + "4b. close (USD)": "31988.71000000", + "5. volume": "140899.88569000", + "6. market cap (USD)": "140899.88569000" + }, + "2021-01-03": { + "1a. open (AUD)": "43661.51206300", + "1b. open (USD)": "32176.45000000", + "2a. high (AUD)": "47191.80858340", + "2b. high (USD)": "34778.11000000", + "3a. low (AUD)": "43371.85965060", + "3b. low (USD)": "31962.99000000", + "4a. close (AUD)": "44779.08784700", + "4b. close (USD)": "33000.05000000", + "5. volume": "120957.56675000", + "6. market cap (USD)": "120957.56675000" + } + } +} diff --git a/tests/pricehist/sources/test_alphavantage/digital-partial.csv b/tests/pricehist/sources/test_alphavantage/digital-partial.csv new file mode 100644 index 0000000..dd72cce --- /dev/null +++ b/tests/pricehist/sources/test_alphavantage/digital-partial.csv @@ -0,0 +1,3 @@ +currency code,currency name +BTC,Bitcoin +ETH,Ethereum diff --git a/tests/pricehist/sources/test_alphavantage/eur-aud-partial.json b/tests/pricehist/sources/test_alphavantage/eur-aud-partial.json new file mode 100644 index 0000000..36bfa85 --- /dev/null +++ b/tests/pricehist/sources/test_alphavantage/eur-aud-partial.json @@ -0,0 +1,60 @@ +{ + "Meta Data": { + "1. Information": "Forex Daily Prices (open, high, low, close)", + "2. From Symbol": "EUR", + "3. To Symbol": "AUD", + "4. Output Size": "Full size", + "5. Last Refreshed": "2021-07-27 11:35:00", + "6. Time Zone": "UTC" + }, + "Time Series FX (Daily)": { + "2021-01-11": { + "1. open": "1.57496", + "2. high": "1.58318", + "3. low": "1.57290", + "4. close": "1.57823" + }, + "2021-01-08": { + "1. open": "1.57879", + "2. high": "1.58140", + "3. low": "1.57177", + "4. close": "1.57350" + }, + "2021-01-07": { + "1. open": "1.57901", + "2. high": "1.58650", + "3. low": "1.57757", + "4. close": "1.57893" + }, + "2021-01-06": { + "1. open": "1.58390", + "2. high": "1.58800", + "3. low": "1.57640", + "4. close": "1.57932" + }, + "2021-01-05": { + "1. open": "1.59698", + "2. high": "1.59886", + "3. low": "1.58100", + "4. close": "1.58389" + }, + "2021-01-04": { + "1. open": "1.58741", + "2. high": "1.60296", + "3. low": "1.58550", + "4. close": "1.59718" + }, + "2021-01-01": { + "1. open": "1.58730", + "2. high": "1.58730", + "3. low": "1.58504", + "4. close": "1.58668" + }, + "2020-12-31": { + "1. open": "1.59946", + "2. high": "1.60138", + "3. low": "1.58230", + "4. close": "1.58730" + } + } +} diff --git a/tests/pricehist/sources/test_alphavantage/ibm-partial.json b/tests/pricehist/sources/test_alphavantage/ibm-partial.json new file mode 100644 index 0000000..7329475 --- /dev/null +++ b/tests/pricehist/sources/test_alphavantage/ibm-partial.json @@ -0,0 +1,81 @@ +{ + "Meta Data": { + "1. Information": "Daily Time Series with Splits and Dividend Events", + "2. Symbol": "IBM", + "3. Last Refreshed": "2021-07-20", + "4. Output Size": "Full size", + "5. Time Zone": "US/Eastern" + }, + "Time Series (Daily)": { + "2021-01-11": { + "1. open": "127.95", + "2. high": "129.675", + "3. low": "127.66", + "4. close": "128.58", + "5. adjusted close": "125.471469081", + "6. volume": "5602466", + "7. dividend amount": "0.0000", + "8. split coefficient": "1.0" + }, + "2021-01-08": { + "1. open": "128.57", + "2. high": "129.32", + "3. low": "126.98", + "4. close": "128.53", + "5. adjusted close": "125.422677873", + "6. volume": "4676487", + "7. dividend amount": "0.0000", + "8. split coefficient": "1.0" + }, + "2021-01-07": { + "1. open": "130.04", + "2. high": "130.46", + "3. low": "128.26", + "4. close": "128.99", + "5. adjusted close": "125.871556982", + "6. volume": "4507382", + "7. dividend amount": "0.0000", + "8. split coefficient": "1.0" + }, + "2021-01-06": { + "1. open": "126.9", + "2. high": "131.88", + "3. low": "126.72", + "4. close": "129.29", + "5. adjusted close": "126.164304226", + "6. volume": "7956740", + "7. dividend amount": "0.0000", + "8. split coefficient": "1.0" + }, + "2021-01-05": { + "1. open": "125.01", + "2. high": "126.68", + "3. low": "124.61", + "4. close": "126.14", + "5. adjusted close": "123.090458157", + "6. volume": "6114619", + "7. dividend amount": "0.0000", + "8. split coefficient": "1.0" + }, + "2021-01-04": { + "1. open": "125.85", + "2. high": "125.9174", + "3. low": "123.04", + "4. close": "123.94", + "5. adjusted close": "120.943645029", + "6. volume": "5179161", + "7. dividend amount": "0.0000", + "8. split coefficient": "1.0" + }, + "2020-12-31": { + "1. open": "124.22", + "2. high": "126.03", + "3. low": "123.99", + "4. close": "125.88", + "5. adjusted close": "122.836743878", + "6. volume": "3574696", + "7. dividend amount": "0.0000", + "8. split coefficient": "1.0" + } + } +} diff --git a/tests/pricehist/sources/test_alphavantage/physical-partial.csv b/tests/pricehist/sources/test_alphavantage/physical-partial.csv new file mode 100644 index 0000000..57460d3 --- /dev/null +++ b/tests/pricehist/sources/test_alphavantage/physical-partial.csv @@ -0,0 +1,4 @@ +currency code,currency name +AUD,Australian Dollar +EUR,Euro +USD,United States Dollar diff --git a/tests/pricehist/sources/test_alphavantage/search-ibm.json b/tests/pricehist/sources/test_alphavantage/search-ibm.json new file mode 100644 index 0000000..97e34b3 --- /dev/null +++ b/tests/pricehist/sources/test_alphavantage/search-ibm.json @@ -0,0 +1,114 @@ +{ + "bestMatches": [ + { + "1. symbol": "IBM", + "2. name": "International Business Machines Corp", + "3. type": "Equity", + "4. region": "United States", + "5. marketOpen": "09:30", + "6. marketClose": "16:00", + "7. timezone": "UTC-04", + "8. currency": "USD", + "9. matchScore": "1.0000" + }, + { + "1. symbol": "IBMJ", + "2. name": "iShares iBonds Dec 2021 Term Muni Bond ETF", + "3. type": "ETF", + "4. region": "United States", + "5. marketOpen": "09:30", + "6. marketClose": "16:00", + "7. timezone": "UTC-04", + "8. currency": "USD", + "9. matchScore": "0.8571" + }, + { + "1. symbol": "IBMK", + "2. name": "iShares iBonds Dec 2022 Term Muni Bond ETF", + "3. type": "ETF", + "4. region": "United States", + "5. marketOpen": "09:30", + "6. marketClose": "16:00", + "7. timezone": "UTC-04", + "8. currency": "USD", + "9. matchScore": "0.8571" + }, + { + "1. symbol": "IBML", + "2. name": "iShares iBonds Dec 2023 Term Muni Bond ETF", + "3. type": "ETF", + "4. region": "United States", + "5. marketOpen": "09:30", + "6. marketClose": "16:00", + "7. timezone": "UTC-04", + "8. currency": "USD", + "9. matchScore": "0.8571" + }, + { + "1. symbol": "IBMM", + "2. name": "iShares iBonds Dec 2024 Term Muni Bond ETF", + "3. type": "ETF", + "4. region": "United States", + "5. marketOpen": "09:30", + "6. marketClose": "16:00", + "7. timezone": "UTC-04", + "8. currency": "USD", + "9. matchScore": "0.8571" + }, + { + "1. symbol": "IBMN", + "2. name": "iShares iBonds Dec 2025 Term Muni Bond ETF", + "3. type": "ETF", + "4. region": "United States", + "5. marketOpen": "09:30", + "6. marketClose": "16:00", + "7. timezone": "UTC-04", + "8. currency": "USD", + "9. matchScore": "0.8571" + }, + { + "1. symbol": "IBMO", + "2. name": "iShares iBonds Dec 2026 Term Muni Bond ETF", + "3. type": "ETF", + "4. region": "United States", + "5. marketOpen": "09:30", + "6. marketClose": "16:00", + "7. timezone": "UTC-04", + "8. currency": "USD", + "9. matchScore": "0.8571" + }, + { + "1. symbol": "IBM.FRK", + "2. name": "International Business Machines Corporation", + "3. type": "Equity", + "4. region": "Frankfurt", + "5. marketOpen": "08:00", + "6. marketClose": "20:00", + "7. timezone": "UTC+02", + "8. currency": "EUR", + "9. matchScore": "0.7500" + }, + { + "1. symbol": "IBM.LON", + "2. name": "International Business Machines Corporation", + "3. type": "Equity", + "4. region": "United Kingdom", + "5. marketOpen": "08:00", + "6. marketClose": "16:30", + "7. timezone": "UTC+01", + "8. currency": "USD", + "9. matchScore": "0.7500" + }, + { + "1. symbol": "IBM.DEX", + "2. name": "International Business Machines Corporation", + "3. type": "Equity", + "4. region": "XETRA", + "5. marketOpen": "08:00", + "6. marketClose": "20:00", + "7. timezone": "UTC+02", + "8. currency": "EUR", + "9. matchScore": "0.6667" + } + ] +} From a3709926e46a5f68b8fd21f2a36f4f0f8386d42e Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 12:32:12 +0200 Subject: [PATCH 016/149] Log when there are no results for symbol search. --- src/pricehist/sources/basesource.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pricehist/sources/basesource.py b/src/pricehist/sources/basesource.py index 52f0df8..61d5318 100644 --- a/src/pricehist/sources/basesource.py +++ b/src/pricehist/sources/basesource.py @@ -65,6 +65,9 @@ class BaseSource(ABC): if (symbols := self.search(query)) is None: logging.error(f"Symbol search is not possible for the {self.id()} source.") exit(1) + elif symbols == []: + logging.info(f"No results found for query '{query}'.") + return "" else: width = max([len(sym) for sym, desc in symbols] + [0]) lines = [sym.ljust(width + 4) + desc + "\n" for sym, desc in symbols] From 96315b1b00808c64185655503dc297f5594c4b18 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 13:04:37 +0200 Subject: [PATCH 017/149] Fix isort in CI. --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e05a7a4..bc80711 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,8 +16,8 @@ before_script: test: script: - - poetry run isort --check - - poetry run black --check + - poetry run isort . --check + - poetry run black . --check - poetry run pytest - poetry run coverage run --source=pricehist -m pytest - - poetry run coverage report \ No newline at end of file + - poetry run coverage report From 260fc428f5295a22b34e18478ff623c81812674a Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 13:08:36 +0200 Subject: [PATCH 018/149] Don't isort/black vendored code. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc80711..d38113c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,8 +16,8 @@ before_script: test: script: - - poetry run isort . --check - - poetry run black . --check + - poetry run isort src tests --check + - poetry run black src tests --check - poetry run pytest - poetry run coverage run --source=pricehist -m pytest - poetry run coverage report From 2e12167dac0f20726da04fa843157c00f4157452 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 13:42:34 +0200 Subject: [PATCH 019/149] Add badges to readme. --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c349dda..6f2eb81 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,12 @@ A command-line tool for fetching and formatting historical price data, with support for multiple data sources and output formats. +[![Pipeline status](https://gitlab.com/chrisberkhout/pricehist/badges/master/pipeline.svg)](https://gitlab.com/chrisberkhout/pricehist/-/commits/master) +[![Coverage report](https://gitlab.com/chrisberkhout/pricehist/badges/master/coverage.svg)](https://gitlab.com/chrisberkhout/pricehist/-/commits/master) +[![PyPI version](https://badge.fury.io/py/pricehist.svg)](https://badge.fury.io/py/pricehist) +[![Downloads](https://pepy.tech/badge/pricehist)](https://pepy.tech/project/pricehist) +[![License](https://img.shields.io/pypi/l/pricehist)](https://gitlab.com/chrisberkhout/pricehist/-/blob/master/LICENSE) + ## Installation Install via [pip](https://pip.pypa.io/en/stable/) or From 582b9fe178401cb52d442fab047deb7894a04cce Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 16:17:44 +0200 Subject: [PATCH 020/149] No prefix to curl commands in debug log output. --- src/pricehist/sources/basesource.py | 2 +- tests/pricehist/sources/test_alphavantage.py | 6 +++--- tests/pricehist/sources/test_coindesk.py | 4 ++-- tests/pricehist/sources/test_coinmarketcap.py | 4 ++-- tests/pricehist/sources/test_ecb.py | 4 ++-- tests/pricehist/sources/test_yahoo.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/pricehist/sources/basesource.py b/src/pricehist/sources/basesource.py index 61d5318..3794522 100644 --- a/src/pricehist/sources/basesource.py +++ b/src/pricehist/sources/basesource.py @@ -52,7 +52,7 @@ class BaseSource(ABC): def log_curl(self, response): curl = curlify.to_curl(response.request, compressed=True) - logging.debug(f"Request to {self.id()}: {curl}") + logging.debug(curl) return response def format_symbols(self) -> str: diff --git a/tests/pricehist/sources/test_alphavantage.py b/tests/pricehist/sources/test_alphavantage.py index b585a7e..237f8e9 100644 --- a/tests/pricehist/sources/test_alphavantage.py +++ b/tests/pricehist/sources/test_alphavantage.py @@ -305,7 +305,7 @@ def test_fetch_stock_requests_logged(src, type, search_ok, ibm_ok, caplog): src.fetch(Series("IBM", "", type, "2021-01-04", "2021-01-08")) logged_requests = 0 for r in caplog.records: - if r.levelname == "DEBUG" and " curl " in r.message: + if r.levelname == "DEBUG" and "curl " in r.message: logged_requests += 1 assert logged_requests == 2 @@ -428,7 +428,7 @@ def test_fetch_physical_requests_logged(src, type, physical_list_ok, euraud_ok, src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) logged_requests = 0 for r in caplog.records: - if r.levelname == "DEBUG" and " curl " in r.message: + if r.levelname == "DEBUG" and "curl " in r.message: logged_requests += 1 assert logged_requests == 2 @@ -527,7 +527,7 @@ def test_fetch_digital_requests_logged( src.fetch(Series("BTC", "AUD", type, "2021-01-04", "2021-01-08")) logged_requests = 0 for r in caplog.records: - if r.levelname == "DEBUG" and " curl " in r.message: + if r.levelname == "DEBUG" and "curl " in r.message: logged_requests += 1 assert logged_requests == 3 diff --git a/tests/pricehist/sources/test_coindesk.py b/tests/pricehist/sources/test_coindesk.py index 6987c83..c53f34a 100644 --- a/tests/pricehist/sources/test_coindesk.py +++ b/tests/pricehist/sources/test_coindesk.py @@ -114,7 +114,7 @@ def test_symbols_requests_logged(src, currencies_response_ok, caplog): with caplog.at_level(logging.DEBUG): src.symbols() assert any( - ["DEBUG" == r.levelname and " curl " in r.message for r in caplog.records] + ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] ) @@ -165,7 +165,7 @@ def test_fetch_requests_logged(src, type, recent_response_ok, caplog): with caplog.at_level(logging.DEBUG): src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) assert any( - ["DEBUG" == r.levelname and " curl " in r.message for r in caplog.records] + ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] ) diff --git a/tests/pricehist/sources/test_coinmarketcap.py b/tests/pricehist/sources/test_coinmarketcap.py index ba4cc0f..a7fec0c 100644 --- a/tests/pricehist/sources/test_coinmarketcap.py +++ b/tests/pricehist/sources/test_coinmarketcap.py @@ -132,7 +132,7 @@ def test_symbols_requests_logged(src, crypto_ok, fiat_ok, caplog): src.symbols() logged_requests = 0 for r in caplog.records: - if r.levelname == "DEBUG" and " curl " in r.message: + if r.levelname == "DEBUG" and "curl " in r.message: logged_requests += 1 assert logged_requests == 2 @@ -258,7 +258,7 @@ def test_fetch_requests_logged(src, type, recent_sym_sym_ok, caplog): with caplog.at_level(logging.DEBUG): src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) assert any( - ["DEBUG" == r.levelname and " curl " in r.message for r in caplog.records] + ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] ) diff --git a/tests/pricehist/sources/test_ecb.py b/tests/pricehist/sources/test_ecb.py index 2d747fe..f081342 100644 --- a/tests/pricehist/sources/test_ecb.py +++ b/tests/pricehist/sources/test_ecb.py @@ -105,7 +105,7 @@ def test_symbols_requests_logged_for(src, response_ok, caplog): with caplog.at_level(logging.DEBUG): src.symbols() assert any( - ["DEBUG" == r.levelname and " curl " in r.message for r in caplog.records] + ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] ) @@ -134,7 +134,7 @@ def test_fetch_requests_logged(src, response_ok, caplog): with caplog.at_level(logging.DEBUG): src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) assert any( - ["DEBUG" == r.levelname and " curl " in r.message for r in caplog.records] + ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] ) diff --git a/tests/pricehist/sources/test_yahoo.py b/tests/pricehist/sources/test_yahoo.py index 322e9a5..51f6ca9 100644 --- a/tests/pricehist/sources/test_yahoo.py +++ b/tests/pricehist/sources/test_yahoo.py @@ -124,7 +124,7 @@ def test_fetch_requests_logged(src, type, spark_ok, recent_ok, caplog): src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) logged_requests = 0 for r in caplog.records: - if r.levelname == "DEBUG" and " curl " in r.message: + if r.levelname == "DEBUG" and "curl " in r.message: logged_requests += 1 assert logged_requests == 2 From c912b676b4c28d35cd9f779f94a541cabf8ed3f3 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 16:22:06 +0200 Subject: [PATCH 021/149] Use a common exceptions handler when interacting with a source. --- src/pricehist/exceptions.py | 15 ++++++++++++++ src/pricehist/fetch.py | 7 +------ src/pricehist/sources/basesource.py | 32 +++++++++++++++++++---------- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/pricehist/exceptions.py b/src/pricehist/exceptions.py index d7fb6f4..5ac7aa7 100644 --- a/src/pricehist/exceptions.py +++ b/src/pricehist/exceptions.py @@ -1,3 +1,18 @@ +import logging +import sys +from contextlib import contextmanager + + +@contextmanager +def handler(): + try: + yield + except SourceError as e: + logging.debug("Critical exception encountered", exc_info=e) + logging.critical(str(e)) + sys.exit(1) + + class SourceError(Exception): """Base exception for errors rased by sources""" diff --git a/src/pricehist/fetch.py b/src/pricehist/fetch.py index e1cc341..71bcdf3 100644 --- a/src/pricehist/fetch.py +++ b/src/pricehist/fetch.py @@ -1,5 +1,4 @@ import logging -import sys from datetime import date, datetime, timedelta from pricehist import exceptions @@ -12,12 +11,8 @@ def fetch(series, source, output, invert: bool, quantize: int, fmt) -> str: f"source start date of {source.start()}." ) - try: + with exceptions.handler(): series = source.fetch(series) - except exceptions.SourceError as e: - logging.debug("Critical exception encountered", exc_info=e) - logging.critical(str(e)) - sys.exit(1) if len(series.prices) == 0: logging.warn(f"No data found for the interval [{series.start}--{series.end}].") diff --git a/src/pricehist/sources/basesource.py b/src/pricehist/sources/basesource.py index 3794522..01fc6cd 100644 --- a/src/pricehist/sources/basesource.py +++ b/src/pricehist/sources/basesource.py @@ -1,9 +1,11 @@ import logging +import sys from abc import ABC, abstractmethod from textwrap import TextWrapper import curlify +from pricehist import exceptions from pricehist.series import Series @@ -56,13 +58,18 @@ class BaseSource(ABC): return response def format_symbols(self) -> str: - symbols = self.symbols() + with exceptions.handler(): + symbols = self.symbols() + width = max([len(sym) for sym, desc in symbols] + [0]) lines = [sym.ljust(width + 4) + desc + "\n" for sym, desc in symbols] return "".join(lines) def format_search(self, query) -> str: - if (symbols := self.search(query)) is None: + with exceptions.handler(): + symbols = self.search(query) + + if symbols is None: logging.error(f"Symbol search is not possible for the {self.id()} source.") exit(1) elif symbols == []: @@ -75,15 +82,18 @@ class BaseSource(ABC): def format_info(self, total_width=80) -> str: k_width = 11 - parts = [ - self._fmt_field("ID", self.id(), k_width, total_width), - self._fmt_field("Name", self.name(), k_width, total_width), - self._fmt_field("Description", self.description(), k_width, total_width), - self._fmt_field("URL", self.source_url(), k_width, total_width, False), - self._fmt_field("Start", self.start(), k_width, total_width), - self._fmt_field("Types", ", ".join(self.types()), k_width, total_width), - self._fmt_field("Notes", self.notes(), k_width, total_width), - ] + with exceptions.handler(): + parts = [ + self._fmt_field("ID", self.id(), k_width, total_width), + self._fmt_field("Name", self.name(), k_width, total_width), + self._fmt_field( + "Description", self.description(), k_width, total_width + ), + self._fmt_field("URL", self.source_url(), k_width, total_width, False), + self._fmt_field("Start", self.start(), k_width, total_width), + self._fmt_field("Types", ", ".join(self.types()), k_width, total_width), + self._fmt_field("Notes", self.notes(), k_width, total_width), + ] return "\n".join(filter(None, parts)) def _fmt_field(self, key, value, key_width, total_width, force=True): From 09b7a25f9d15572955fcaa6d427981e65f31e91b Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 16:42:21 +0200 Subject: [PATCH 022/149] Test isocurrencies. --- src/pricehist/isocurrencies.py | 8 +++---- tests/pricehist/test_isocurrencies.py | 33 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 tests/pricehist/test_isocurrencies.py diff --git a/src/pricehist/isocurrencies.py b/src/pricehist/isocurrencies.py index fadee06..f7079c4 100644 --- a/src/pricehist/isocurrencies.py +++ b/src/pricehist/isocurrencies.py @@ -90,13 +90,11 @@ def _parse(entry): except (IndexError, ValueError): minor_units = None - name_tags = entry.cssselect("CcyNm") - if name_tags: + name = None + is_fund = None + if name_tags := entry.cssselect("CcyNm"): name = name_tags[0].text is_fund = name_tags[0].attrib.get("IsFund", "").upper() in ["TRUE", "WAHR"] - else: - name = None - is_fund = None countries = [t.text for t in entry.cssselect("CtryNm")] diff --git a/tests/pricehist/test_isocurrencies.py b/tests/pricehist/test_isocurrencies.py new file mode 100644 index 0000000..5e79c97 --- /dev/null +++ b/tests/pricehist/test_isocurrencies.py @@ -0,0 +1,33 @@ +from datetime import datetime + +from pricehist import isocurrencies + + +def test_current(): + currency = isocurrencies.by_code()["EUR"] + assert currency.code == "EUR" + assert currency.number == 978 + assert currency.minor_units == 2 + assert currency.name == "Euro" + assert "GERMANY" in currency.countries + assert "FRANCE" in currency.countries + assert not currency.is_fund + assert not currency.historical + assert not currency.withdrawal_date + + +def test_historical(): + currency = isocurrencies.by_code()["DEM"] + assert currency.code == "DEM" + assert currency.number == 276 + assert currency.minor_units is None + assert currency.name == "Deutsche Mark" + assert "GERMANY" in currency.countries + assert not currency.is_fund + assert currency.historical + assert currency.withdrawal_date == "2002-03" + + +def test_data_dates(): + assert datetime.strptime(isocurrencies.current_data_date(), "%Y-%m-%d") + assert datetime.strptime(isocurrencies.historical_data_date(), "%Y-%m-%d") From c13e329208dbc74860b6d6d80fb808d6129e0c4f Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 17:04:32 +0200 Subject: [PATCH 023/149] Test exceptions handler. --- tests/pricehist/test_exceptions.py | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/pricehist/test_exceptions.py diff --git a/tests/pricehist/test_exceptions.py b/tests/pricehist/test_exceptions.py new file mode 100644 index 0000000..9837d03 --- /dev/null +++ b/tests/pricehist/test_exceptions.py @@ -0,0 +1,43 @@ +import logging +import sys + +import pytest + +from pricehist import exceptions + + +def test_handler_logs_debug_information(caplog): + with caplog.at_level(logging.DEBUG): + try: + with exceptions.handler(): + raise exceptions.RequestError("Some message") + except SystemExit: + pass + + assert caplog.records[0].levelname == "DEBUG" + assert "exception encountered" in caplog.records[0].message + assert caplog.records[0].exc_info + + +def test_handler_exits_nonzero(caplog): + with pytest.raises(SystemExit) as e: + with exceptions.handler(): + raise exceptions.RequestError("Some message") + + assert e.value.code == 1 + + +def test_handler_logs_critical_information(caplog): + with caplog.at_level(logging.CRITICAL): + try: + with exceptions.handler(): + raise exceptions.RequestError("Some message") + except SystemExit: + pass + + assert any( + [ + "CRITICAL" == r.levelname and "Some message" in r.message + for r in caplog.records + ] + ) From 1124b6f86c5ea995dc1c13db80cb1a78282d3418 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 29 Jul 2021 17:15:36 +0200 Subject: [PATCH 024/149] Test formatting of list of sources. --- tests/pricehist/test_sources.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/pricehist/test_sources.py diff --git a/tests/pricehist/test_sources.py b/tests/pricehist/test_sources.py new file mode 100644 index 0000000..977cb79 --- /dev/null +++ b/tests/pricehist/test_sources.py @@ -0,0 +1,16 @@ +import re + +from pricehist import sources + + +def test_formatted_includes_ecb(): + lines = sources.formatted().splitlines() + assert any(re.match(r"ecb +European Central Bank", line) for line in lines) + + +def test_formatted_names_aligned(): + lines = sources.formatted().splitlines() + offsets = [len(re.match(r"(\w+ +)[^ ]", line)[1]) for line in lines] + first = offsets[0] + assert first > 1 + assert all(offset == first for offset in offsets) From 4f6dafcbeb092d7faa22672f542f790498cd99ba Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 30 Jul 2021 14:09:47 +0200 Subject: [PATCH 025/149] Test Series. --- tests/pricehist/test_series.py | 95 ++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 tests/pricehist/test_series.py diff --git a/tests/pricehist/test_series.py b/tests/pricehist/test_series.py new file mode 100644 index 0000000..2cee45c --- /dev/null +++ b/tests/pricehist/test_series.py @@ -0,0 +1,95 @@ +from dataclasses import replace +from decimal import Decimal + +import pytest + +from pricehist.price import Price +from pricehist.series import Series + + +@pytest.fixture +def series(): + return Series( + "BASE", + "QUOTE", + "type", + "2021-01-01", + "2021-06-30", + [ + Price("2021-01-01", Decimal("1.0123456789")), + Price("2021-01-02", Decimal("2.01234567890123456789")), + Price("2021-01-03", Decimal("3.012345678901234567890123456789")), + ], + ) + + +def test_invert(series): + result = series.invert() + assert (series.base, series.quote) == ("BASE", "QUOTE") + assert (result.base, result.quote) == ("QUOTE", "BASE") + + +def test_rename_base(series): + result = series.rename_base("NEWBASE") + assert series.base == "BASE" + assert result.base == "NEWBASE" + + +def test_rename_quote(series): + result = series.rename_quote("NEWQUOTE") + assert series.quote == "QUOTE" + assert result.quote == "NEWQUOTE" + + +def test_quantize_rounds_half_even(series): + subject = replace( + series, + prices=[ + Price("2021-01-01", Decimal("1.14")), + Price("2021-01-02", Decimal("2.25")), + Price("2021-01-03", Decimal("3.35")), + Price("2021-01-04", Decimal("4.46")), + ], + ) + amounts = [p.amount for p in subject.quantize(1).prices] + assert amounts == [ + Decimal("1.1"), + Decimal("2.2"), + Decimal("3.4"), + Decimal("4.5"), + ] + + +def test_quantize_does_not_extend(series): + subject = replace( + series, + prices=[ + Price("2021-01-01", Decimal("1.14")), + Price("2021-01-02", Decimal("2.25")), + Price("2021-01-03", Decimal("3.35")), + Price("2021-01-04", Decimal("4.46")), + ], + ) + amounts = [p.amount for p in subject.quantize(3).prices] + assert amounts == [ + Decimal("1.14"), + Decimal("2.25"), + Decimal("3.35"), + Decimal("4.46"), + ] + + +def test_quantize_does_not_go_beyond_context_max_prec(series): + subject = replace( + series, + prices=[ + Price("2021-01-01", Decimal("1.012345678901234567890123456789")), + ], + ) + assert subject.prices[0].amount == Decimal("1.012345678901234567890123456789") + result0 = subject.quantize(26) + result1 = subject.quantize(27) + result2 = subject.quantize(35) + assert result0.prices[0].amount == Decimal("1.01234567890123456789012346") + assert result1.prices[0].amount == Decimal("1.012345678901234567890123457") + assert result2.prices[0].amount == Decimal("1.012345678901234567890123457") From bc482957ec7eb399235c1cd32e1d23d4e32b05c4 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 30 Jul 2021 14:23:44 +0200 Subject: [PATCH 026/149] Fix flake8 warnings. --- src/pricehist/outputs/beancount.py | 6 ++++-- src/pricehist/outputs/ledger.py | 6 ++++-- src/pricehist/sources/basesource.py | 1 - tests/pricehist/test_exceptions.py | 1 - 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/pricehist/outputs/beancount.py b/src/pricehist/outputs/beancount.py index 366cc86..6ebd31e 100644 --- a/src/pricehist/outputs/beancount.py +++ b/src/pricehist/outputs/beancount.py @@ -10,9 +10,11 @@ the Beancount format in mind. Relevant sections of the Beancount documentation: -* `Commodities / Currencies `_ +* `Commodities / Currencies + `_ * `Prices `_ -* `Fetching Prices in Beancount `_ +* `Fetching Prices in Beancount + `_ Classes: diff --git a/src/pricehist/outputs/ledger.py b/src/pricehist/outputs/ledger.py index 6682677..84fcf81 100644 --- a/src/pricehist/outputs/ledger.py +++ b/src/pricehist/outputs/ledger.py @@ -11,8 +11,10 @@ format. Relevant sections of the Ledger manual: -* `Commodities and Currencies `_ -* `Commoditized Amounts `_ +* `Commodities and Currencies + `_ +* `Commoditized Amounts + `_ Relevant sections of the hledger manual: diff --git a/src/pricehist/sources/basesource.py b/src/pricehist/sources/basesource.py index 01fc6cd..4273678 100644 --- a/src/pricehist/sources/basesource.py +++ b/src/pricehist/sources/basesource.py @@ -1,5 +1,4 @@ import logging -import sys from abc import ABC, abstractmethod from textwrap import TextWrapper diff --git a/tests/pricehist/test_exceptions.py b/tests/pricehist/test_exceptions.py index 9837d03..525cbad 100644 --- a/tests/pricehist/test_exceptions.py +++ b/tests/pricehist/test_exceptions.py @@ -1,5 +1,4 @@ import logging -import sys import pytest From 944265a7e90ee686769ad5a8d04441c57e1b32f8 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 31 Jul 2021 12:01:49 +0200 Subject: [PATCH 027/149] Test format. --- tests/pricehist/test_format.py | 57 ++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/pricehist/test_format.py diff --git a/tests/pricehist/test_format.py b/tests/pricehist/test_format.py new file mode 100644 index 0000000..2795037 --- /dev/null +++ b/tests/pricehist/test_format.py @@ -0,0 +1,57 @@ +from collections import namedtuple +from decimal import Decimal + +from pricehist.format import Format + + +def test_fromargs(): + arg_values = { + "formatquote": None, + "formattime": "23:59:59", + "formatdecimal": None, + "formatthousands": None, + "formatsymbol": None, + "formatdatesep": None, + "formatcsvdelim": None, + "formatbase": None, + } + args = namedtuple("args", arg_values.keys())(**arg_values) + fmt = Format.fromargs(args) + assert fmt.time == "23:59:59" + assert fmt.symbol == "rightspace" + + +def test_format_date(): + assert Format().format_date("2021-01-01") == "2021-01-01" + assert Format(datesep="/").format_date("2021-01-01") == "2021/01/01" + + +def test_format_quote_amount(): + assert ( + Format(decimal=",").format_quote_amount("USD", Decimal("1234.5678")) + == "1234,5678 USD" + ) + assert ( + Format(symbol="rightspace").format_quote_amount("USD", Decimal("1234.5678")) + == "1234.5678 USD" + ) + assert ( + Format(symbol="right").format_quote_amount("€", Decimal("1234.5678")) + == "1234.5678€" + ) + assert ( + Format(symbol="leftspace").format_quote_amount("£", Decimal("1234.5678")) + == "£ 1234.5678" + ) + assert ( + Format(symbol="left").format_quote_amount("$", Decimal("1234.5678")) + == "$1234.5678" + ) + + +def test_format_num(): + assert Format().format_num(Decimal("1234.5678")) == "1234.5678" + assert ( + Format(decimal=",", thousands=".").format_num(Decimal("1234.5678")) + == "1.234,5678" + ) From 5053d57fec6530a4380a698be77fd1137ee01557 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 31 Jul 2021 12:02:00 +0200 Subject: [PATCH 028/149] Test logger. --- tests/pricehist/test_logger.py | 74 ++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/pricehist/test_logger.py diff --git a/tests/pricehist/test_logger.py b/tests/pricehist/test_logger.py new file mode 100644 index 0000000..c431d45 --- /dev/null +++ b/tests/pricehist/test_logger.py @@ -0,0 +1,74 @@ +import logging +import sys + +from pricehist import logger + + +class Record: + pass + + +def test_formatter_no_prefix_for_info(): + record = Record() + record.levelno = logging.INFO + record.levelname = "INFO" + record.msg = "A message %s" + record.args = "for you" + record.exc_info = None + record.exc_text = "" + + s = logger.Formatter().format(record) + + assert s == "A message for you" + + +def test_formatter_prefix_for_other_levels(): + record = Record() + record.levelno = logging.WARNING + record.levelname = "WARNING" + record.msg = "A warning %s" + record.args = "for you" + record.exc_info = None + record.exc_text = "" + + s = logger.Formatter().format(record) + + assert s == "WARNING A warning for you" + + +def test_formatter_formats_given_exception(): + + try: + raise Exception("Something happened") + except Exception: + exc_info = sys.exc_info() + + record = Record() + record.levelno = logging.DEBUG + record.levelname = "DEBUG" + record.msg = "An exception %s:" + record.args = "for you" + record.exc_info = exc_info + record.exc_text = "" + + s = logger.Formatter().format(record) + lines = s.splitlines() + + assert "DEBUG An exception for you:" in lines + assert "DEBUG Traceback (most recent call last):" in lines + assert any('DEBUG File "' in line for line in lines) + assert "DEBUG Exception: Something happened" in lines + + +def test_init_sets_dest_formatter_and_level(capfd): + logger.init() + logging.info("Test message") + out, err = capfd.readouterr() + assert "Test message" not in out + assert "Test message" in err.splitlines() + assert logging.root.level == logging.INFO + + +def test_show_debug(): + logger.show_debug() + assert logging.root.level == logging.DEBUG From de4a8f22278b5979688e7042002a7c2565585cc4 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 1 Aug 2021 14:03:42 +0200 Subject: [PATCH 029/149] Add pytest-mock. --- poetry.lock | 20 +++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 63df0c9..f5f3ffb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -267,6 +267,20 @@ toml = "*" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +name = "pytest-mock" +version = "3.6.1" +description = "Thin-wrapper around the mock package for easier use with pytest" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "tox", "pytest-asyncio"] + [[package]] name = "regex" version = "2021.4.4" @@ -357,7 +371,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "da28bdba188bda9b955066da37862253e0aebadb52fa2d8aeecdea5ad3165efd" +content-hash = "4434fa4fdfb1a7f4d9b833dd611330228497cdd37c0502de9d83f64752dd6480" [metadata.files] appdirs = [ @@ -547,6 +561,10 @@ pytest = [ {file = "pytest-6.2.2-py3-none-any.whl", hash = "sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839"}, {file = "pytest-6.2.2.tar.gz", hash = "sha256:9d1edf9e7d0b84d72ea3dbcdfd22b35fb543a5e8f2a60092dd578936bf63d7f9"}, ] +pytest-mock = [ + {file = "pytest-mock-3.6.1.tar.gz", hash = "sha256:40217a058c52a63f1042f0784f62009e976ba824c418cced42e88d5f40ab0e62"}, + {file = "pytest_mock-3.6.1-py3-none-any.whl", hash = "sha256:30c2f2cc9759e76eee674b81ea28c9f0b94f8f0445a1b87762cadf774f0df7e3"}, +] regex = [ {file = "regex-2021.4.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:619d71c59a78b84d7f18891fe914446d07edd48dc8328c8e149cbe0929b4e000"}, {file = "regex-2021.4.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:47bf5bf60cf04d72bf6055ae5927a0bd9016096bf3d742fa50d9bf9f45aa0711"}, diff --git a/pyproject.toml b/pyproject.toml index eab3ab4..8daa1dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ flake8 = "^3.9.1" isort = "^5.8.0" responses = "^0.13.3" coverage = "^5.5" +pytest-mock = "^3.6.1" [build-system] requires = ["poetry-core>=1.0.0"] From 3218338bff8051f649aabf2855def71e6a5da0eb Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 1 Aug 2021 14:04:04 +0200 Subject: [PATCH 030/149] Use logging.warning() instead of deprecated logging.warn(). --- src/pricehist/fetch.py | 8 +++++--- src/pricehist/outputs/gnucashsql.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/pricehist/fetch.py b/src/pricehist/fetch.py index 71bcdf3..e56a92b 100644 --- a/src/pricehist/fetch.py +++ b/src/pricehist/fetch.py @@ -6,7 +6,7 @@ from pricehist import exceptions def fetch(series, source, output, invert: bool, quantize: int, fmt) -> str: if series.start < source.start(): - logging.warn( + logging.warning( f"The start date {series.start} preceeds the {source.name()} " f"source start date of {source.start()}." ) @@ -15,7 +15,9 @@ def fetch(series, source, output, invert: bool, quantize: int, fmt) -> str: series = source.fetch(series) if len(series.prices) == 0: - logging.warn(f"No data found for the interval [{series.start}--{series.end}].") + logging.warning( + f"No data found for the interval [{series.start}--{series.end}]." + ) else: first = series.prices[0].date last = series.prices[-1].date @@ -28,7 +30,7 @@ def fetch(series, source, output, invert: bool, quantize: int, fmt) -> str: if first == series.start and last == expected_end: logging.debug(message) # Missing today's price is expected else: - logging.warn(message) + logging.warning(message) if invert: series = series.invert() diff --git a/src/pricehist/outputs/gnucashsql.py b/src/pricehist/outputs/gnucashsql.py index 0701908..1cb536b 100644 --- a/src/pricehist/outputs/gnucashsql.py +++ b/src/pricehist/outputs/gnucashsql.py @@ -110,7 +110,7 @@ class GnuCashSQL(BaseOutput): if too_big: # https://code.gnucash.org/docs/MAINT/group__Numeric.html # https://code.gnucash.org/docs/MAINT/structgnc__price__s.html - logging.warn( + logging.warning( "This SQL contains numbers outside of the int64 range required " "by GnuCash for the numerators and denominators of prices. " "Using the --quantize option to limit the number of decimal " @@ -132,7 +132,7 @@ class GnuCashSQL(BaseOutput): def _warn_about_backslashes(self, fields): hits = [name for name, value in fields.items() if "\\" in value] if hits: - logging.warn( + logging.warning( f"Before running this SQL, check the formatting of the " f"{self._english_join(hits)} strings. " f"SQLite treats backslahes in strings as plain characters, but " From c8337b9c2ce16ea78b52995cae76cbb595806eaa Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 1 Aug 2021 18:05:27 +0200 Subject: [PATCH 031/149] Test fetch. --- src/pricehist/fetch.py | 10 +- tests/pricehist/test_fetch.py | 199 ++++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 4 deletions(-) create mode 100644 tests/pricehist/test_fetch.py diff --git a/src/pricehist/fetch.py b/src/pricehist/fetch.py index e56a92b..db051de 100644 --- a/src/pricehist/fetch.py +++ b/src/pricehist/fetch.py @@ -21,16 +21,18 @@ def fetch(series, source, output, invert: bool, quantize: int, fmt) -> str: else: first = series.prices[0].date last = series.prices[-1].date + message = ( + f"Available data covers the interval [{first}--{last}], " + f"{_cov_description(series.start, series.end, first, last)}." + ) if first > series.start or last < series.end: - message = ( - f"Available data covers the interval [{first}--{last}], " - f"{_cov_description(series.start, series.end, first, last)}." - ) expected_end = _yesterday() if series.end == _today() else series.end if first == series.start and last == expected_end: logging.debug(message) # Missing today's price is expected else: logging.warning(message) + else: + logging.debug(message) if invert: series = series.invert() diff --git a/tests/pricehist/test_fetch.py b/tests/pricehist/test_fetch.py new file mode 100644 index 0000000..89759f6 --- /dev/null +++ b/tests/pricehist/test_fetch.py @@ -0,0 +1,199 @@ +import logging +from datetime import date, timedelta +from decimal import Decimal + +import pytest + +from pricehist import exceptions +from pricehist.fetch import fetch +from pricehist.format import Format +from pricehist.price import Price +from pricehist.series import Series +from pricehist.sources.basesource import BaseSource + + +@pytest.fixture +def res_series(mocker): + series = mocker.MagicMock() + series.start = "2021-01-01" + series.end = "2021-01-03" + return series + + +@pytest.fixture +def source(res_series, mocker): + source = mocker.MagicMock(BaseSource) + source.start = mocker.MagicMock(return_value="2021-01-01") + source.fetch = mocker.MagicMock(return_value=res_series) + return source + + +@pytest.fixture +def output(mocker): + output = mocker.MagicMock() + output.format = mocker.MagicMock(return_value="") + return output + + +@pytest.fixture +def fmt(mocker): + return Format() + + +def test_fetch_warns_if_start_before_source_start(source, output, fmt, mocker, caplog): + req_series = Series("BTC", "EUR", "close", "2020-12-31", "2021-01-03") + source.start = mocker.MagicMock(return_value="2021-01-01") + with caplog.at_level(logging.INFO): + fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) + assert any( + [ + "WARNING" == r.levelname and "start date 2020-12-31 preceeds" in r.message + for r in caplog.records + ] + ) + + +def test_fetch_returns_formatted_output(source, res_series, output, fmt, mocker): + req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") + output.format = mocker.MagicMock(return_value="rendered output") + + result = fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) + + output.format.assert_called_once_with(res_series, source, fmt=fmt) + assert result == "rendered output" + + +def test_fetch_inverts_if_requested(source, res_series, output, fmt, mocker): + req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") + inv_series = mocker.MagicMock(res_series) + res_series.invert = mocker.MagicMock(return_value=inv_series) + + fetch(req_series, source, output, invert=True, quantize=None, fmt=fmt) + + res_series.invert.assert_called_once_with() + output.format.assert_called_once_with(inv_series, source, fmt=fmt) + + +def test_fetch_quantizes_if_requested(source, res_series, output, fmt, mocker): + req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") + qnt_series = mocker.MagicMock(res_series) + res_series.quantize = mocker.MagicMock(return_value=qnt_series) + + fetch(req_series, source, output, invert=False, quantize=2, fmt=fmt) + + res_series.quantize.assert_called_once_with(2) + output.format.assert_called_once_with(qnt_series, source, fmt=fmt) + + +def test_fetch_warns_if_no_data(source, res_series, output, fmt, mocker, caplog): + req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") + res_series.prices = mocker.MagicMock(return_value=[]) + with caplog.at_level(logging.INFO): + fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) + assert any( + [ + "WARNING" == r.levelname and "No data found" in r.message + for r in caplog.records + ] + ) + + +def test_fetch_warns_if_missing_data_at_start(source, res_series, output, fmt, caplog): + req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") + res_series.prices = [ + Price("2021-01-02", Decimal("1.2")), + Price("2021-01-03", Decimal("1.3")), + ] + with caplog.at_level(logging.INFO): + fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) + r = caplog.records[0] + assert r.levelname == "WARNING" + assert r.message == ( + "Available data covers the interval [2021-01-02--2021-01-03], " + "which starts 1 day later than requested." + ) + + +def test_fetch_warns_if_missing_data_at_end(source, res_series, output, fmt, caplog): + req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") + res_series.prices = [Price("2021-01-01", Decimal("1.1"))] + with caplog.at_level(logging.INFO): + fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) + r = caplog.records[0] + assert r.levelname == "WARNING" + assert r.message == ( + "Available data covers the interval [2021-01-01--2021-01-01], " + "which ends 2 days earlier than requested." + ) + + +def test_fetch_warns_if_missing_data_at_both_ends( + source, res_series, output, fmt, caplog +): + req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") + res_series.prices = [Price("2021-01-02", Decimal("1.2"))] + with caplog.at_level(logging.INFO): + fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) + r = caplog.records[0] + assert r.levelname == "WARNING" + assert r.message == ( + "Available data covers the interval [2021-01-02--2021-01-02], " + "which starts 1 day later and ends 1 day earlier than requested." + ) + + +def test_fetch_debug_not_warning_message_if_only_today_missing( + source, res_series, output, fmt, caplog +): + start = (date.today() - timedelta(days=2)).isoformat() + yesterday = (date.today() - timedelta(days=1)).isoformat() + today = date.today().isoformat() + req_series = Series("BTC", "EUR", "close", start, today) + res_series.start = start + res_series.end = today + res_series.prices = [Price(start, Decimal("1.1")), Price(yesterday, Decimal("1.2"))] + with caplog.at_level(logging.DEBUG): + fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) + r = caplog.records[0] + assert r.levelname == "DEBUG" + assert r.message == ( + f"Available data covers the interval [{start}--{yesterday}], " + "which ends 1 day earlier than requested." + ) + + +def test_fetch_debug_not_warning_message_if_as_requested( + source, res_series, output, fmt, caplog +): + req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") + res_series.prices = [ + Price("2021-01-01", Decimal("1.1")), + Price("2021-01-02", Decimal("1.2")), + Price("2021-01-03", Decimal("1.3")), + ] + with caplog.at_level(logging.DEBUG): + fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) + r = caplog.records[0] + assert r.levelname == "DEBUG" + assert r.message == ( + "Available data covers the interval [2021-01-01--2021-01-03], as requested." + ) + + +def test_fetch_handles_source_exceptions(source, output, fmt, mocker, caplog): + req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") + + def side_effect(_): + raise exceptions.RequestError("something strange") + + source.fetch = mocker.MagicMock(side_effect=side_effect) + + with caplog.at_level(logging.INFO): + with pytest.raises(SystemExit) as e: + fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) + + r = caplog.records[0] + assert r.levelname == "CRITICAL" + assert "something strange" in r.message + + assert e.value.code == 1 From f21636c39758c7780930faa98fa07e4fe3afbcaf Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 1 Aug 2021 18:11:38 +0200 Subject: [PATCH 032/149] Version 0.1.7. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8daa1dc..193c1fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "0.1.6" +version = "0.1.7" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 0a8da88..f1380ee 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "0.1.6" +__version__ = "0.1.7" From 16d0405725f7f4f89ec1020e4050890f5ade6711 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 2 Aug 2021 16:24:32 +0200 Subject: [PATCH 033/149] Parse provided args when available. --- src/pricehist/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pricehist/cli.py b/src/pricehist/cli.py index d845bf0..634d811 100644 --- a/src/pricehist/cli.py +++ b/src/pricehist/cli.py @@ -10,13 +10,13 @@ from pricehist.format import Format from pricehist.series import Series -def cli(args=None, output_file=sys.stdout): +def cli(argv=sys.argv, output_file=sys.stdout): start_time = datetime.now() logger.init() parser = build_parser() - args = parser.parse_args() + args = parser.parse_args(argv[1:]) if args.verbose: logger.show_debug() From 338acf29709ad7cc18e24eb53cf9c4de1e5ad77e Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 2 Aug 2021 16:25:30 +0200 Subject: [PATCH 034/149] Set all defaults for source in arg parser. --- src/pricehist/cli.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/pricehist/cli.py b/src/pricehist/cli.py index 634d811..9adc8be 100644 --- a/src/pricehist/cli.py +++ b/src/pricehist/cli.py @@ -42,21 +42,16 @@ def cli(argv=sys.argv, output_file=sys.stdout): elif args.command == "fetch": source = sources.by_id[args.source] output = outputs.by_type[args.output] - if args.start: - start = args.start - else: - start = source.start() - logging.info(f"Using the source default start date of {start}.") - if args.end < start: + if args.end < args.start: logging.critical( - f"The end date '{args.end}' preceeds the start date '{start}'!" + f"The end date '{args.end}' preceeds the start date '{args.start}'!" ) sys.exit(1) series = Series( base=source.normalizesymbol(args.pair[0]), quote=source.normalizesymbol(args.pair[1]), - type=args.type or (source.types() + ["(none)"])[0], - start=start, + type=args.type, + start=args.start, end=args.end, ) if series.type not in source.types(): @@ -116,6 +111,15 @@ def build_parser(): def formatter(prog): return argparse.HelpFormatter(prog, max_help_position=50) + class SetSourceDefaults(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + source = sources.by_id[value] + setattr(namespace, self.dest, value) + if getattr(namespace, "type") is None: + setattr(namespace, "type", source.types()[0]) + if getattr(namespace, "start") is None: + setattr(namespace, "start", source.start()) + default_fmt = Format() parser = argparse.ArgumentParser( prog="pricehist", @@ -206,6 +210,7 @@ def build_parser(): metavar="SOURCE", type=str, choices=sources.by_id.keys(), + action=SetSourceDefaults, help="the source identifier", ) fetch_parser.add_argument( @@ -226,7 +231,7 @@ def build_parser(): dest="type", metavar="TYPE", type=str, - help="price type, e.g. close", + help="price type, e.g. close (default: first for source)", ) fetch_start_group = fetch_parser.add_mutually_exclusive_group(required=False) fetch_start_group.add_argument( From 784770d9f3bada51aa0d6067a2413e44b540d880 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 2 Aug 2021 16:41:34 +0200 Subject: [PATCH 035/149] Generate parser errors when basic argument validation fail. --- src/pricehist/cli.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/pricehist/cli.py b/src/pricehist/cli.py index 9adc8be..1f8d5b2 100644 --- a/src/pricehist/cli.py +++ b/src/pricehist/cli.py @@ -43,10 +43,14 @@ def cli(argv=sys.argv, output_file=sys.stdout): source = sources.by_id[args.source] output = outputs.by_type[args.output] if args.end < args.start: - logging.critical( + parser.error( f"The end date '{args.end}' preceeds the start date '{args.start}'!" ) - sys.exit(1) + if args.type not in source.types(): + parser.error( + f"The requested price type '{args.type}' is not " + f"recognized by the {source.id()} source!" + ) series = Series( base=source.normalizesymbol(args.pair[0]), quote=source.normalizesymbol(args.pair[1]), @@ -54,12 +58,6 @@ def cli(argv=sys.argv, output_file=sys.stdout): start=args.start, end=args.end, ) - if series.type not in source.types(): - logging.critical( - f"The requested price type '{series.type}' is not " - f"recognized by the {source.id()} source!" - ) - sys.exit(1) fmt = Format.fromargs(args) result = fetch(series, source, output, args.invert, args.quantize, fmt) print(result, end="", file=output_file) From 66e95c1ac44b8b3963781930872705a4eb7d84cc Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 2 Aug 2021 16:42:12 +0200 Subject: [PATCH 036/149] Pull validation helpers out of build_parser() for easier testing. --- src/pricehist/cli.py | 82 ++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/src/pricehist/cli.py b/src/pricehist/cli.py index 1f8d5b2..669a980 100644 --- a/src/pricehist/cli.py +++ b/src/pricehist/cli.py @@ -69,43 +69,49 @@ def cli(argv=sys.argv, output_file=sys.stdout): logging.debug(f"Ended pricehist run at {datetime.now()}.") +def valid_pair(s): + base, quote = (s + "/").split("/")[0:2] + if base == "": + msg = f"No base found in the requested pair '{s}'." + raise argparse.ArgumentTypeError(msg) + return (base, quote) + + +def valid_date(s): + if s == "today": + return today() + try: + return datetime.strptime(s, "%Y-%m-%d").date().isoformat() + except ValueError: + msg = f"Not a valid YYYY-MM-DD date: '{s}'." + raise argparse.ArgumentTypeError(msg) + + +def valid_date_before(s): + return ( + datetime.strptime(valid_date(s), "%Y-%m-%d").date() - timedelta(days=1) + ).isoformat() + + +def valid_date_after(s): + return ( + datetime.strptime(valid_date(s), "%Y-%m-%d").date() + timedelta(days=1) + ).isoformat() + + +def valid_char(s): + if len(s) == 1: + return s + else: + msg = f"Not a single character: '{s}'." + raise argparse.ArgumentTypeError(msg) + + +def today(): + return datetime.now().date().isoformat() + + def build_parser(): - def valid_pair(s): - base, quote = (s + "/").split("/")[0:2] - if base == "": - msg = f"No base found in the requested pair '{s}'." - raise argparse.ArgumentTypeError(msg) - return (base, quote) - - def valid_date(s): - if s == "today": - return today() - try: - return datetime.strptime(s, "%Y-%m-%d").date().isoformat() - except ValueError: - msg = f"Not a valid YYYY-MM-DD date: '{s}'." - raise argparse.ArgumentTypeError(msg) - - def previous_valid_date(s): - return ( - datetime.strptime(valid_date(s), "%Y-%m-%d").date() - timedelta(days=1) - ).isoformat() - - def following_valid_date(s): - return ( - datetime.strptime(valid_date(s), "%Y-%m-%d").date() + timedelta(days=1) - ).isoformat() - - def today(): - return datetime.now().date().isoformat() - - def valid_char(s): - if len(s) == 1: - return s - else: - msg = f"Not a single character: '{s}'." - raise argparse.ArgumentTypeError(msg) - def formatter(prog): return argparse.HelpFormatter(prog, max_help_position=50) @@ -245,7 +251,7 @@ def build_parser(): "--startx", dest="start", metavar="DATE", - type=following_valid_date, + type=valid_date_after, help="start date, exclusive", ) @@ -264,7 +270,7 @@ def build_parser(): "--endx", dest="end", metavar="DATE", - type=previous_valid_date, + type=valid_date_before, help="end date, exclusive", ) From 0ddaf5893bd27205554815760ab761dc5b32fec4 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 3 Aug 2021 14:06:41 +0200 Subject: [PATCH 037/149] Basic tests for cli(). --- src/pricehist/cli.py | 16 ++-- tests/pricehist/test_cli.py | 168 ++++++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 tests/pricehist/test_cli.py diff --git a/src/pricehist/cli.py b/src/pricehist/cli.py index 669a980..d676856 100644 --- a/src/pricehist/cli.py +++ b/src/pricehist/cli.py @@ -10,7 +10,7 @@ from pricehist.format import Format from pricehist.series import Series -def cli(argv=sys.argv, output_file=sys.stdout): +def cli(argv=sys.argv): start_time = datetime.now() logger.init() @@ -25,20 +25,20 @@ def cli(argv=sys.argv, output_file=sys.stdout): try: if args.version: - print(f"pricehist {__version__}", file=output_file) + print(f"pricehist {__version__}") elif args.command == "sources": result = sources.formatted() - print(result, file=output_file) + print(result) elif args.command == "source" and args.symbols: result = sources.by_id[args.source].format_symbols() - print(result, file=output_file, end="") + print(result, end="") elif args.command == "source" and args.search: result = sources.by_id[args.source].format_search(args.search) - print(result, file=output_file, end="") + print(result, end="") elif args.command == "source": total_width = shutil.get_terminal_size().columns result = sources.by_id[args.source].format_info(total_width) - print(result, file=output_file) + print(result) elif args.command == "fetch": source = sources.by_id[args.source] output = outputs.by_type[args.output] @@ -60,9 +60,9 @@ def cli(argv=sys.argv, output_file=sys.stdout): ) fmt = Format.fromargs(args) result = fetch(series, source, output, args.invert, args.quantize, fmt) - print(result, end="", file=output_file) + print(result, end="") else: - parser.print_help(file=sys.stderr) + parser.print_help() except BrokenPipeError: logging.debug("The output pipe was closed early.") finally: diff --git a/tests/pricehist/test_cli.py b/tests/pricehist/test_cli.py new file mode 100644 index 0000000..cd78e46 --- /dev/null +++ b/tests/pricehist/test_cli.py @@ -0,0 +1,168 @@ +import argparse +import logging +from decimal import Decimal + +import pytest + +from pricehist import __version__, cli, sources + + +def w(string): + return string.split(" ") + + +def test_valid_pair(): + assert cli.valid_pair("BTC/AUD") == ("BTC", "AUD") + assert cli.valid_pair("BTC/AUD/ignored") == ("BTC", "AUD") + assert cli.valid_pair("SYM") == ("SYM", "") + assert cli.valid_pair("SYM/") == ("SYM", "") + with pytest.raises(argparse.ArgumentTypeError): + cli.valid_pair("/SYM") + with pytest.raises(argparse.ArgumentTypeError): + cli.valid_pair("") + + +def test_valid_date(): + assert cli.valid_date("today") == cli.today() + assert cli.valid_date("2021-12-30") == "2021-12-30" + with pytest.raises(argparse.ArgumentTypeError) as e: + cli.valid_date("2021-12-40") + assert "Not a valid" in str(e.value) + + +def test_valid_date_before(): + assert cli.valid_date_before("2021-12-30") == "2021-12-29" + with pytest.raises(argparse.ArgumentTypeError) as e: + cli.valid_date_before("2021-12-40") + assert "Not a valid" in str(e.value) + + +def test_valid_date_after(): + assert cli.valid_date_after("2021-12-30") == "2021-12-31" + with pytest.raises(argparse.ArgumentTypeError) as e: + cli.valid_date_after("2021-12-40") + assert "Not a valid" in str(e.value) + + +def test_valid_char(): + assert cli.valid_char(",") == "," + with pytest.raises(argparse.ArgumentTypeError): + cli.valid_char("") + with pytest.raises(argparse.ArgumentTypeError): + cli.valid_char("12") + + +def test_cli_no_args_shows_usage(capfd): + cli.cli(w("pricehist")) + out, err = capfd.readouterr() + assert "usage: pricehist" in out + assert "optional arguments:" in out + assert "commands:" in out + + +def test_cli_help_shows_usage_and_exits(capfd): + with pytest.raises(SystemExit) as e: + cli.cli(w("pricehist -h")) + assert e.value.code == 0 + out, err = capfd.readouterr() + assert "usage: pricehist" in out + assert "optional arguments:" in out + assert "commands:" in out + + +def test_cli_verbose(capfd, mocker): + cli.cli(w("pricehist --verbose")) + out, err = capfd.readouterr() + assert "Ended pricehist run at" in err + + +def test_cli_version(capfd): + cli.cli(w("pricehist --version")) + out, err = capfd.readouterr() + assert f"pricehist {__version__}\n" == out + + +def test_cli_sources(capfd): + cli.cli(w("pricehist sources")) + out, err = capfd.readouterr() + for source_id in sources.by_id.keys(): + assert source_id in out + + +def test_cli_source(capfd): + expected = sources.by_id["ecb"].format_info() + "\n" + cli.cli(w("pricehist source ecb")) + out, err = capfd.readouterr() + assert out == expected + + +def test_cli_source_symbols(capfd, mocker): + sources.by_id["ecb"].symbols = mocker.MagicMock( + return_value=[("EUR/AUD", "Euro against Australian Dollar")] + ) + cli.cli(w("pricehist source ecb --symbols")) + out, err = capfd.readouterr() + assert out == "EUR/AUD Euro against Australian Dollar\n" + + +def test_cli_source_search(capfd, mocker): + sources.by_id["alphavantage"].search = mocker.MagicMock( + return_value=[("TSLA", "Tesla Inc, Equity, United States, USD")] + ) + cli.cli(w("pricehist source alphavantage --search TSLA")) + out, err = capfd.readouterr() + assert out == "TSLA Tesla Inc, Equity, United States, USD\n" + + +def test_cli_source_fetch(capfd, mocker): + formatted_result = "P 2021-01-01 00:00:00 BTC 24139.4648 EUR\n" + cli.fetch = mocker.MagicMock(return_value=formatted_result) + argv = w("pricehist fetch coindesk BTC/EUR -s 2021-01-01 -e 2021-01-01 -o ledger") + cli.cli(argv) + out, err = capfd.readouterr() + assert out == formatted_result + + +def test_cli_source_fetch_invalid_start(capfd, mocker): + argv = w("pricehist fetch coindesk BTC/EUR -s 2021-01-01 -e 2020-12-01") + with pytest.raises(SystemExit) as e: + cli.cli(argv) + assert e.value.code != 0 + out, err = capfd.readouterr() + assert "end date '2020-12-01' preceeds the start date" in err + + +def test_cli_source_fetch_invalid_type(capfd, mocker): + argv = w("pricehist fetch coindesk BTC/EUR -t notype") + with pytest.raises(SystemExit) as e: + cli.cli(argv) + assert e.value.code != 0 + out, err = capfd.readouterr() + assert "price type 'notype' is not recognized" in err + + +def test_cli_source_fetch_sets_source_defaults(mocker): + cli.fetch = mocker.MagicMock(return_value="") + cli.cli(w("pricehist fetch coindesk BTC/EUR")) + captured_series = cli.fetch.call_args.args[0] + assert captured_series.start == sources.by_id["coindesk"].start() + assert captured_series.type == sources.by_id["coindesk"].types()[0] + + +def test_cli_source_fetch_normalizes_symbols(mocker): + cli.fetch = mocker.MagicMock(return_value="") + cli.cli(w("pricehist fetch coindesk btc/eur")) + captured_series = cli.fetch.call_args.args[0] + assert captured_series.base == "BTC" + assert captured_series.quote == "EUR" + + +def test_cli_source_fetch_handles_brokenpipeerror(caplog, mocker): + cli.fetch = mocker.MagicMock(side_effect=BrokenPipeError()) + cli.cli(w("pricehist fetch coindesk BTC/EUR --verbose")) + assert any( + [ + "DEBUG" == r.levelname and "output pipe was closed early" in r.message + for r in caplog.records + ] + ) From c0af189ae03d0d85d71544ff4d7e0029b09bcced Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 3 Aug 2021 14:09:20 +0200 Subject: [PATCH 038/149] Remove unused imports. --- tests/pricehist/test_cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/pricehist/test_cli.py b/tests/pricehist/test_cli.py index cd78e46..16bd383 100644 --- a/tests/pricehist/test_cli.py +++ b/tests/pricehist/test_cli.py @@ -1,6 +1,4 @@ import argparse -import logging -from decimal import Decimal import pytest From 2d1553e2d9509088f04655a51f801e9b2b8621db Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Wed, 4 Aug 2021 12:18:49 +0200 Subject: [PATCH 039/149] Test basesource methods. --- tests/pricehist/sources/test_basesource.py | 182 +++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 tests/pricehist/sources/test_basesource.py diff --git a/tests/pricehist/sources/test_basesource.py b/tests/pricehist/sources/test_basesource.py new file mode 100644 index 0000000..cfd5e80 --- /dev/null +++ b/tests/pricehist/sources/test_basesource.py @@ -0,0 +1,182 @@ +import logging + +import pytest + +from pricehist.series import Series +from pricehist.sources.basesource import BaseSource + + +class TestSource(BaseSource): + def id(self) -> str: + return "" + + def name(self) -> str: + return "" + + def description(self) -> str: + return "" + + def source_url(self) -> str: + return "" + + def start(self) -> str: + return "" + + def types(self) -> list[str]: + return [] + + def notes(self) -> str: + return "" + + def symbols(self) -> list[(str, str)]: + return [] + + def fetch(self, series: Series) -> Series: + pass + + +@pytest.fixture +def src(): + return TestSource() + + +def test_normalizesymbol_default_uppercase(src): + assert src.normalizesymbol("eur") == "EUR" + + +def test_format_symbols_one(src, mocker): + src.symbols = mocker.MagicMock(return_value=[("A", "Description")]) + assert src.format_symbols() == "A Description\n" + + +def test_format_symbols_many(src, mocker): + src.symbols = mocker.MagicMock( + return_value=[ + ("A", "Description"), + ("BB", "Description longer"), + ("CCC", "Description longer again"), + ("DDDD", f"Description {'very '*15}long"), + ] + ) + assert src.format_symbols() == ( + "A Description\n" + "BB Description longer\n" + "CCC Description longer again\n" + "DDDD Description very very very very very very very very " + "very very very very very very very long\n" + ) + + +def test_format_search(src, mocker): + src.search = mocker.MagicMock( + return_value=[ + ("A", "Description"), + ("BB", "Description longer"), + ("CCC", "Description longer again"), + ("DDDD", f"Description {'very '*15}long"), + ] + ) + assert src.format_search("some query") == ( + "A Description\n" + "BB Description longer\n" + "CCC Description longer again\n" + "DDDD Description very very very very very very very very " + "very very very very very very very long\n" + ) + + +def test_format_search_not_possible(src, mocker, caplog): + src.search = mocker.MagicMock(return_value=None) + with caplog.at_level(logging.INFO): + with pytest.raises(SystemExit) as e: + src.format_search("some query") + assert e.value.code == 1 + r = caplog.records[0] + assert r.levelname == "ERROR" + assert "Symbol search is not possible for" in r.message + + +def test_format_search_no_results(src, mocker, caplog): + src.search = mocker.MagicMock(return_value=[]) + with caplog.at_level(logging.INFO): + results = src.format_search("some query") + r = caplog.records[0] + assert r.levelname == "INFO" + assert "No results found" in r.message + assert results == "" + + +def test_format_info_skips_renderes_all_fields(src, mocker): + src.id = mocker.MagicMock(return_value="sourceid") + src.name = mocker.MagicMock(return_value="Source Name") + src.description = mocker.MagicMock(return_value="Source description.") + src.source_url = mocker.MagicMock(return_value="https://example.com/") + src.start = mocker.MagicMock(return_value="2021-01-01") + src.types = mocker.MagicMock(return_value=["open", "close"]) + src.notes = mocker.MagicMock(return_value="Notes for user.") + output = src.format_info() + assert output == ( + "ID : sourceid\n" + "Name : Source Name\n" + "Description : Source description.\n" + "URL : https://example.com/\n" + "Start : 2021-01-01\n" + "Types : open, close\n" + "Notes : Notes for user." + ) + + +def test_format_info_skips_empty_fields(src, mocker): + src.notes = mocker.MagicMock(return_value="") + output = src.format_info() + assert "Notes" not in output + + +def test_format_info_wraps_long_values_with_indent(src, mocker): + notes = ( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim " + "ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut " + "aliquip ex ea commodo consequat." + ) + src.notes = mocker.MagicMock(return_value=notes) + output = src.format_info(total_width=60) + assert output == ( + "Notes : Lorem ipsum dolor sit amet, consectetur\n" + " adipiscing elit, sed do eiusmod tempor\n" + " incididunt ut labore et dolore magna aliqua.\n" + " Ut enim ad minim veniam, quis nostrud\n" + " exercitation ullamco laboris nisi ut aliquip\n" + " ex ea commodo consequat." + ) + + +def test_format_info_newline_handling(src, mocker): + notes = ( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore.\n" + "Ut enim ad minim veniam.\n" + "\n" + "Quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea " + "commodo consequat." + ) + src.notes = mocker.MagicMock(return_value=notes) + output = src.format_info(total_width=60) + assert output == ( + "Notes : Lorem ipsum dolor sit amet, consectetur\n" + " adipiscing elit, sed do eiusmod tempor\n" + " incididunt ut labore.\n" + " Ut enim ad minim veniam.\n" + "\n" + " Quis nostrud exercitation ullamco laboris nisi\n" + " ut aliquip ex ea commodo consequat." + ) + + +def test_format_info_does_not_wrap_source_url(src, mocker): + url = "https://www.example.com/longlonglonglonglonglonglonglong/" + src.source_url = mocker.MagicMock(return_value=url) + output = src.format_info(total_width=60) + assert output == ( + "URL : https://www.example.com/longlonglonglonglonglonglonglong/" + ) From 5d0c6aaf0388f3edee18d51d2cdf4b36495f7a70 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Wed, 4 Aug 2021 12:19:23 +0200 Subject: [PATCH 040/149] Don't test baseoutput abstract method. --- src/pricehist/outputs/baseoutput.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pricehist/outputs/baseoutput.py b/src/pricehist/outputs/baseoutput.py index f3cb58f..f803e14 100644 --- a/src/pricehist/outputs/baseoutput.py +++ b/src/pricehist/outputs/baseoutput.py @@ -8,4 +8,4 @@ from pricehist.sources.basesource import BaseSource class BaseOutput(ABC): @abstractmethod def format(self, series: Series, source: BaseSource, fmt: Format) -> str: - pass + pass # pragma: nocover From 78b622eadea18b3e70407cc91b063458cc4a0170 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Wed, 4 Aug 2021 15:34:52 +0200 Subject: [PATCH 041/149] Test outputs. --- src/pricehist/outputs/gnucashsql.py | 3 +- tests/pricehist/outputs/__init__.py | 0 tests/pricehist/outputs/test_beancount.py | 44 +++++++ tests/pricehist/outputs/test_csv.py | 50 ++++++++ tests/pricehist/outputs/test_gnucashsql.py | 140 +++++++++++++++++++++ tests/pricehist/outputs/test_ledger.py | 52 ++++++++ 6 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 tests/pricehist/outputs/__init__.py create mode 100644 tests/pricehist/outputs/test_beancount.py create mode 100644 tests/pricehist/outputs/test_csv.py create mode 100644 tests/pricehist/outputs/test_gnucashsql.py create mode 100644 tests/pricehist/outputs/test_ledger.py diff --git a/src/pricehist/outputs/gnucashsql.py b/src/pricehist/outputs/gnucashsql.py index 1cb536b..2034754 100644 --- a/src/pricehist/outputs/gnucashsql.py +++ b/src/pricehist/outputs/gnucashsql.py @@ -58,6 +58,7 @@ class GnuCashSQL(BaseOutput): self._warn_about_backslashes( { + "date": fmt.format_date("1970-01-01"), "time": fmt.time, "base": base, "quote": quote, @@ -135,7 +136,7 @@ class GnuCashSQL(BaseOutput): logging.warning( f"Before running this SQL, check the formatting of the " f"{self._english_join(hits)} strings. " - f"SQLite treats backslahes in strings as plain characters, but " + f"SQLite treats backslashes in strings as plain characters, but " f"MariaDB/MySQL and PostgreSQL may interpret them as escape " f"codes." ) diff --git a/tests/pricehist/outputs/__init__.py b/tests/pricehist/outputs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pricehist/outputs/test_beancount.py b/tests/pricehist/outputs/test_beancount.py new file mode 100644 index 0000000..102fe21 --- /dev/null +++ b/tests/pricehist/outputs/test_beancount.py @@ -0,0 +1,44 @@ +from decimal import Decimal + +import pytest + +from pricehist.format import Format +from pricehist.outputs.beancount import Beancount +from pricehist.price import Price +from pricehist.series import Series + + +@pytest.fixture +def out(): + return Beancount() + + +@pytest.fixture +def series(): + prices = [ + Price("2021-01-01", Decimal("24139.4648")), + Price("2021-01-02", Decimal("26533.576")), + Price("2021-01-03", Decimal("27001.2846")), + ] + return Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03", prices) + + +def test_format_basics(out, series, mocker): + source = mocker.MagicMock() + result = out.format(series, source, Format()) + assert result == ( + "2021-01-01 price BTC 24139.4648 EUR\n" + "2021-01-02 price BTC 26533.576 EUR\n" + "2021-01-03 price BTC 27001.2846 EUR\n" + ) + + +def test_format_custom(out, series, mocker): + source = mocker.MagicMock() + fmt = Format(base="XBT", quote="EURO", thousands=".", decimal=",", datesep="/") + result = out.format(series, source, fmt) + assert result == ( + "2021/01/01 price XBT 24.139,4648 EURO\n" + "2021/01/02 price XBT 26.533,576 EURO\n" + "2021/01/03 price XBT 27.001,2846 EURO\n" + ) diff --git a/tests/pricehist/outputs/test_csv.py b/tests/pricehist/outputs/test_csv.py new file mode 100644 index 0000000..f436f73 --- /dev/null +++ b/tests/pricehist/outputs/test_csv.py @@ -0,0 +1,50 @@ +from decimal import Decimal + +import pytest + +from pricehist.format import Format +from pricehist.outputs.csv import CSV +from pricehist.price import Price +from pricehist.series import Series + + +@pytest.fixture +def out(): + return CSV() + + +@pytest.fixture +def series(): + prices = [ + Price("2021-01-01", Decimal("24139.4648")), + Price("2021-01-02", Decimal("26533.576")), + Price("2021-01-03", Decimal("27001.2846")), + ] + return Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03", prices) + + +def test_format_basics(out, series, mocker): + source = mocker.MagicMock() + source.id = mocker.MagicMock(return_value="sourceid") + result = out.format(series, source, Format()) + assert result == ( + "date,base,quote,amount,source,type\n" + "2021-01-01,BTC,EUR,24139.4648,sourceid,close\n" + "2021-01-02,BTC,EUR,26533.576,sourceid,close\n" + "2021-01-03,BTC,EUR,27001.2846,sourceid,close\n" + ) + + +def test_format_custom(out, series, mocker): + source = mocker.MagicMock() + source.id = mocker.MagicMock(return_value="sourceid") + fmt = Format( + base="XBT", quote="€", thousands=".", decimal=",", datesep="/", csvdelim="/" + ) + result = out.format(series, source, fmt) + assert result == ( + "date/base/quote/amount/source/type\n" + '"2021/01/01"/XBT/€/24.139,4648/sourceid/close\n' + '"2021/01/02"/XBT/€/26.533,576/sourceid/close\n' + '"2021/01/03"/XBT/€/27.001,2846/sourceid/close\n' + ) diff --git a/tests/pricehist/outputs/test_gnucashsql.py b/tests/pricehist/outputs/test_gnucashsql.py new file mode 100644 index 0000000..3b058ad --- /dev/null +++ b/tests/pricehist/outputs/test_gnucashsql.py @@ -0,0 +1,140 @@ +import dataclasses +import logging +import re +from decimal import Decimal + +import pytest + +from pricehist.format import Format +from pricehist.outputs.gnucashsql import GnuCashSQL +from pricehist.price import Price +from pricehist.series import Series + + +@pytest.fixture +def out(): + return GnuCashSQL() + + +@pytest.fixture +def series(): + prices = [ + Price("2021-01-01", Decimal("24139.4648")), + Price("2021-01-02", Decimal("26533.576")), + Price("2021-01-03", Decimal("27001.2846")), + ] + return Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03", prices) + + +@pytest.fixture +def src(mocker): + source = mocker.MagicMock() + source.id = mocker.MagicMock(return_value="coindesk") + return source + + +def test_format_base_and_quote(out, series, src): + result = out.format(series, src, Format()) + base, quote = re.findall(r"WHERE mnemonic = (.*) LIMIT", result, re.MULTILINE) + assert base == "'BTC'" + assert quote == "'EUR'" + + +def test_format_new_price_values(out, series, src): + result = out.format(series, src, Format()) + values = re.search( + r"\(guid, date, base, quote, source, type, " + r"value_num, value_denom\) VALUES\n([^;]*);", + result, + re.MULTILINE, + )[1] + assert values == ( + "('0c4c01bd0a252641b806ce46f716f161', '2021-01-01 00:00:00', " + "'BTC', 'EUR', 'coindesk', 'close', 241394648, 10000),\n" + "('47f895ddfcce18e2421387e0e1b636e9', '2021-01-02 00:00:00', " + "'BTC', 'EUR', 'coindesk', 'close', 26533576, 1000),\n" + "('0d81630c4ac50c1b9b7c8211bf99c94e', '2021-01-03 00:00:00', " + "'BTC', 'EUR', 'coindesk', 'close', 270012846, 10000)\n" + ) + + +def test_format_customized(out, series, src): + fmt = Format( + base="XBT", + quote="EURO", + datesep="/", + time="23:59:59", + ) + result = out.format(series, src, fmt) + base, quote = re.findall(r"WHERE mnemonic = (.*) LIMIT", result, re.MULTILINE) + values = re.search( + r"\(guid, date, base, quote, source, type, " + r"value_num, value_denom\) VALUES\n([^;]*);", + result, + re.MULTILINE, + )[1] + assert base == "'XBT'" + assert quote == "'EURO'" + assert values == ( + "('448173eef5dea23cea9ff9d5e8c7b07e', '2021/01/01 23:59:59', " + "'XBT', 'EURO', 'coindesk', 'close', 241394648, 10000),\n" + "('b6c0f4474c91c50e8f65b47767f874ba', '2021/01/02 23:59:59', " + "'XBT', 'EURO', 'coindesk', 'close', 26533576, 1000),\n" + "('2937c872cf0672863e11b9f46ee41e09', '2021/01/03 23:59:59', " + "'XBT', 'EURO', 'coindesk', 'close', 270012846, 10000)\n" + ) + + +def test_format_escaping_of_strings(out, series, src): + result = out.format(series, src, Format(base="B'tc''n")) + base, quote = re.findall(r"WHERE mnemonic = (.*) LIMIT", result, re.MULTILINE) + assert base == "'B''tc''''n'" + + +def test_format_insert_commented_out_if_no_values(out, series, src): + empty_series = dataclasses.replace(series, prices=[]) + result = out.format(empty_series, src, Format()) + ( + "-- INSERT INTO new_prices (guid, date, base, quote, source, type, " + "value_num, value_denom) VALUES\n" + "-- \n" + "-- ;\n" + ) in result + + +def test_format_warns_about_backslash(out, series, src, caplog): + with caplog.at_level(logging.WARNING): + out.format(series, src, Format(quote="EU\\RO")) + r = caplog.records[0] + assert r.levelname == "WARNING" + assert "backslashes in strings" in r.message + + +def test__english_join_other_cases(out): + assert out._english_join([]) == "" + assert out._english_join(["one"]) == "one" + assert out._english_join(["one", "two"]) == "one and two" + assert out._english_join(["one", "two", "three"]) == "one, two and three" + + +def test_format_warns_about_out_of_range_numbers(out, series, src, caplog): + too_big_numerator = Decimal("9223372036854.775808") + s = dataclasses.replace(series, prices=[Price("2021-01-01", too_big_numerator)]) + with caplog.at_level(logging.WARNING): + out.format(s, src, Format()) + r = caplog.records[0] + assert r.levelname == "WARNING" + assert "outside of the int64 range" in r.message + + +def test__rational_other_exponent_cases(out): + assert out._rational(Decimal("9223372036854e6")) == ( + "9223372036854000000", + "1", + True, + ) + assert out._rational(Decimal("9223372036854e-6")) == ( + "9223372036854", + "1000000", + True, + ) diff --git a/tests/pricehist/outputs/test_ledger.py b/tests/pricehist/outputs/test_ledger.py new file mode 100644 index 0000000..a1a242c --- /dev/null +++ b/tests/pricehist/outputs/test_ledger.py @@ -0,0 +1,52 @@ +from decimal import Decimal + +import pytest + +from pricehist.format import Format +from pricehist.outputs.ledger import Ledger +from pricehist.price import Price +from pricehist.series import Series + + +@pytest.fixture +def out(): + return Ledger() + + +@pytest.fixture +def series(): + prices = [ + Price("2021-01-01", Decimal("24139.4648")), + Price("2021-01-02", Decimal("26533.576")), + Price("2021-01-03", Decimal("27001.2846")), + ] + return Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03", prices) + + +def test_format_basics(out, series, mocker): + source = mocker.MagicMock() + result = out.format(series, source, Format()) + assert result == ( + "P 2021-01-01 00:00:00 BTC 24139.4648 EUR\n" + "P 2021-01-02 00:00:00 BTC 26533.576 EUR\n" + "P 2021-01-03 00:00:00 BTC 27001.2846 EUR\n" + ) + + +def test_format_custom(out, series, mocker): + source = mocker.MagicMock() + fmt = Format( + base="XBT", + quote="€", + time="23:59:59", + thousands=".", + decimal=",", + symbol="left", + datesep="/", + ) + result = out.format(series, source, fmt) + assert result == ( + "P 2021/01/01 23:59:59 XBT €24.139,4648\n" + "P 2021/01/02 23:59:59 XBT €26.533,576\n" + "P 2021/01/03 23:59:59 XBT €27.001,2846\n" + ) From 9050b7948b359e4acd02dac311b6d6dedbcd6728 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 5 Aug 2021 12:57:10 +0200 Subject: [PATCH 042/149] Run flake8 linting on CI. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d38113c..5822113 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ test: script: - poetry run isort src tests --check - poetry run black src tests --check + - poetry run flake8 - poetry run pytest - poetry run coverage run --source=pricehist -m pytest - poetry run coverage report From 637650245e1b9092dd3a2e033ab7fdc3684d7c6a Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 10 Aug 2021 13:58:15 +0200 Subject: [PATCH 043/149] Set up pre-commit hook and related make targets. --- Makefile | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c76460c..130c0f0 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,15 @@ help: ## List make targets @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -lint: ## Lint source code - poetry run flake8 - +.PHONY: format format: ## Format source code poetry run isort . poetry run black . +.PHONY: lint +lint: ## Lint source code + poetry run flake8 + .PHONY: test test: ## Run non-live tests poetry run pytest -m "not live" --color=yes @@ -22,3 +24,14 @@ coverage: ## Generate and open coverage report poetry run coverage run --source=pricehist -m pytest poetry run coverage html xdg-open htmlcov/index.html + +.PHONY: install-pre-commit-hook +install-pre-commit-hook: ## Install the git pre-commit hook + echo -e "#!/bin/bash\nmake pre-commit" > .git/hooks/pre-commit + chmod +x .git/hooks/pre-commit + +.PHONY: pre-commit +pre-commit: ## Checks to run before each commit + poetry run isort src tests --check + poetry run black src tests --check + poetry run flake8 From d19bc66c610c0fd3f48b08329db48c1ec30da749 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 10 Aug 2021 14:16:21 +0200 Subject: [PATCH 044/149] Avoid running flake8 on cache, etc. --- .gitlab-ci.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5822113..8c54d1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,7 @@ test: script: - poetry run isort src tests --check - poetry run black src tests --check - - poetry run flake8 + - poetry run flake8 src tests - poetry run pytest - poetry run coverage run --source=pricehist -m pytest - poetry run coverage report diff --git a/Makefile b/Makefile index 130c0f0..e219af0 100644 --- a/Makefile +++ b/Makefile @@ -34,4 +34,4 @@ install-pre-commit-hook: ## Install the git pre-commit hook pre-commit: ## Checks to run before each commit poetry run isort src tests --check poetry run black src tests --check - poetry run flake8 + poetry run flake8 src tests From 74d9c211c9f25c68b14ccb55301828ca33c5d19e Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Wed, 11 Aug 2021 15:00:58 +0200 Subject: [PATCH 045/149] Add some live tests. --- .gitlab-ci.yml | 14 ++++-- Makefile | 6 +-- pyproject.toml | 4 +- tests/live.sh | 131 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 9 deletions(-) create mode 100755 tests/live.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8c54d1e..29ec768 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,11 +14,19 @@ before_script: - pip install poetry - poetry install +pre-commit: + script: + - make pre-commit + test: script: - - poetry run isort src tests --check - - poetry run black src tests --check - - poetry run flake8 src tests - poetry run pytest + +test-live: + script: + - tests/live.sh + +coverage: + script: - poetry run coverage run --source=pricehist -m pytest - poetry run coverage report diff --git a/Makefile b/Makefile index e219af0..1a26a11 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,12 @@ lint: ## Lint source code poetry run flake8 .PHONY: test -test: ## Run non-live tests - poetry run pytest -m "not live" --color=yes +test: ## Run tests + poetry run pytest --color=yes .PHONY: test-live test-live: ## Run live tests - poetry run pytest -m live --color=yes + tests/live.sh .PHONY: coverage coverage: ## Generate and open coverage report diff --git a/pyproject.toml b/pyproject.toml index 193c1fb..52a3341 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,4 @@ profile = "black" multi_line_output = 3 [tool.pytest.ini_options] -markers = [ - "live: makes a live request to a source" -] +markers = [] diff --git a/tests/live.sh b/tests/live.sh new file mode 100755 index 0000000..268111e --- /dev/null +++ b/tests/live.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +# These are basic happy path tests that run pricehist from the command line and +# confirm that the results come out as expected. They help ensure that the main +# endpoints for each source are still working. + +# Run this from the project root. + +export ALPHAVANTAGE_API_KEY="TEST_KEY_$RANDOM" +cmd_prefix="poetry run" + +passed=0 +failed=0 + +run_test(){ + name=$1 + cmd=$2 + expected=$3 + echo "TEST: $name" + echo " Action: $cmd" + echo -n " Result: " + full_cmd="$cmd_prefix $cmd" + actual=$($full_cmd 2>&1) + if [[ "$actual" == "$expected" ]]; then + passed=$((passed+1)) + echo "passed, output as expected" + else + failed=$((failed+1)) + echo "failed, output differs as follows..." + echo + diff <(echo "$expected") <(echo "$actual") + fi + echo +} + +report(){ + total=$((passed+failed)) + if [[ "$failed" -eq "0" ]]; then + echo "SUMMARY: $passed tests passed, none failed" + else + echo "SUMMARY: $failed/$total tests failed" + exit 1 + fi +} + +name="Alpha Vantage stocks" +cmd="pricehist fetch alphavantage TSLA -s 2021-01-04 -e 2021-01-08" +read -r -d '' expected < Date: Wed, 11 Aug 2021 15:12:02 +0200 Subject: [PATCH 046/149] Tidy README. --- README.md | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/README.md b/README.md index 6f2eb81..601f2f9 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ support for multiple data sources and output formats. [![PyPI version](https://badge.fury.io/py/pricehist.svg)](https://badge.fury.io/py/pricehist) [![Downloads](https://pepy.tech/badge/pricehist)](https://pepy.tech/project/pricehist) [![License](https://img.shields.io/pypi/l/pricehist)](https://gitlab.com/chrisberkhout/pricehist/-/blob/master/LICENSE) +[![Code style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) ## Installation @@ -157,14 +158,3 @@ method for fetching historical prices. The GnuCash wiki documents [wrapper scripts](https://wiki.gnucash.org/wiki/Stocks/get_prices) for the [Finance::QuoteHist](https://metacpan.org/pod/Finance::QuoteHist) Perl module. - -Other projects with related goals include: - -* [`hledger-stockquotes`](https://github.com/prikhi/hledger-stockquotes): - A CLI addon for hledger that reads a journal file and pulls the historical prices for commodities. -* [`ledger_get_prices`](https://github.com/nathankot/ledger-get-prices): - Uses Yahoo Finance to generate a price database based on your current Ledger commodities and time period. -* [LedgerStockUpdate](https://github.com/adchari/LedgerStockUpdate): - Locates any stocks you have in your ledger-cli file, then generates a price database of those stocks. -* [`market-prices`](https://github.com/barrucadu/hledger-scripts#market-prices): - Downloads market values of commodities from a few different sources. From a4c0a142e3dd1586a92b649b09c6b2acc43a4548 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 12 Aug 2021 16:30:53 +0200 Subject: [PATCH 047/149] Expand README. --- README.md | 269 ++++++++++++++++++++++++++++++++++++++++---- example-gnuplot.png | Bin 0 -> 32357 bytes 2 files changed, 246 insertions(+), 23 deletions(-) create mode 100644 example-gnuplot.png diff --git a/README.md b/README.md index 601f2f9..8d02e6a 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,13 @@ pipx install pricehist - **`gnucash-sql`**: [GnuCash](https://www.gnucash.org/) SQL - **`ledger`**: [Ledger](https://www.ledger-cli.org/) and [hledger](https://hledger.org/) -## Examples +## Usage -Show usage information: +The following sections demonstrate different uses of `pricehist`. + +### Online help + +The `-h` option will show usage information: ```bash pricehist -h @@ -58,7 +62,7 @@ commands: fetch fetch prices ``` -Show usage information for the `fetch` command: +Here's the usage information for the `fetch` command: ``` pricehist fetch -h @@ -94,28 +98,48 @@ optional arguments: --fmt-csvdelim CHAR field delimiter for CSV output (default: ',') ``` -Fetch prices after 2021-01-04, ending 2021-01-15, as CSV: +### Fetching prices + +Fetch prices by choosing a source, a pair and, optionally, a time interval: ```bash -pricehist fetch ecb EUR/AUD -sx 2021-01-04 -e 2021-01-15 -o csv +pricehist fetch ecb EUR/AUD -s 2021-01-04 -e 2021-01-08 ``` ``` date,base,quote,amount,source,type +2021-01-04,EUR,AUD,1.5928,ecb,reference 2021-01-05,EUR,AUD,1.5927,ecb,reference 2021-01-06,EUR,AUD,1.5824,ecb,reference 2021-01-07,EUR,AUD,1.5836,ecb,reference 2021-01-08,EUR,AUD,1.5758,ecb,reference -2021-01-11,EUR,AUD,1.5783,ecb,reference -2021-01-12,EUR,AUD,1.5742,ecb,reference -2021-01-13,EUR,AUD,1.5734,ecb,reference -2021-01-14,EUR,AUD,1.5642,ecb,reference -2021-01-15,EUR,AUD,1.568,ecb,reference ``` -In Ledger format: +The default output format is CSV, which is suitable for use in spreadsheets and +with other tools. For example, on the command line using +[gnuplot](http://www.gnuplot.info/) we can chart Bitcoin prices: ```bash -pricehist fetch ecb EUR/AUD -s 2021-01-01 -o ledger | head +pricehist fetch coindesk BTC/USD -s 2021-01-01 | \ + cut -d, -f1,4 | \ + sed 1d | \ + gnuplot -p -e ' + set datafile separator ","; + set xdata time; + set timefmt "%Y-%m-%d"; + set format x "%b\n%Y"; + plot "/dev/stdin" using 1:2 with lines title "BTC/USD" + ' +``` + +![BTC/USD prices](example-gnuplot.png) + +### Choosing and customizing the output format + +You can choose a different output format: `ledger`, `beancount` or +`gnucash-sql`. + +```bash +pricehist fetch ecb EUR/AUD -s 2021-01-04 -e 2021-01-08 -o ledger ``` ``` P 2021-01-04 00:00:00 EUR 1.5928 AUD @@ -123,14 +147,28 @@ P 2021-01-05 00:00:00 EUR 1.5927 AUD P 2021-01-06 00:00:00 EUR 1.5824 AUD P 2021-01-07 00:00:00 EUR 1.5836 AUD P 2021-01-08 00:00:00 EUR 1.5758 AUD -P 2021-01-11 00:00:00 EUR 1.5783 AUD -P 2021-01-12 00:00:00 EUR 1.5742 AUD -P 2021-01-13 00:00:00 EUR 1.5734 AUD -P 2021-01-14 00:00:00 EUR 1.5642 AUD -P 2021-01-15 00:00:00 EUR 1.568 AUD ``` -Generate SQL for a GnuCash database and apply it immediately: +The formatting options let you control certain details of the output. The +following example removes the time of day (to make it suitable for hledger) and +makes a few other adjustments. + +```bash +pricehist fetch ecb EUR/AUD -s 2021-01-04 -e 2021-01-08 -o ledger \ + --fmt-time '' --fmt-datesep / --fmt-base € --fmt-quote $ --fmt-symbol left +``` +``` +P 2021/01/04 € $1.5928 +P 2021/01/05 € $1.5927 +P 2021/01/06 € $1.5824 +P 2021/01/07 € $1.5836 +P 2021/01/08 € $1.5758 +``` + +### GnuCash SQL output + +You can generate SQL for a GnuCash database and apply it immediately with one +of the following commands: ```bash pricehist fetch ecb EUR/AUD -s 2021-01-01 -o gnucash-sql | sqlite3 Accounts.gnucash @@ -138,12 +176,197 @@ pricehist fetch ecb EUR/AUD -s 2021-01-01 -o gnucash-sql | mysql -u username -p pricehist fetch ecb EUR/AUD -s 2021-01-01 -o gnucash-sql | psql -U username -d databasename -v ON_ERROR_STOP=1 ``` -## Design choices +Beware that the GnuCash project itself does not support integration at the +database level, so there is a risk that the SQL generated by `pricehist` will +be ineffective or even damaging for some new version of GnuCash. -To keep things simple, at least for now, `pricehist` provides only univariate -time series of daily historical prices. It doesn't provide other types of -market, financial or economic data, real-time prices, or other temporal -resolutions. Multiple or multivariate series require multiple invocations. +In practice, this strategy has been used successfully by other projects. +Reviewing the SQL and keeping regular database backups is recommended. + +### Source information + +Some basic information about each source is available: + +```bash +pricehist source ecb +``` +``` +ID : ecb +Name : European Central Bank +Description : European Central Bank Euro foreign exchange reference rates +URL : https://www.ecb.europa.eu/stats/exchange/eurofxref/html/index.en.html +Start : 1999-01-04 +Types : reference +``` + +Symbol information can be listed for most sources, either as full pairs or as +separate base and quote symbols that work in particular combinations: + +```bash +pricehist source ecb --symbols +``` +``` +EUR/AUD Euro against Australian Dollar +EUR/BGN Euro against Bulgarian Lev +EUR/BRL Euro against Brazilian Real +EUR/CAD Euro against Canadian Dollar +EUR/CHF Euro against Swiss Franc +... +``` + +It may also be possible to search for symbols: + +```bash +pricehist source alphavantage --search Tesla +``` +``` +TL0.DEX Tesla, Equity, XETRA, EUR +TL0.FRK Tesla, Equity, Frankfurt, EUR +TSLA34.SAO Tesla, Equity, Brazil/Sao Paolo, BRL +TSLA Tesla Inc, Equity, United States, USD +TXLZF Tesla Exploration Ltd, Equity, United States, USD +``` + +### Inspecting source interactions + +You can see extra information, including `curl` commands to reproduce each +request to a source, by adding the verbose option (`--verbose` or `-vvv`): + +```bash +pricehist fetch coindesk BTC/USD -s 2021-01-01 -e 2021-01-05 -vvv +``` +``` +DEBUG Began pricehist run at 2021-08-12 14:38:26.630357. +DEBUG Starting new HTTPS connection (1): api.coindesk.com:443 +DEBUG https://api.coindesk.com:443 "GET /v1/bpi/historical/close.json?currency=USD&start=2021-01-01&end=2021-01-05 HTTP/1.1" 200 319 +DEBUG curl -X GET -H 'Accept: */*' -H 'Accept-Encoding: gzip, deflate' -H 'Connection: keep-alive' -H 'User-Agent: python-requests/2.25.1' --compressed 'https://api.coindesk.com/v1/bpi/historical/close.json?currency=USD&start=2021-01-01&end=2021-01-05' +DEBUG Available data covers the interval [2021-01-01--2021-01-05], as requested. +date,base,quote,amount,source,type +2021-01-01,BTC,USD,29391.775,coindesk,close +2021-01-02,BTC,USD,32198.48,coindesk,close +2021-01-03,BTC,USD,33033.62,coindesk,close +2021-01-04,BTC,USD,32017.565,coindesk,close +2021-01-05,BTC,USD,34035.0067,coindesk,close +DEBUG Ended pricehist run at 2021-08-12 14:38:26.709428. +``` + +You can run a logged `curl` command to see exactly what data is returned by the +source: + +```bash +pricehist fetch coindesk BTC/USD -s 2021-01-01 -e 2021-01-05 -vvv 2>&1 \ + | grep 'DEBUG curl' | sed 's/^DEBUG //' | bash | jq . +``` +```json +{ + "bpi": { + "2021-01-01": 29391.775, + "2021-01-02": 32198.48, + "2021-01-03": 33033.62, + "2021-01-04": 32017.565, + "2021-01-05": 34035.0067 + }, + "disclaimer": "This data was produced from the CoinDesk Bitcoin Price Index. BPI value data returned as USD.", + "time": { + "updated": "Jan 6, 2021 00:03:00 UTC", + "updatedISO": "2021-01-06T00:03:00+00:00" + } +} +``` + +### Use as a library + +While not yet optimized for use as a library, you may find `pricehist`'s source +classes useful in your own scripts: + +```python +$ python +Python 3.9.6 (default, Jun 30 2021, 10:22:16) +[GCC 11.1.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> from pricehist.series import Series +>>> from pricehist.sources.ecb import ECB +>>> series = ECB().fetch(Series("EUR", "AUD", "reference", "2021-01-04", "2021-01-08")) +>>> series.prices +[Price(date='2021-01-04', amount=Decimal('1.5928')), Price(date='2021-01-05', amount=Decimal('1.5927')), Price(date='2021-01-06', amount=Decimal('1.5824')), Price(date='2021-01-07', amount=Decimal('1.5836')), Price(date='2021-01-08', amount=Decimal('1.5758'))] +``` + +A subclass of `pricehist.exceptions.SourceError` will be raised for any error. + +## Terminology + +A **source** is the upstream service that can provide a series of prices. + +Each **series** of prices is for one pair and price type. + +The [**pair**](https://en.wikipedia.org/wiki/Currency_pair) is made up of a +base and a quote, each given as a symbol. Sometimes you will give the base +only, and the quote will be determined with information from the source. The +available pairs, the symbols used in them and the available price types all +depend on the particular source used. + +The **base** is the currency or commodity being valued. Each price expresses +the value of one unit of the base. + +The **quote** is the unit used to express the value of the base. + +A **symbol** is a code or abbreviation for a currency or commodity. + +The **prices** in a series each have a date and an amount. + +The **amount** is the number of units of the quote that are equal to one unit +of the base. + +Consider the following command: + +```bash +pricehist fetch coindesk BTC/USD --type close +``` + +- **`coindesk`** is the ID of the CoinDesk Bitcoin Price Index source. +- **`BTC`** is the symbol for Bitcoin. +- **`USD`** is the symbol for the United States Dollar. +- **`BTC/USD`** is the pair Bitcoin against United States Dollar. +- **`close`** is the price type for last price of each day. + +A BTC/USD price of the amount 29,391.775 can be written as +"BTC/USD = 29391.775" or "BTC 29391.775 USD", and means that one Bitcoin is +worth 29,391.775 United States Dollars. + +## Initial design choices + +To keep things simple, `pricehist` provides only univariate time series of +daily historical prices. It doesn't provide other types of market, financial or +economic data, real-time prices, or other temporal resolutions. Multiple or +multivariate series require multiple invocations. + +## Future potential features + +While the following features are relatively low value or beyond `pricehist`'s +primary use case, they could be built upon the foundation it has established. + +- **Time of day**: Sources sometimes provide specific times for each day's + high/low prices and these could be preserved for output. This would require + changes to how dates are handled internally, clarification of time zone + handling and extension of the time formatting option. +- **Alternate resolutions**: Some sources can provide higher or lower + resolution data, such as hourly or weekly. These could be supported where + available. For other cases an option could be provided for downsampling data + before output. +- **Real-time prices**: These generally come from different soruce endpoints + than the historical data. Real-time prices will usually have a different + price type, such as `last`, `bid` or `ask`. Support for real-time prices + would allow adding sources that don't provide historical data. Start and end + times are irrelevant when requesting real-time prices. A "follow" option + could continuously poll for new prices. +- **Related non-price data**: Trading volume, spreads, split and dividend + events and other related data could be supported. The base/quote/type model + used for prices would work for some of this. Other things may require + extending the model. +- **Multivariate series**: Would allow, for example, fetching + high/low/open/close prices in a single invocation. +- **`format` command**: A command for rewriting existing CSV data into one of + the other output formats. ## Alternatives diff --git a/example-gnuplot.png b/example-gnuplot.png new file mode 100644 index 0000000000000000000000000000000000000000..f73ca28f39cbbfd48fdd15c9da0f9884eee184b5 GIT binary patch literal 32357 zcmagG1yq!8w>JI^Lkc3&4bt66DG1Ua4bmkI(kV4k3P?&wcc-8-&icOpdC&RJT8e9UX71Ye-q*hNwT;wJQ@DML@)iU^x0MuSwIB!$0YR{NY&7sr zOT9`x_<`f9Xy5@s_+6+!uw+hrDhQ&7lw_rKd^5La{C#y+n)xpMZ3fd1#faP^!U(WS z+3fAhum~%1jXIwt%T#@A==-bP_vgBB#bM6W+{~=J^g15?ZjT#+ zKn6SSl922i9rX|QFc|s0Faiz;4~cy*Oao>P$%CJ_AUK~4_B|bl9B~&sgocFPfQLyS zO!F`TnNo-kVgx@i5dZe0^kvWkK0epshbKCEdXzC=J=TU`$e&e4a;YC08V+6=RQ=Sg z(6bjxKRaBb6Zd>gx4pUP;OMAbrn9rN!^p_!hKU;SAVCz8ppGTs)cftXmbJ9Bq*vkf z=XWz?em}!3=*&93*gpNw#>PV2?+^CcUS1z3oq`s^$-p2An#4VBjz25?Hp@!5Nc#^~ z`bC_6L69aE>Pzbwlut!Jn_VBI(ghxka@MpFik}wo25#iNqHS5r_LsrBjrsWL)2G_n zM9^>FrI@+1As?IJ?7)*5pB$-RTIjd4kFM_c9_r<~oI7eaaF7gkPDYN)4YSQ>cXqPg z)-3qy_{ozebC;_k4{7f}qFF=7TN6|+OGaiqh{Z3%1ajrGat>DySlov9m+Il za=j;&uT=)N01FGN0DpCP5p^t<%u2pp;5FOsf3iJUpjGy>(%|s7$MDtOE2)5`7!mA{ z_+G1vlU=h`@8?gSrmC`NSB!V&e0TUr8r^LrAcy~=efC@Dx09Xi?SSP3g#bx9=-Y~W zkVehK*73Ox?{}N}eW{GiJ1JkbtPU73q+TWlXE|l5zRyx8t>KKW(PO%aR^E1Z-wz4D zGPA0mJf*`sI19)SahYv$dsfaGFA8ZBAGhxY4qFMpkVNVa_QAwp2(^->j@B2)n=AdP zbYkvSgXzLNshp?UY9;k~d7XAb5w|&#`$|I-b}x8u4tU9Uj5XMZse5{@1cKt1n!&(VERLqri#TwEM< zR6w`F1c?X>KT%cPGrPFDHW#3j9s6Kd_0vx1MSGB)UFrMxccB;l{!c6QN56mnJ~7cl zx4Q2e@#>Y4kx@x;F#{Ik+|!x~UZaNX)eM(aQRl6nRb{67rMB6AEQe(5;}gBha<_ue zm~X@<#nUV$a=1*NlA2X1UXvyNT3KGih~5mo3keAe7bQ^02zVgxJUkll7=9=sF7D7c zZT4cLXxgQN2J-&%+rEC1Ji_mAP((db)UBVp5kgLVbZmS+y?XrOWM_I}Y^>gWrSIoY zz0l6uzzgq=xtq(wVKAowaE>&pOFw|+-dj-4gv|j%L>WwrC;JyZl!+hovcl$f8t1Mj zg0ChX(Yp=`PqZBlNhBvHL$%iZDX-{)epv~OkB{$6eN?F!M?=QaNYDnIF0keVxc4S9 zOZfa*HWS=5|KsWDS$MLg6l=<(ZO~(-FqBurCD}uy|27oQ`yDCs?&QZp-PQ_ znU57e$+-Xg7R}ib-3__lJJXuEdRz5h^NZclt*xzZDF(9s{{5T(BFm|_?~M2|f6s+= zf;UmF=sdNX^LV zXZ9AeP(Ii^dJ*`7EZLW}QfA>I=MJ5u@j8(-_@t$>grNlv<$v>9#N8P#pV$c_nFu45 ziwr;5jSdI>6@*S#MLO79zI#oWU_sukfnOqSA0HpPH@6iPtvu~G*WL4?4Zhq@HGNT5 zQ^V`j|4M<@Z+Y(KY7U0n@C@2)S@ z*%Q&jxc|7${MLmCDzuk~L7R(7FmTpxKuJZ#*7#Gi6*}39v5oWdbGvlOpi9vwV~xMv z`{(E9!PVw9Z9B~X-l9I|itFYUv0VG{Pa|-$k%v5U*MA$c)YgzlWSXFDcF_4odOBs$ zW&6o&fqhGBtN&!3H4P0-5m{(-J1*D=4T}V($=TT;U;&vDKGjuKPq?ysf1l!!K;GNG zEb1q(uTMJOGT}h|Dcrc%+n)k2rlzLK%gc*PN}jb}|N8X{SXPhA`Uh}%M_eLh^CKc6 zc7ty&yj!oZq)t1*JU!fAhKEpqh>+(?9wGMPXdU=&m$v<{rC8LWuJfH(bjt-U?7%Sz11_7NXJDWx4-39QSFB>D8Ll%^3(bQ4i*eGn`+Xt)vE@exbuGdF5Braec0> z{;)gP#LUd4esbH}fA6dn4u^9-t;TzJ-7xLk3gS~NwV3->RbxbZH`tu(Zr&jNpA~ZIIY2nR8!%7}q>Fu1I4^>WypHo|*leO(?^@vAY`#J_rsyKj_ z%qL4b=k)qWeeKn(7s+`cf%2OicuKDoFyV@)HM6xAkAHmm;c%S5s36NQ3PSd6%rAY( zoWq%7hZl1<*T8;mb5nNwopN+Mv z@ELle;Vkzv2li;1%?r;0^NohVm;ZWeOjev9E??Sa0m-v9vPm$G%UK~svEFD*{`1J$ z^EX`i=vCFM4~by+H43RZ8Nv=w_^!Eo_c4W7Y@j}NvKVSQdOz^UA)y#8*1s0_biM+N zmt>^5hJcm4yvyMaa9|%dmo#Ho{y!7x=8|4=PQp$W`R8+$c=}T694S!@J$GfS(NNz^ z!U_NR@f+rx_wr2uKuF*{4od3GNo&p{B`dQ^7OpZqRR`=7ay2-kPB9-;V0m}1ZdXcZ zvz{+pGJz|)g@k#yNzrEKc0Qidpw{M6* zAfaLztri?fesV6#%b<2shwIW?FQ2%%2Mv6w)1p3hlYth|Rj7?VQ^r@YbIo8OQ5Y-c z$~ElldRU}jKYi>TD?f<8&vQmgLW^E`7 z4poe|IE-_|$O{HvUp{_?zyC^(=D$H;&9s8;H{~f@?+HCCpVx-CG84*JbGkv^qSU}S zHGv@C)6-L>xAnW(h7*6;OGc+Eape8GfX zi@zpVF;5;x-!Z3%;yh}$dQusa8_Eal#z~2gcK<7tOQrt%#rHk5OE+wUk&rp?geE2? zx55{PBzB#goo73OZ%lkezo~|=;|hqJJ8V5#!`a?Hn*uVK}lKs zT#03yiYMe%&l5?{J?j{U7|dz!%P{IR5Ms>(KpXV8KBxWqe2dq($ra$CwxjPVJgNO# z$M=PW_p!JPKa_TK%sJgxz&{W86pl-Fz{rtY-q7If>beKkn-H1+`(;x<>C}KiX`ZzO z1qJQx58vX-%SFfOmWxOFb7`Q#P6u3`AJTpAnZ%4PsZ1a@wEko-@BG?_T&7gf)7zU? zGCUIwiy6rTwH2ED)Fazt)nyl$dRN>r?zoC)-M)QWzJNbXx=4${i|-)t_(i)~$#jdCBjwZZu-7i+t12Vm zlBsVlk2i3vldXDLv=6VQzHIf>TzSh~E?K|$>;kTLJRiVs;1w1NR#r@gAhJb*3y|O|OA{@dg{XratXO=yK%oorfpCJ;0s=Y*3VH+5(aqm^r|g z(+$rLU$kHR-PzjMFlctS{?YkA;Msw*@ceuqc6bW*3toK3xEjm%4~pm#HoKEz21n26 z!=m&Sp#YF?x%{fN*!W(=&CUJP#B2Pmx+D%{)Zmo%?j2v6DwT-y?~jhDdKEFkP0xB_ z$;<*q?S%dVCM*F&OV8nP8AX$qn!5u}a&~s0u@azJXnY17=A_aCEIbI{*C~)RuD|{d z@M@un1%kuxnVGBsdJbD^To|uPdDo_Rq{=sn+GmLGk|~Bz$|lY&6>!7pen%0KgwLnM z$P6QB;7Rw#@yOQ@cR5tZ5t^D{kNDXWH*j7`(!bvKki;!0e^Opc%rq8#g>uAaoxs_k z&7lnjn|uh=r^dCNb9Gf6IL5Ru?11RU?-CEe7c zT#>@g(rPIwk!K|^|F981@RXM*ToN}-+1O!*0eeMp{n>?;{AZWjcNA0913z%G+UL7O zosE^>o3MK{!SykxApOtx%$v7|3wanWGY{4usK3JeG5YwAQR775xwd0vsym*Bmq^94 zK#>%NOn9-$z`C?g-Y{Ao;VTL-!=%sz2U`L2w3mD8udtY=2E`TJ1^6EXbybx!(ovYW zy4a4vdsZFRl<7`O4FofO4BX3FHr{9otp2lMB>JT9)SHo7#;%l2!$em!84bYM7BFzw{VdG z04yZuXsXi2xl?Wwb+qNDV>aX*Z>{zgis%1jSsBU6440${QziwGk65NPmr>9h9E}NY zS(Bdg8Iui``-=xeu=z&@L`77Zf(n}tq_jis2QLeM=u`Yi4slE`Yf|M2;vnTdQlwl_ z7dfV&A@h6ip3#|elFBrPp!RSit2Q5f{;uK7BRx~rob68n2i6Zo1TD)ko-3*9kLrr6n@d(RufA4C8kc%8TNyT7aV zA=Qb#63KmJS&4DpB0bMlQRHF#k$xf%L#PZaZT)b zg=vli5VINZek6&_{TIFJTrgfv7CFmEHc=iC!cXxtY%>KP%Cpg-F|Vl38}ESY8y$>} z@g_rwBenD~_NEH1n#x4q+FLOi&ft{7mmB%CVtlu2S%t2`IR(v!Wgj^+obEl?I*hWOE-#n#5u^SY ztSRjFgKDo(o8@E9lUlDsRVa2n>5~8)zjRYzM*E?H2NGlw<&rf*CY&PNnyP`BKL{OA zpqmPSulUThjf(3@*B2L_wHWDSl!#CwB$S)M*}m!a>{oQ5NqEEG^TKprI#`fM-#4|5 z%G6wsBp4Q%k1kJwdJuIWj@K}6afv_gE-&yId2OAL@t>gJok`qa?g2yf6W>St(M_BC zQS~lUq0tDx_h_yFhRf6PRDR>G^&2S!XY9 z0Y(mj#|j?iyt}Q~f=W$LhS-{)vY-M7;p3~SO~gf%M?6K>^+sBPIhGz*@fNZ6%!ZTZ z26etr%swf;{i-6(W1Vhc_t9iJVi}IaO`IArXp%#Etr&+h4Nsrz?&P_xcnh3XZA`X& zi}z+uPw#DZWafjgTld0XO_h_wT7`zxXX|;FuJ~i=lS~j-aG6c-Sn-HUQ4;KxmlN|a zzpUeu4KmDFwomq4&tSPRM9I}#c|`oZ6qW}PtYZ^XVBR8-6cY;2WAR)d#p0V0Btpds z8M06qh)R9Tcu4C(?JV;VqTV8J|Fvh7R?m6B?k?Xl4;eIh335b#`VL-EWg;;Tq@6A{%8AT_iCW;K$47M)zV53-WS@8gP^LDF{+)i3NZ)vFh% z#_u+O3jqRQiWAO<%BAr!=j-kVPSrTm<7UGYZUr*rsj}(~s}U@JR}7&EO=@6wgQ}lD zEqBPbM&2%T3>#9$B86Ue(Q8K9+on{EBv9;?nLQc)^w|Gw^cD38ZNsYkcy|Hq*PbU? zd*M9$XsEo25R70Wu{0s9G?wvbFw^l}EQvuV1>cG}@8x(wtWikBhvJt+YEVy!J$)9V zr=EDNFF-QGe;Hpg3eIbHq&o*aBEv&o!p8fFi4@usj%^|OLWW2!;|F)59+f7ULp+WR zV?kGF2K=eHz(=K5$6VKjamK1Ygsat|)AXz2h*Qz%w;8L#Ur8>Hj&OM3X(UBgwM*Z1 zI?d0l)M2bXZ#!7zAO2-MZ`0mj@^8~yJ8rHcUD?)## z$rJR68!koI$PwH*rySKWjYdqoF?ga24D+XTMj@C`n(20eCOP7uW*F)qj?8*ol_wl{P6 zS`ap{r2b}3m4Bh4ND%9RtbKhv468hqS+d>As{Ib9(A7$ngfJZ>8i6bSVyZWhk{=o{ zEEhfN^E@WYH^!wym@o?UKdLl)M<8R5jMF}=%gQ@CEVAd#2VNS-+>{T>NbE&$GD)g9 zw98P~AQ|}CDwt!XIUJeSN8lXKZ6zPQCNJPsHKW+Rk4 zq(`Kui1T6@A<;CZHx+WBA%9Eq;-zUN2f9JF7~TsEGtXC>n`EpixG8!Za=^|o%h zJsz$YOMZS3LE_NO71=KJg@ub{yZ4rK9BOECt1RwJ7O^m|b^i-BqbqY7k{kgXLTle39r@%{lPnwRwK! zOnTRSv<=HM)J4MOGS0y$h7_j!8b z5$=?EEZw9#QdcmuMrO&Uj)2WDg}+()-|iqZ+F3m*ld*qh7r$_Qy-_-5~czs0uW9nz_qI_ z_r1ML&3%i5EK_cxNI7GyY0%^lvHox-Kl|-T*zo5^nEO0NTZmBXw0HZ@a47RCl`bbG z0#0k$K)DPZa++xo7X3ao)kkOl(eP=NVbQ4F>Tpi*`pCVY+KP$_K#dn^eQa!ud-JA? zLiX%#5qj1f6tMrU@j%f1J(EPgZ73n>_h_lB+ixuhGY7=yty1n`C@nMm#=*_wQofuN z%ow~Qc}gFseoJHKGg6KEt`!Yg?VP>ZzCLYg1aOQ=WxTJID*3u8vM;j%0)S@_)SjLdOm2fp1 z##nm5wk4TFX4DSYcszL$Bg=q8RasSao9cJFzYnPR$>gDEpp1-+lifL~|Amo395A>~ z@eX?+$s~`sP7VJ~=PqJ_B+`8!+x+G*6#rcdOyh7P*ic4c(Hfabdl|1qCWOKi)m;>OW9m{@gDye! z`T2{>y>Cxu{apeBuNdPV8~~Pz0R7(EUcml#c<)SASM#K)7VDW+?}F2g?zR+2W58-R zby+rFzL;_X$MJVSa5xO#+tc$yV`GU&9ySDtlA@X3ce?E?39sFGV`f8dS*1QGdx`hU z{kxglN3JcGbnPeu*@J26n`hB}n_l&T*-+riln~p;y_LH%^UdQ!Z z`GV1{mNnZ2C^$0$A@Ox@XTv(VyjQq^>MZ0Oq*mH1) zx;bU`Ww})nIrtyDaSd8;bxOK5_1L+2)6S8pIVYV~@_$JNbgQN6Yf^6DN2})E4kW;M zB?9}*CW}j*wA&cAn*U4}pY9AY@GB-+4tGB~09gEyq8#2J3@iRHm<5}pCib7#NW#$2 z&=H#JV%(gOOS`bRc%PfGkIxCHIvgP3cOc|Ymc;Y>RA2a9KAo*b60tfruaH9et=>Bg z&!^i=cR?xZ#fuku)y9)kQx1IO4K8zkPA+nSug^gJ6BtHmsC}lm^5e?#XS3xa*^!Sf zxpT)-*x)urFJ4x?P!GZv^)&goPuYB2$g74$s$S0cfaqnC@(7|;lOahrQYt7gFc4JC zKn2k3tnVop@N3T$Fn#~oHa`*)5~;IZ^;w`z=<8F4e$`p`cXfsJT1_>%ErHs^2SaXX z``7!WZZb1KielZN4cPfOaXRrv{Ph97-Z+0S?9}p+LGoLOtA;)x<(J9G=K;hRHj?CN zKNss24%um+r{%CLeG#FMISFuY$^o3e+x!k!2Q$84-kkPG$jjY{e!Bdn|%qNJ(&Nab7%RXat0V#{j{V@vbHf5Wg zeb#N)lntDmaHSx+zBKL6MM!MxIHa=@+hb;96gW)q-F=zRcfZ;3D5{T}YMxvwbq}XY zjxEi8zLwRfluEI0$afGY1Wf<51m3&*xc3X8kc86M3~UGf01I91jA_kN@x zk&h?PFP>Mb+O@jw9dXx%QJIbW*&A%nG&*LIoHo+<4)+0)h zWyW8Ygb&S77V=pr)w&9rYjI~wuEYi7?N};#`C_V05_(B*VSnu@YQw|%4`}8WnFU)b zK+nnnRUjQNt97AT*EPf!9wx#bG!?h7%cxKH<%+^qphagj&hcmpx{Px*{a<0>8b1M) z@nhenu3fW4j-+3)!W5#uA;PCHs;b{MeO(l`kSp$Jd!_b~&2cHP#acWhPt@tNOeq3o zO`lV8568kVpv@Ob>ssb8TjX(}zmO>=H7DelQM7MnP|nx{6@DU0dAkOWGC@LI+jydXOsN5+ zQ)~h|nQ%(sIs|w91Tw_Dc9qXV2*zygyBxz5DB`$};h%*r3Ry`1Q5o@@4i6tE20LgJW#P#o`UZ5r1o~XphVSyT$&8-qzNf=Hfhr-lqv{o>We*aR)+zY9s$`xKk;;4y2OCE0NDUm43x<4Zq;} zoGThL^3SG(1wNDM-7_GU!*DxqY(T4q1LGIr8t_4->@hwqD)jK!!v_&t6Yb1^^P;}`lzKd8&&INT<#agh(lDj&&iyX~!UwwJLyJgfWbHneO zumTTYDyaRLc|CylF{&>57yIICSmHv4DQz3;2+6{&OA%Xn8{SAny=h;} z7z=7T$64ylG5De&hJC6Z$xxGH`Chea(MBKJ8jd;ty~h@bbEn?~r-dGH0U(7PQHy08 z$gtHmdDSO)_uZevSNN+EfZ?r_VN7~Rj!?<*iPC&WSb}Jt#;E{U2zubzQsj_Ur?HzY z_w+{U>szUI2ArWoN+w!RK~CaEeZYl-UW9oF;h`VGV+{D~DZlRz$m~6s>Ta3j;*pDd zkztciUicceXdXaKHJ{l0WDVpIBX~_#ubJ~BrhfoJ08`;s5^TwP1lt-5bDnT5>lRz0 zV(`6G3cj8Yx4W40T&1SPnWCFG*YD~7W$mPRNai7lUN(-mo3J5G%nfnI$^pfVY+~XeC-03%>3S_6m8EqnqD*ZBJkrEO27^PdA2E<&wHd^AzJu95 zwIei+XW>F!Ueq=iCj!clu!wEG*WEpo!qr=sS6h`jD-H829{D}#FOczKVh%h4uY3e# znNopbmIbm95>IscRyK@C;BKf9KqI#!rP^QJp(71{_wF6^-zvxo^8coS%u}clGL(Dw z{Ala1`i%$2W+NYbh-b{}V8c}EMl=k~8rg&o(r3lX4dMSQr-7WNK9sez0PNhc_Zb(+ z;R@I|HtGMpqEa)KX)X+d!U_o!A=;C>A+iJ|*0v&Oo*)*lUDy?Vun6(j#Fwz_s;g%$qyiQtEa z=Ws`U_)v>jqhsa6>!}qE6_vz5V;5Eq}Co)&DLcYm9M_e5hSuhzGe{P&2u&!v+vzc6f+nN3#*eb%of^lUI%xl0%i0PL4ncV5Nf-M+nX zd@f|lv0rksQIaDWcz*Qb$B#cVt-)7YX1l-Mf7v=NAK#pZ0ezz7dn-}$4q;~dL$d8ywP3E zM4R{+m2p6|00IJyDIJhuU0>|_G#huqGu3P#KMozWQ?=13(~+ln6JXYES~IrM;5?K5 zRrg+YaC(A6Hv_{Yx==gehc(MbHuOJ^2`BU)jj`RnO7Tz9TR?Yh=;g$gE3#@CHrr#V zd@EJ&{ZPtp=El*mEstTiUC7uq?oD z`0(NA_;{a@HqO@9SM;YjWl1cdGW%6{yaHPALE^ZdA#U~dpoOx4lm?WXv5;Rp)X>;Wqz1_#KVqc6&j zzXDu~nNUOXPtwx-rri8L?-`}0r?s8D(P z69Kv3a8U;D@z>kPq)_H0%R{NQ@d?hXFG)5HI=Z}Rr>(5pd;==u9be$1@;)r+B^7ZM$sq|6VNZE9yY2vnFJUV(`erFy#hzyfwq`IGqdcXJ z5Hp9c<;{RtTmuuh#kwR?*cVky)o7)JpqD=!V||_$WmN`{T3o*(TNJUdz|0h^mnr@w!JTwLNeD~Tp$pL6&w||?Z4_{jAgl8WzM^Y(3t29n2vjJi zbH(RCA{bnsXAv&`lQTE*C7>xbK>e!j`!~en?iv{^LGx(?WshcbZnqdr`8zE`yFOJ@ z60GBUap<>D1`Qt7lAOAKrV_||^KnHxxLcUr(>z>Lhj6y{osVu1zP%*VJw_IuDC)}JUl_ae(vzAuz#pP|zA7J^w0lY~aEW{B@BNT!@1U3qYmOf88=Nfpz zJ6?EItoVAFB+MTVWGPzULMXjIJ|lbJBzj@yy;UOl%*Ng>9~y{H-+S2b<9Jzw#R#`f zjDdrG8`#oA-x@9&Cxl0K$yP^cqC?kX|E#nhIrl7Zrh1iyPvfv)=72o(TzL4w^Vj+1 zxgG^H;IAQakj?p))DC*xK`e@j?iLJf+JJIbrkH&k?%>;T$UE0UATBVYKw($}@$E{_ zc;-lbQ7b*2mInHg5GxWwDS)_@X~4sM4{6u0x;^dQWd1p%GPH(rvpCzYIw%DULebq+ zMyPS$#3i%s)7?Qsf`nn?vwx|^S*2?C$AeijRV)QiZigtJItt>&yP>)Mz~|CwSeE~6 z+YN42A0LF9JlFSw=jq=rY-F1D0E6X2Lh`L)HawP2>>`*;fy_$1K#NUTP`2kXBAO28 zzJ+zT>^;)Ek6M3ybFsi8y%m}PfEi|6%q{RO-`;eA5`O@&TjhQ{S?`JYujMje8K2F8 zREBczK8KDZC!+}AA?=SdF4&1aee1EUY2{8XI~L>kA~jzN9xxnxkcsF>sys5i2Sc>p z5xTl=xyDuaZ2*dImexqHeHN~QTw&i4jhcCjDkGN@bMK6O|8``0{P}k9#V5I*i53sZ zhJ&&OP}T_5*k@j!r<=%1Pd#oN5mJU{+cDc^!sVZ$U^Qwsid=*IL9se(e`{*$o=CRM z(SQ(u&?FC6TI>dyhE^C%KmPOgIqEczS2n&KZ=>~Jhf(m|LQ(kk$mvr0a5ZR6XNhAF zh$JjRK|c2xl%v^h`%;k|B;Aj#59MQ2W0ASVnQ%ZZE$w5nN*lF01C%2Zr6o%h+2)=HI%pkAQXFT$529djxU@SdNo|I>u-o@zt zxZn&SRB3?$8X0~&HKY|kN9d_x^2%YmHe=t70S6%d2OScG=)V)-)>*b8+5Rxf4+ExtyMJQP`w_P_Gga&nCZGluOxV! zV9$-P85>$=2`PG6C+KH{MO|F|Soj4p=Gc}vx-w)BSb1HKM}nr_-DH*JAc-yV+*^U- z1`$45Fzgf{SrJipkXS025eyvl)0@ttlv@0X&qa*uEnTpEpgPj$KnSe{D)~1omyh_e zk|4~bxi2V7^;{t547AsRS(|9kAEr>oHFTGb!~7bVE~$U@JXLcj$Abp2_h%(@1(?^ zrll#zn%lAeYR8f~UoY@j9hd}F8Xz49=>PxC6#^yafLQtqW1`0gcLD#j+Zne?vS&pQrS6=VH2dO-4grj}|GP%~lUE0fo#EU- z=Wk_I2>KL22h8qlyRNS8%*+hX(1HZ;!=w}5m$tLj3|6(wrj-;U=-;VQ9UUt=&3}Cg zY@OQ$C4#k7ZX+9kw0!*470YmrO7s{FP}5LkGm3IW6Ky5@{ZlEi@>5}5ofa1<=m{i3 z-~T)9GMph&14>e?fcpmqt8HNL?_{YM2pS3Rw$sIE`GWoV&d{3nzmzw1npm1#d7y8A zMC^AAp|#z=paf)1fY5Gg`1rj>I&;6A1z%uI*aZ{UGI5QTNJa3$TFz%I&Sq>)>;KO2FX`Y5)S%OJ8QDG{gf)BjO_!Kl%>s zP#Q^v14$=SCL|vc+NzoVK@5}q`CDR$(plS|CCZtqavRWi+qLL1=vb*}1s^cru1g#- zuZhbXRK%dvgOACWfHUdrv^P(>zeZ4H&jfo^%u1VKs=q$g%dJ`!W%5gY1e-+4DNNoT z*9N%)!Cl`2wwI84%*%a(&1Aj)Q3y)bb{A00A!7j@4Qp#-K*GwT*h%Oq2Dq|qS4KM~ z(epC_K!$H0ke6?zSi+7bguTigTg&OFdMPQ~yLgk2Uf)qI-nx+~qkH(x5;*Zw2 z^oaeP=&Ghp%INhQUt3=9CK6^swn!Q3y*kutn1(D21*udrE#UaKtPvV&1s{$Rf#ws zK_)mau<}4XvYYRS)i$vw)d-6e>{k$Kt~{BTJhp%K5#HIJd)x|%JxkPl9OA%67Z`{w zG2yPw?s22_k`>g7;=mB!MB(4JxVONN%s}~+A(<2C>?TwxN&9#L)|FO$!8yvkI(VVcmgPJonoafP%;6dxU9vc#3A#RP8FOWuc~JJ zC;VT(66x&xlGHL_nEPMfos!$10=N^luFG~6fDoiqx}|8rA&`Tfo6q6Q>vb!%cS;>> z#mf_Mkk(C@IH+y>I!p-oApFI){3+-zCqsWWAu?vq`ZOqVf-bnxj;WQS9m#;-%AUaYv7QMKX`+j~0I zKkK+y|A03kA3(};5uk#+iTEz1PJ=`BZmg$s(CWVBA z1kq56^ap>kuGG_k&h4*iSH>n8DQZdKJdYPeV~Ny63rx1t=BDr;N}8?{-7T-h6AfY% z!kkB!1hM^OE?DZc2ehf`K7E=oxJqW~YXQ0viRI*ImjA6J!+dq1C$L_U(d;_dG0urC zYZwpQ3F)_NaxYcl7_IEh**gcY)KAVpvF)OA;9HJKxFp%azAAc>E+KZaSGU2fZR%pB zB7J9u15fvErZPx%>hnmrGA2nmEuk%np0Bu#@x@LRXR(k861mEk0F1PE4XWe>eCQSXg zZxldZ;qi-%^z>6Ho_~65z+6CcDJhTfx42TBa?l;v+1Xhdnr7Aul-w(}#1K7UWMzqR z;RIaJoXIW^_HO_E2iKqI)2L)Qtz6H2T#CxyY)~dbLr;bbhSi;p$uo`51u0%0ErI?( zR3oxN94!r94*L$!6nr$KdGM%W+>k4!G?e8ndoLs7+{Axt;yEyPtTCJ5Vzy#{G(_U7 z?}{&Y;h!QyrSYg<$4*LGZ7xBGQsfi zM0?y!BO*rK+pos&^Sc_Nfsp@3&q_l!K*AP%)dB?%NuXs?#W+Yq+kapA?-(g9d8phz6w>`*xA4_A8Vt z=2E9q>*?yN>>SL6;+J2Ga#%mg4I?o0LSpe$2_)`!r_w`o65iXO@3ca%3Jr>|sMK@x z@;V+ZP&j^ZH8P^{>=`3;^!IPI{(G5%QQ7~bPkC4VKP~Hqpo`zLrOK$0#e*U*IGi~d zb|EHNrJ(p)M$o)(C`6G@E^9-wZ-fH_dQP-K&XsVXR2_J*_~YZ;efy03T{eSuRlkF^ zh3WJfw%y?fEnxQ>U#P3hzJQ)nzVX@ zW*^fUpSe$gAX@+ps_2Ms|5LXQ=w5&Pg!V)?$i{{=uMn;F(H}i*Ig;Nv(fe-nXsB)v zTyyrsE3TsF402)E(1#~lJ@hHLZx4E-Qyg!K7=HR}JuQ88()zjh9v5ri;K_!2u!9_j zG{mA*cb*7j7w`3rO-!B;bfs_`v9PeHvW$V&V4t0-qHzZp{pLLAb~g>!mn#?rv}-50 z05QRnz_|J7PKH`^LVgagWPM>KLKvWdFkXvY_SzcpOtd2_VFSB4Vb1`A01*o0wDA>Kv#h*oGna;em(NybOCGj=X_&7_y-0Q{DKp=%!%LR&7M@WU{PcuI zTuVUL4tT|pea-~X=~zsJ%M4$gpvSm*t-BuPRB|Q2sM*XU&6edK=RN;iVLkS$K``hl zz+gLv&8^7Q!zXUkX+8>PkMI_%Q#34Q4$qMVeqZ@g54KHAl>mDNsUe@?*Y;k~Ieaqn z{}Y#5F)~(i6@fvH2p{m^&#i39pE3r0c2yZzfFV!V8?-{+moH|u!h9brzOOZZ(jk66 zn>seh5Of6h(UUy2(1cOt5k<8e%Ikn*9NK9(G#led@vi&q@N%eH3ma+b@ra-EBNXuG zi9+scI&JZwt{uwaoAV=dfQe*81hJfKi^SjmX4_{@fT`1*#013;Xp5D{ZasWYe%m3Vk+GBU(1*=W*J+KGo0&2l|LTIj5n3X_8?FQew#)`;Vi#~Ze zwa=KH;?*KpGkrP61z=O|zQ5ZZg)8`P9IoxZK9qK(Dq6kSFFrSg-N`spqq6|k07$8< z$K;45H?v$_VXMU@2w06f|D{li5XB+KHECl#Kb|xEJk96 zul<_d^5u-6QHzcn{x=;CkVe!^`c2QAXR3{BoM4eiT~0yLc;bN6#O3a-JVwHzcRJDS ztBdW=h>d@Z1us=JOM5NUII~@l+Ofqz)M>CDUTFzTyY9-x+epB>JW*K=(?N$$0!14) zc%J7CW9JA4^*?*_PkHQ8#B;!Yw!dD)kW;`1Ck=pN0V}zvlb02mRIRkh^AnF8v?DR6w&a2mKM);Mv$C^J zU-WOZ{7w5m-MwX0)o~jwc<4iSmxM@3Hv-a#v`9%KC7>WBQb(nf29XAl4hcbNIFxjY zgp`DcfPfOx_u*Z0@2oZR-nH(0o%01H{_(3P_uj#$#kkhp2zztRel`5Gr1g$RQ84Ro z=lHvCKB1v9-Y4$+yWNmy=CPuj*!OBtQ2xXDK;)g!Po#Qbp{Cq-SCI=L1`vmE%L;fa zcJnEYlf3!z>RZb_zdzKsxAKEeliXhQmQyk0Y^`x$wgC) zEtB8E-5=|Muayujbb5Iy7*Lve)%9wL2z_Tx-lF>T7v%Xi4x6PD6w^iIzSo9!Tc7I_ zL%9YL4`^snw(4)7z?`%%#0EE@C>q9HBIdc*u0+?S5#oPSMC zy_JuCKkK4|DY*MwqDJLWIY14dQysbneMJzdf-}Pse}!Ys|B-qqaYF~$IE%!h!%>6I z7hbU~YYn&8j>J4#A-=rD;gVCs`V9^|h!t)uT$F#T81s`SM7}E#uhsAw5Hx|~LK~ia z<{7alq&%YgK|A}$kfSHB@j2xYSf~=UC4rV^DKH__#>v|q^?K{^+^VmmS2uqwq9^h(8 z(>d1=fuk6J(}T+OQnWfRzbX{?IlFfwIZI4)B;wjBf+-jOwk6N5w%QSs3`EU$eB_Hv z*Xqfxth_U64dmcd2~kp!t+&>Ql))SDRa8GOYvb=NhQz**3OZT1T)kT$C55DZF~0BIzxCs=z&Z zceSMVQu~fsB}Pjug5@V_ZBU+*Xv14Hfi4ejsJ}&UN=X?Ta$=+qyE|T3uYGH)Jde^O z%^|~^h3LlhiyKqIFsl0Q^O%We)OjUW=ay42tk{~Rt+B6?q3rC@WVr+uuJl6ojtjv~ zUQ0!Ih_Y*eYMW@K{7@@ArrBBHbCEouUPrVo#fLxjIvE&~w`T1^TOU!8&J1D_)_E&F z4LJ)Ql zx3HO{mBbG8(;m=!kkBR3yj1G^XX31ZKQptj9q`zklfqQz_+J~*?U7v8x{X$;>R_5; z8Z5jT**09W7*E5pHIeU3?wU82YbZ8`*C~7uF(09WN(9Tt3Rxs@olY>pUZY%pM#zdw z|E2o8@2!t2nv7{2hNsYXuk6wGa^lxUQudkUD%ORQ(bro6$?{qB4(MDwi~U-2OZ~Kg z?XSU>vS3%kUj)xeuMq`HYTUq!vKTQYXfA$dh=1af{Q#Hgc*E@n;X!q~ID_8g?H}`{ za?Rsm`;LZBIW>iEdyL(_>Qtptv*_{^+56D>8lwxYunulhZqD}3vB&ww2=x6WOh+et zd3jACt+#dfJXvnh-p!FaVwb;Nua6gYYuqWx#MNP0@)618ebGQswoOC^Hx>yg`m#_o z6wsg%O*hqfPeocE>zpwT{fH@qN;2?E-Z(DvXYpLe~oyEW8Z0FXQ4Ed*aF%$yIn64o$37!#O#g>@)wcAP1iQUb+ z@(tt`BRQUEgwTfQt0D+>8E1MQGg+^R^15BrG`3Z8H?;|+V06!fC(vHQ!X+0g(_^IR zY4^b#`KSTw^|Qj&b8SYWBDcWv1`=uq^!lzjk;X^cmzdku&J+h1QVWJRA8NK@&$&1L z*n3pa5%g>4csEh-UbTN#)ea^4$WJhYAbgu^-Qc!+4)|7^AMGPe2IOT zorq2zcG=dih6oNFjp)2Fj8w$FU27P0G){vUxkP*?nuTH4QSe|Fw9>W+OJh`nCYJY$X}5B)f3WUdV7CC$Az{!6OPU*WE-2#56gM%f+Mf z>`(+x>O#IT*x}3cNUJzkV)0>NlA!ScMk|7-7VG{eZ4lUXH5_L#{xE~M);Vo^JnP#Q zYIfQ8drt==n_cI2%Q_eQP)k~-MqVAaEWoY4+ zxx8r*#Kz3}dF#O!ikgCWFXHiDTx-`5T*LrMlxqei%kLc}RBdr---D ziQZ}G6Xc_4l8i>)V1E65Z>GSxdV8CdR!L(_;#T5(`nRF6dQi38Ta7m$!tkio;tJKO@+HxHJ=_a#V&&?y*CvcCz_o$QB&)TOI|8Bpcl-6Oq zr!#l;`z6tX_0 zC-R&MvOl|gv3Cyc(H4jEU10(n5xr&Stg-P?_TKYi0YbkEr>1XAY=sN-jZQYWI7m(W zm>Y8lQkr_*s%B~OE$VD4u>WGO*jcQa{pa<3v({J??f-R4e`5><4Rqij#-ZIWmk$-S z*rmuYY@#sf%`NAzGv-4>zHP7Gel-7TQ%k+@t2x8#b(Ev`nRUvsq;sd#oBrp;Edpn3 zTKLOEdztfWZqYptYOKy%udA!g@tDmNmHiRL<139Kw@Z0`-=W*PIanPX+3>Y9ob6{D z{&(!nRbHpIvtaT^0@c6SeOQJ|YWKe$_-Qw`p-SCRHZj+8)~VkF4Jo$we2{XpKcsL! z5ziD$Tpn!ez^-v@7+YdpZWK)URf=h8knt~)nWJe#RtY&MN5P?GCvYPj9Y5y%K1c`O z*O*>)eIR9ewt$e$2wNHF#uKb-i-N7EkActR28*sh@}q?oW|!HPK-txnFJBDH^otDz z%)*Hfql#-Yc+hB2e)Ip)*Zm^QNWuRFJmd;Qf<5{@;Z}#8N-En$?47*|bOq z2@BJno;3-Ce;B(mWZgCJL)-#+>`eR2Ue83O7)wwc?~G0gV{&4m`S@3Z;pRWlm|S%a4In zK-gg__1NXIHq}$mC~0I2b*hC&%}EsWti8%^D@>bEtY0DKv=$l=t}|?$P+KF7BkZ8; z7%*e}R9kyUUHQTv^WX`?m~{xgQvr)Y1xrPl$gA6Ous_q+%Q z&3hij;p$iejUB%ahlGUB>AziXZ*L@9s1jRp|N35d&>0}pR##VDCH&i-+F&plP8-7gBSpuPLZ*Z1{?{{yY(obLweeat7-b^f?9i zh08(71r=oR#4m5?>@}_Cl9vwx%AVVA`1QG|o;J%13zPc?Jbii$MR_1XH8W%Q+P?aD z^|7Cun;QTS{d4^>jfxFvC!j;E3xrgC$kDo{YXG81M?o^&~ zC%B=@c_1r0;x($DIG)n<$ss0dvVa9I50A&g;?hz|g3utQzEY1Acyte`4B!}rhFT68 z83wjJXcd_F@&y2X@EyCZSJcoA3~)DKH`?Wcz5$@HnwNYUVA*(W%3@v%9+qCqKL0{_ zM6m?*o~!Yey?rc7)x_E)*VEZha2|U~hF^gTB|NlIIl#b}shp4PbyU1SUHPH!icV|t zXBFpx0~Zo;aJreA15-b(=I1;UvsYf$0-=;@Ouwy3h^` z&AEiwhvjr1okrh5wjFSHKUPxGOcfTRgpim5qs7UM>dn%$S`IQyq6ZzH|A%QqZ1 zBikEk|MkFB_V#)_toK?^2*#iK7NU}GFg`X0Y1JPFPdP#c97JQgm;W9q<&dSbsUe~2 zo?lSFb?q8d#qF-kWR~aTLet5l_r;IOwn_>GX2`T-JCoCJTEo*Ca^d7W zQ>g#xT5?6Y?2NH_k2dHoT4$pYvjvlotrJ<8c2u}^@AFiA6%tmOTBIiF1v2L~>tpF7_>fLQjy$1Ew2CBVo* zYlkB8x>ZX6aIHOUKHr?KhBj&FkiD&~P5n{@2M0$j*JHBW!qn81;$^2@ZpGQb4-pfR zcU}(a^;xSY*g^y|WMcn$=sV{!w(Z&M&3<{-oT4{Ihijw$%9LHXi~cPC1NG63MmuqX zr+x71Ol;xg+wkzW{u2#2H;gAvW4Si7f5BJnaAyNb@wujK23h3Q@JnH7$kIDg-M9xMoz>33>U#hC7Vnu87sN7 zOr{yEBlbO3KKM?zlrcKcBGgXkI2G&MRq-tUVti?A%CL+)|HHX#_1Ah#H6*^esD%K5 z#YwS@Ek7ZfN)h&rjB6+?mqqpuZ{MPucb8)od>&B|gcj;%`#3W}6$9#Q+uFIhM9eg8 z+>a)eUyYg8E^Hh~V&J(TWtQS?8p6mB3Y{0P!sCpKXP70^M%{xIaPL%jqg@^yl>pEr z4L8v`Z)42fqEuBPTJ%Bj+dpw*M`gUO3{p013!`HJ`m#rY@kln_%=V=531WvJrgN)V z>dF0F&$!ADed06_E zTxrRxpkO+xpUGV}dc8VyA8-0@(j#x#T4g9k_3XT(Ps59>ys_OWL}&9|>`p~ELEV0d zkw})?&&SQTCm;M!5hhbAiwJ+p&cytq$>YvAj`qMNo@%Q#RU(owM_caejtZ`;3LRNa z8md`=l#WehO>dov_s;`TYU3A7&(GaYb)8$O_>jQN?@`mno^KjdLcQFK<9_vz0fR4L)z!pFFEBCAzZ3Ko7m+_Lp^XO-$H zM9C0UE;bn4#yIwBngB$ZTIasUXWl5NfUhV&QY%B7b8I3 zF6P7!LXjaO8P9(%O$@S-8XOkONLv*-i`?_Vkbki}M4l8(%YnrbdLNUe4_hKd)kM21 zEk*ipUMXR>Oy_d4L--t>?p0zl{gbv{DaL%*Sdyf@#vWoq(eRgh4?(?i7CIuQkZS9o zVCXb9{iV`f^V{Tc`~_tUnWj0#yngwl*gT)=OzY@^^vsL7a=E$mI;k6vG5Ho%ceW1w z)jIxpE{0s*Iq~~XrjiNk>XWSJ%*(@cm-p@Q5H7R6YF$fEw}EH3r$wR^L9OV?68hT( zhX(gvxcA`;7VTU=!z`B)yIVhVjpev+<`#IGRm^{D%{h-cK3d_;%5pkioBk!G76Kdx%Z?@ojYAdvrVUPHJCz*oSRmJ{N=8 zZHsfe4}W7#Fm)`sZX!Qgjt>1ZHtcP1@RN}A5N2hgl76D?I z&BNZFWMk133a{Vz^@ml>?{D6R_1RCQW}80!_L%{=j?;ts*YjsqR+U2Z(0q%ZN@aA! zh~{t%G`JSoSlajPnBP==>suUG(+c|exOj@&Yp}bSE2(Sw(>DZaF?GLn)Zu+J&pY8z z#R`-x?_T`*i$g*Lg8A>IMgZmS<}(Yv#UNl9#96%?_0F^Dm(2>ck?+UeceNj_#?^K1 z>(f)oq#u9Dq$RN&S0&(0^BlSj>Lzw}89ac!)QuP$T7 ziMu)YKgu*sA)-YS?2t1f_TKe9OiCIeSr(p6*VbRUn>@ooScdQ@ zrk;72hD?~|B=4DGEHskX-?Sb_PyMStX^y9yV#Z(y1z~_LdLa)Q>zrhiww#elPDm)J z#qilw{K&E;xqj?pt>#bahblPbE(nB@dnsx^o^t!D^{0^X!(%Z{D;xZ}{a0T-ac(@D zpnt3HL>EjF7IzC>JHKvpZhiAfM#pjyYM;R6%rsiEb!k^Km5DxZb(qCS0u!N>>4{Tj zY~o52qTTr}oX`5jSl^<)vke9#RB~nRfx)_Jr7Hl8m3u5BniP>ZZEZhKMjw2ie^!k{-#U6T~olaap*7e9yjz%qi|P_m$4CUZPtnnJLP5SaYmYbvnA&W7q=K8`NtQg!$$~ic^L5;L0xtG z75WkaIySK%M-g>LO+hSi{_MX?6+TgaUT{8D@^^D-WYx71pB5)}XyZ1)WVU8YKn**q zo$Ed^k@{HpS!~Y}9iwV8L*u!^sq?q%S>dz&Od1J8G=hkc6x_ONkm)EQB8XOPQyazf zCrL;)A+I@N?UXOoJn4!dzn+X+&N|zM?H(6$6H~&)HB*vEe_VQuU&lX{q?^sjE%K^N z`nVT4JSke4iFT>d$;}2c?RM2q6D0GLXIvAlya*8+=NV4p--5PSeWw%zNdlP$ns`lY z3K8kbTe#(NgJS0lp3Om2GK?*|*ZYEI`myN=>-YC>`VVL1vVi{c3LitHZ7qBH^t|~y z@76EXc2qKS$|Lge!z-U&ADcr|&BU1#hl>2-Q6CTz#~(RoRKU7a>i#x-4ORd6=>UIu zF4AZ+om{CwqQ5>r+9P(iM*HxM8JbKC(V?A!dhn}CoW{j=t!}7ChC)a{d1UAvM3ybF zN_;I7ia1O}aOG`dbbR8Jg(MLZPo_vsvk${_3D4E2pVPBH2+cx9c6r1v2G5VkYD=7) zJ4$9oo1DVH6CwMMwkeUD*+%-T^*Z8;UZuWMvfGGI8L}a#|2f&%Clhv-Hqpf1YB{3C z{p}g6Z+@duQ$dq{7#C#Cnj7!MZ+-s2R9epdS?KzYoZre|`VH=RZ_)U0W-6)vp`ZO) z;q@r&?)i_>cn+@W>oV!8j)hg+vvgq%Uqh@h=}7ea&c3Vf*<_NM#S;7WNY#73y9w3dSfh z|GBU%!~Fa?yXuYfwpxX=ZkL4YJI^AM25&>OJB1LULoWsO)qE-ERP`|Yt&47HRp!%O z>dZmsmX!(lw(H3sXK4CYT2*ixZ+Fmqr`+k7Sjl*v66&}VUPn5`&TkQ!%F${Z5o^ig zb-~adnd6eij}&-;gRZ2a(Qy+T@2b;A7f+T^M89kzyc_rNT|cUz_%XAyP)+8bPQispRgFuU-(!P z_+GMvWyV*W&bS?caW_1 zFtr3JS0*?^6>VPOovU~9TAKiR=8F8v@+KR%jQa&k)# zC8TzbBbU+D8xUiUQs4-2HBwn4I;~ym8kx`zp%cYJAgtQR_Q*K)7LsiX$Zc?Fl$_cf z@a-3ymB})-^F#Gh*p-6cV;~U2+)jH0ivP(DV&OkQlF8g9b41EKWD)c!H~wZL!r??D zz;;Z8K(yG#TGv=kutDBagX{*zI5Gj%+)Y$y8~G0dab-nAbb4!Y56Nbcp33n;)uc`J zCw3H46nje!H~D$f)H(wKfz(LrauytCOTl8H_-|Q6$ny}=3atD+;^E-|dgRT*s-~s@ zKrfgJNXp7KR#jDHtNC_<=g_QwJCp;za!mq(`1g#pyE1LFK_c?gM0+UP&nt}oo1C8$ zgySNqT9EmhYm`(_Shz3Z=UVe^%g@h`QNs1-uU~1>o<9J3f+G7f^^@G=2dY`p$F6fK znjBI>2*fF{z0_EZE8^V1#yvlgKiRCFiUtLX-Q{k`FS3Q`WZf{ev>bA%rsX$@d;Qwb z(6HvouI=IHx;m-E-WrENhL91+YvdUe8x%vYpZg=K$gbbI+8K985C|cyo)1&nR~v-n zC(aB^uT(ZceaeI%$b|eLv-j{}xl+!^#{SEfhZ6Iey1K3YhvnU$0eFy?pKo*BT)=Yd@^Vao4B)0lqQm2vmNBIxS?s1`%8GlnzX5GxUp)=V4Nn0-4Q;Tohl?2* zAiV$_R?U{%^W5)%UcZ*A{W39uAF8B4vU(RM0BFu$wCNcc9RN=jNORoervd)&?&b#{(>lHr zeHdN-JNDkjC}#9rC|mjRg5omaeJqNFRo1|4*2q{k8msIl+jH}OW`B$f3_bwoSB{Q5G2 z0M;?fic3N>d;3*@Hs6o{p`TYO@18%$1nx1H$?~!?{+B9CRiho0>CO>0`y8AI1meZ6 zis{&rC;pSe=ci4YXjrmS(v2Whu~6eospJIhitwR$d4KtHG_<4=lag+iR#jH+>`z+J zMs~sHQR|t6k5+870JlS0rVYNyTeu`5UJof-dfdNDxK2#2+Nabc3u_Ot=L|ATK8L^^7Vh|ngy8wLTT$o?A~1O(!y zZ5#^zk6^lq3C9=09f3~=U(gtkZ-Nir|L=duyiG#12n)ccQSRC`yzLll3g_qS+E<1& z=&9HrnMZi##Pc)J=0)j1)_luHAt?SQeqCN(NI(V6fBm(gZEPZC*tejrhO2|FeY9X% zX^oNG9~~0|iZzm7>p)AXFA$VuOh7#gga+8Mhjnk=s(-S}{?)|NQr=@pMIbpnGjk3O z4_&(*bC3{$dr5WUzeerhK>#{|f1rpNQ6@+ec3+Ubx7|Z6?V{ruE-K;{5E!~I0{MMhwi^3>8fdCF4wBa2dxSnLQ zvNw;Hu}ef%Hgplo>}D0X!DI=c$}5&fzD$Q!@fw`L$K%_h}QuD z)(tvQfyZJJS2o*SCH_9SA)hTsMZnBcT~fj;E-o%DUEk2a;4-@I1xU6)aoj6Z@V8gI zh%!)wgoLC_QajM84&)}$YPpjeSYV{4)zJr7<7_>RxS90ut60$D1S zZn*=Z@2wv9CG%Yd9o4p;x$WR_>S1!_gBoG# zEuRyhq7i%Wg`267$|N*2bbWmtX0N*uyr^Sif)J*$rg|ZmCP?0|oS&V5%>$95#LP^& ze|8jN7_ET$+uhxzc)0>oHo5MF%$c2?#lj6x!|1t{Ij3bH!yI+V+q^mk-dGs z1}8Xbb$hdqzy*SmNeru^ow5)x}hd$Pe-6t{f-LxhfqilD2? z@f5UYymuBAL*gbLoSkh43npnoNeQH!nmapHW@Y9oKp!MtL(KQLIZJHvn>V1zAr=xh z_b?bJUG9sd8uHFn+2I}Z!1%quBQ40!f7!JJ$7fbnR!hvS5$w|`yV$dT%dvxlgCHRl zv%L>Oh?gz{tmpgpSF+^%ceZj^0j&!)ixwaWh1CpzX}SF`cT7%56;9*g<3}eZ?&|4P zK3>&S3aqU(;fF$VO!6WuXMi(^r4LTUilhQ@Odx3AT=EJ19ae}H2Vl~}xrTkPUe|Q9 zIt#}w{V&cAv!5)zG_7_Jmyr1S^{Z|4)8>}Eyu44J4jQ*x;7t#8GG#(Pf{h4rlZIXBdQOenIvWlyP`@c`MEN$dFlnKj?XrWPs#oU|;~8t>w85 zJQ(v=)O;~4a-e_+kQPvhyNn&4gm-@a432rKQqIbcAD}^Fd-XL;HOQY)Qcz5}N|Xn( zn%LNUFS%RFP1Of~Pr@p{>9hb+lO`r6U~|&MUBJy3EfbcO_5=048!V)Z16LTVtcsxs zYy~6I*vQD{1p{$<8erbYYJDlGHs$K0qS9GrS`B;{&+i{U^{kim`?CV`$VyJ$w)R&N zR@;5KJ?g~XrpK$VbP8|l=zy{%$HDyNuzszu5~PW&vONS2+AfwR(8Pnn9H7Mg{yKC! zM#vtM?a$H5WIhD15+K!|zHPxGjL{V4&YetI-i+WRUp^5SSX7{G8gXSLI= zCjMadVzN7-v|NK&sR}Vvkl&SG!b^e14Tte;1k=_zGQu^Tk%?tdth)- z4m=wpqc@;l)DuF}lEGJ~cUra+QMqb;aigW)oMjmawpCjo*{+sZkV*Jf{#fn&(+v>V z^WW{Fg~^Y+}96iJTuj+TK8^XrZC_S5#bF@BSS#ykok_ zuR1qV^Qu$xJZZefN>3e!T2w@I^!wsskD!}yVyLT!4?1+b1I^m5a~FJjD9@|MFoBJq z`CuQJTUA)-2;9MI!`ayqZfQoP0Cb9JR9Z^9EO^4|n5?jhJxfVX_lM(t^P7Q-i;I_+ zm{L(o%lz)2P!&D-W`j%ZKwv^0pP6ZJo&FFPF@%$;Q)~8q%B^%_W@2Ksj*qINZI99r zx%=-&nv6Fu0P$4TG;YHqx*IZ^Xe+sc*#pBF6&YD=_8z|b1jfL)+ZMd9?Z|rgPa!#Q&qGa zU+z(;_LH=nc>xsg0|4THPT3K1{0PJ+L+~k`#LR+%&VjV7>eg=vy^fEcZnlB2-otzM(np;@cga{Gn++_r zL^^T0h5vefGMsVZusC7!iD+w&I@O56p(7t?#!u6~`Uf0Io6E2sjvb1~F2KCOQ*tg| ziPGXq7qQp0w9NWTe+O#X5cO;5>Rv7ZShVj&Ex~9}P*8wl%yba?c#xBe1kE2Xdfm3F@ENg_fjcKrjGjsJ^~F z*mGEh7dsuK*?!1 z^Hs;aDk*u%9yiqA&$tsGQiOu@o0AL)9-QUV^{Y)Mm^X{c19qVaV*;XP5C(N_l3luP z5Tq9;!krClbEjYyCDOGE|KkB4@2y|1h^A8_HWeJ#+Hb(ihJ%SQ&y0bnGAV0lXbAeM z9oGTlwDIfL0W?Yk96fG2`Ck_^Zw(3rWiR+>ad|@6i$}r~5fOoIyz?RR$!^HfZi*boqbem{gF!^Z{5yaA;muOANB~|<4)BW?%8+|0R!CCn;(8^(8LfLsL^)5-TXHV;UCCH zf{5hSw^|U9?(gf{^f>{`@wMCn#8)rvN2->C7y~sy7nnw1H__%Azmzzgoi*Q_1ScPk zv?-6}&zWFvyWtKT0*_U`$GrB47vaWBP*wlnZXPGnu!s}ti0LB%s(p}*Xq*$EA^;Kz z$1)EXG z?7f~5zasNQ38nx zeRUf2ud#aHahOU8XwX}rU$A%jg#51{Ni8t2`ER)ov^@ Date: Tue, 17 Aug 2021 12:58:49 +0200 Subject: [PATCH 048/149] More README changes. --- README.md | 161 +++++++++++++++++++++++------------------------------- 1 file changed, 68 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index 8d02e6a..06e2842 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,10 @@ support for multiple data sources and output formats. ## Installation -Install via [pip](https://pip.pypa.io/en/stable/) or +Install via pip or [pipx](https://pypa.github.io/pipx/): -```bash +``` pipx install pricehist ``` @@ -36,33 +36,44 @@ pipx install pricehist ## Usage -The following sections demonstrate different uses of `pricehist`. +### Fetch prices -### Online help +Fetch prices by choosing a source, a pair and, optionally, a time interval: -The `-h` option will show usage information: - -```bash -pricehist -h +``` +pricehist fetch ecb EUR/AUD -s 2021-01-04 -e 2021-01-08 ``` ``` -usage: pricehist [-h] [--version] [-vvv] {sources,source,fetch} ... - -Fetch historical price data - -optional arguments: - -h, --help show this help message and exit - --version show version information - -vvv, --verbose show all log messages - -commands: - {sources,source,fetch} - sources list sources - source show source details - fetch fetch prices +date,base,quote,amount,source,type +2021-01-04,EUR,AUD,1.5928,ecb,reference +2021-01-05,EUR,AUD,1.5927,ecb,reference +2021-01-06,EUR,AUD,1.5824,ecb,reference +2021-01-07,EUR,AUD,1.5836,ecb,reference +2021-01-08,EUR,AUD,1.5758,ecb,reference ``` -Here's the usage information for the `fetch` command: +The default output format is CSV, which is suitable for use in spreadsheets and +with other tools. For example, on the command line using +[gnuplot](http://www.gnuplot.info/) we can chart Bitcoin prices: + +``` +pricehist fetch coindesk BTC/USD -s 2021-01-01 | \ + cut -d, -f1,4 | \ + sed 1d | \ + gnuplot -p -e ' + set datafile separator ","; + set xdata time; + set timefmt "%Y-%m-%d"; + set format x "%b\n%Y"; + plot "/dev/stdin" using 1:2 with lines title "BTC/USD" + ' +``` + +![BTC/USD prices](example-gnuplot.png) + +### Show usage information + +Add `-h` to any command to see usage information: ``` pricehist fetch -h @@ -98,47 +109,12 @@ optional arguments: --fmt-csvdelim CHAR field delimiter for CSV output (default: ',') ``` -### Fetching prices +### Choose and customize the output format -Fetch prices by choosing a source, a pair and, optionally, a time interval: - -```bash -pricehist fetch ecb EUR/AUD -s 2021-01-04 -e 2021-01-08 -``` -``` -date,base,quote,amount,source,type -2021-01-04,EUR,AUD,1.5928,ecb,reference -2021-01-05,EUR,AUD,1.5927,ecb,reference -2021-01-06,EUR,AUD,1.5824,ecb,reference -2021-01-07,EUR,AUD,1.5836,ecb,reference -2021-01-08,EUR,AUD,1.5758,ecb,reference -``` - -The default output format is CSV, which is suitable for use in spreadsheets and -with other tools. For example, on the command line using -[gnuplot](http://www.gnuplot.info/) we can chart Bitcoin prices: - -```bash -pricehist fetch coindesk BTC/USD -s 2021-01-01 | \ - cut -d, -f1,4 | \ - sed 1d | \ - gnuplot -p -e ' - set datafile separator ","; - set xdata time; - set timefmt "%Y-%m-%d"; - set format x "%b\n%Y"; - plot "/dev/stdin" using 1:2 with lines title "BTC/USD" - ' -``` - -![BTC/USD prices](example-gnuplot.png) - -### Choosing and customizing the output format - -You can choose a different output format: `ledger`, `beancount` or +As the output format you can choose one of `beancount`, `csv`, `ledger` or `gnucash-sql`. -```bash +``` pricehist fetch ecb EUR/AUD -s 2021-01-04 -e 2021-01-08 -o ledger ``` ``` @@ -149,11 +125,9 @@ P 2021-01-07 00:00:00 EUR 1.5836 AUD P 2021-01-08 00:00:00 EUR 1.5758 AUD ``` -The formatting options let you control certain details of the output. The -following example removes the time of day (to make it suitable for hledger) and -makes a few other adjustments. +Formatting options let you control certain details of the output. -```bash +``` pricehist fetch ecb EUR/AUD -s 2021-01-04 -e 2021-01-08 -o ledger \ --fmt-time '' --fmt-datesep / --fmt-base € --fmt-quote $ --fmt-symbol left ``` @@ -165,12 +139,12 @@ P 2021/01/07 € $1.5836 P 2021/01/08 € $1.5758 ``` -### GnuCash SQL output +### Load prices into GnuCash You can generate SQL for a GnuCash database and apply it immediately with one of the following commands: -```bash +``` pricehist fetch ecb EUR/AUD -s 2021-01-01 -o gnucash-sql | sqlite3 Accounts.gnucash pricehist fetch ecb EUR/AUD -s 2021-01-01 -o gnucash-sql | mysql -u username -p -D databasename pricehist fetch ecb EUR/AUD -s 2021-01-01 -o gnucash-sql | psql -U username -d databasename -v ON_ERROR_STOP=1 @@ -178,31 +152,32 @@ pricehist fetch ecb EUR/AUD -s 2021-01-01 -o gnucash-sql | psql -U username -d d Beware that the GnuCash project itself does not support integration at the database level, so there is a risk that the SQL generated by `pricehist` will -be ineffective or even damaging for some new version of GnuCash. +be ineffective or even damaging for some version of GnuCash. In practice, this strategy has been used successfully by other projects. -Reviewing the SQL and keeping regular database backups is recommended. +Reading the SQL and keeping regular database backups is recommended. -### Source information +### Show source information -Some basic information about each source is available: +The `source` command shows information about a source. -```bash -pricehist source ecb +``` +pricehist source alphavantage ``` ``` -ID : ecb -Name : European Central Bank -Description : European Central Bank Euro foreign exchange reference rates -URL : https://www.ecb.europa.eu/stats/exchange/eurofxref/html/index.en.html -Start : 1999-01-04 -Types : reference +ID : alphavantage +Name : Alpha Vantage +Description : Provider of market data for stocks, forex and cryptocurrencies +URL : https://www.alphavantage.co/ +Start : 1995-01-01 +Types : close, open, high, low, adjclose, mid +Notes : Alpha Vantage has data on... ``` -Symbol information can be listed for most sources, either as full pairs or as -separate base and quote symbols that work in particular combinations: +Available symbols can be listed for most sources, either as full pairs or as +separate base and quote symbols that will work in certain combinations: -```bash +``` pricehist source ecb --symbols ``` ``` @@ -216,7 +191,7 @@ EUR/CHF Euro against Swiss Franc It may also be possible to search for symbols: -```bash +``` pricehist source alphavantage --search Tesla ``` ``` @@ -227,12 +202,13 @@ TSLA Tesla Inc, Equity, United States, USD TXLZF Tesla Exploration Ltd, Equity, United States, USD ``` -### Inspecting source interactions +### Inspect source interactions -You can see extra information, including `curl` commands to reproduce each -request to a source, by adding the verbose option (`--verbose` or `-vvv`): +You can see extra information by adding the verbose option (`--verbose` or +`-vvv`). This will include `curl` commands to reproduce each request to a +source. -```bash +``` pricehist fetch coindesk BTC/USD -s 2021-01-01 -e 2021-01-05 -vvv ``` ``` @@ -253,9 +229,9 @@ DEBUG Ended pricehist run at 2021-08-12 14:38:26.709428. You can run a logged `curl` command to see exactly what data is returned by the source: -```bash +``` pricehist fetch coindesk BTC/USD -s 2021-01-01 -e 2021-01-05 -vvv 2>&1 \ - | grep 'DEBUG curl' | sed 's/^DEBUG //' | bash | jq . + | grep '^DEBUG curl' | sed 's/^DEBUG //' | bash | jq . ``` ```json { @@ -276,10 +252,9 @@ pricehist fetch coindesk BTC/USD -s 2021-01-01 -e 2021-01-05 -vvv 2>&1 \ ### Use as a library -While not yet optimized for use as a library, you may find `pricehist`'s source -classes useful in your own scripts: +You may find `pricehist`'s source classes useful in your own scripts: -```python +``` $ python Python 3.9.6 (default, Jun 30 2021, 10:22:16) [GCC 11.1.0] on linux @@ -319,7 +294,7 @@ of the base. Consider the following command: -```bash +``` pricehist fetch coindesk BTC/USD --type close ``` From b5cfdaad1ed564a95f35f7c2ea846f1491382bb9 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 17 Aug 2021 16:53:45 +0200 Subject: [PATCH 049/149] More README changes. --- README.md | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 06e2842..a277082 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ support for multiple data sources and output formats. ## Installation Install via pip or -[pipx](https://pypa.github.io/pipx/): +[pipx](https://pypa.github.io/pipx/). ``` pipx install pricehist @@ -34,11 +34,11 @@ pipx install pricehist - **`gnucash-sql`**: [GnuCash](https://www.gnucash.org/) SQL - **`ledger`**: [Ledger](https://www.ledger-cli.org/) and [hledger](https://hledger.org/) -## Usage +## How to ### Fetch prices -Fetch prices by choosing a source, a pair and, optionally, a time interval: +Fetch prices by choosing a source, a pair and, optionally, a time interval. ``` pricehist fetch ecb EUR/AUD -s 2021-01-04 -e 2021-01-08 @@ -53,13 +53,13 @@ date,base,quote,amount,source,type ``` The default output format is CSV, which is suitable for use in spreadsheets and -with other tools. For example, on the command line using -[gnuplot](http://www.gnuplot.info/) we can chart Bitcoin prices: +with other tools. For example, you can generate a price chart from the command +line as follows (or using [an alias](https://gitlab.com/-/snippets/2163031)). ``` pricehist fetch coindesk BTC/USD -s 2021-01-01 | \ - cut -d, -f1,4 | \ sed 1d | \ + cut -d, -f1,4 | \ gnuplot -p -e ' set datafile separator ","; set xdata time; @@ -73,7 +73,7 @@ pricehist fetch coindesk BTC/USD -s 2021-01-01 | \ ### Show usage information -Add `-h` to any command to see usage information: +Add `-h` to any command to see usage information. ``` pricehist fetch -h @@ -142,7 +142,7 @@ P 2021/01/08 € $1.5758 ### Load prices into GnuCash You can generate SQL for a GnuCash database and apply it immediately with one -of the following commands: +of the following commands. ``` pricehist fetch ecb EUR/AUD -s 2021-01-01 -o gnucash-sql | sqlite3 Accounts.gnucash @@ -175,7 +175,7 @@ Notes : Alpha Vantage has data on... ``` Available symbols can be listed for most sources, either as full pairs or as -separate base and quote symbols that will work in certain combinations: +separate base and quote symbols that will work in certain combinations. ``` pricehist source ecb --symbols @@ -189,7 +189,7 @@ EUR/CHF Euro against Swiss Franc ... ``` -It may also be possible to search for symbols: +It may also be possible to search for symbols. ``` pricehist source alphavantage --search Tesla @@ -205,8 +205,7 @@ TXLZF Tesla Exploration Ltd, Equity, United States, USD ### Inspect source interactions You can see extra information by adding the verbose option (`--verbose` or -`-vvv`). This will include `curl` commands to reproduce each request to a -source. +`-vvv`), including `curl` commands that reproduce each request to a source. ``` pricehist fetch coindesk BTC/USD -s 2021-01-01 -e 2021-01-05 -vvv @@ -226,8 +225,8 @@ date,base,quote,amount,source,type DEBUG Ended pricehist run at 2021-08-12 14:38:26.709428. ``` -You can run a logged `curl` command to see exactly what data is returned by the -source: +Running a logged `curl` command shows exactly what data is returned by the +source. ``` pricehist fetch coindesk BTC/USD -s 2021-01-01 -e 2021-01-05 -vvv 2>&1 \ @@ -252,7 +251,7 @@ pricehist fetch coindesk BTC/USD -s 2021-01-01 -e 2021-01-05 -vvv 2>&1 \ ### Use as a library -You may find `pricehist`'s source classes useful in your own scripts: +You may find `pricehist`'s source classes useful in your own scripts. ``` $ python @@ -292,17 +291,17 @@ The **prices** in a series each have a date and an amount. The **amount** is the number of units of the quote that are equal to one unit of the base. -Consider the following command: +Consider the following command. ``` pricehist fetch coindesk BTC/USD --type close ``` - **`coindesk`** is the ID of the CoinDesk Bitcoin Price Index source. -- **`BTC`** is the symbol for Bitcoin. -- **`USD`** is the symbol for the United States Dollar. +- **`BTC`** is the symbol for Bitcoin, used here as the base. +- **`USD`** is the symbol for the United States Dollar, used here as the quote. - **`BTC/USD`** is the pair Bitcoin against United States Dollar. -- **`close`** is the price type for last price of each day. +- **`close`** is the price type for the last price of each day. A BTC/USD price of the amount 29,391.775 can be written as "BTC/USD = 29391.775" or "BTC 29391.775 USD", and means that one Bitcoin is @@ -315,10 +314,10 @@ daily historical prices. It doesn't provide other types of market, financial or economic data, real-time prices, or other temporal resolutions. Multiple or multivariate series require multiple invocations. -## Future potential features +## Potential features -While the following features are relatively low value or beyond `pricehist`'s -primary use case, they could be built upon the foundation it has established. +In the future, `pricehist` may be extended to cover some of the following +features: - **Time of day**: Sources sometimes provide specific times for each day's high/low prices and these could be preserved for output. This would require @@ -328,7 +327,7 @@ primary use case, they could be built upon the foundation it has established. resolution data, such as hourly or weekly. These could be supported where available. For other cases an option could be provided for downsampling data before output. -- **Real-time prices**: These generally come from different soruce endpoints +- **Real-time prices**: These generally come from different source endpoints than the historical data. Real-time prices will usually have a different price type, such as `last`, `bid` or `ask`. Support for real-time prices would allow adding sources that don't provide historical data. Start and end From f501cc8f9af428b4b021fbac76f6fc5ea7ab34c6 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 17 Aug 2021 17:01:23 +0200 Subject: [PATCH 050/149] Last README change for now. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a277082..53a1844 100644 --- a/README.md +++ b/README.md @@ -269,7 +269,7 @@ A subclass of `pricehist.exceptions.SourceError` will be raised for any error. ## Terminology -A **source** is the upstream service that can provide a series of prices. +A **source** is an upstream service that can provide a series of prices. Each **series** of prices is for one pair and price type. From 6708800699acaaa1801fde6a569cb4b80dc655ec Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 19 Aug 2021 13:29:38 +0200 Subject: [PATCH 051/149] Simplify README example. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 53a1844..2c21d00 100644 --- a/README.md +++ b/README.md @@ -304,8 +304,8 @@ pricehist fetch coindesk BTC/USD --type close - **`close`** is the price type for the last price of each day. A BTC/USD price of the amount 29,391.775 can be written as -"BTC/USD = 29391.775" or "BTC 29391.775 USD", and means that one Bitcoin is -worth 29,391.775 United States Dollars. +"BTC/USD = 29391.78" or "BTC 29391.78 USD", and means that one Bitcoin is +worth 29,391.78 United States Dollars. ## Initial design choices From fc2561b19e5b735cb7fb58a72a1624562a6564be Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 20 Aug 2021 11:31:12 +0200 Subject: [PATCH 052/149] Version 1.0.0. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 52a3341..18d5acd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "0.1.7" +version = "1.0.0" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index f1380ee..5becc17 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "0.1.7" +__version__ = "1.0.0" From c97728615348f8b07931774c832f683ba0243438 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 20 Aug 2021 12:28:53 +0200 Subject: [PATCH 053/149] Include image for readme in package. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 18d5acd..2752c0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ homepage = "https://gitlab.com/chrisberkhout/pricehist" repository = "https://gitlab.com/chrisberkhout/pricehist" include = [ "LICENSE", + "example-gnuplot.png", ] [tool.poetry.dependencies] From ed44502def546a4d19e951b4030164a4944aa1cb Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 20 Aug 2021 14:30:25 +0200 Subject: [PATCH 054/149] Version 1.0.1. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2752c0e..c0cb56a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.0.0" +version = "1.0.1" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 5becc17..5c4105c 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.0.1" From 98d71392c2822957370bf8321df508bb80896a4c Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 20 Aug 2021 18:07:22 +0200 Subject: [PATCH 055/149] Fix example image link in README to work on pypi.org. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c21d00..b5095ea 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ pricehist fetch coindesk BTC/USD -s 2021-01-01 | \ ' ``` -![BTC/USD prices](example-gnuplot.png) +![BTC/USD prices](https://gitlab.com/chrisberkhout/pricehist/-/raw/master/example-gnuplot.png) ### Show usage information From 1430ce97f7154f74b8290f9738e18dfac21098b8 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 20 Aug 2021 18:08:04 +0200 Subject: [PATCH 056/149] Remove unused fixture. --- tests/pricehist/test_fetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pricehist/test_fetch.py b/tests/pricehist/test_fetch.py index 89759f6..61af29b 100644 --- a/tests/pricehist/test_fetch.py +++ b/tests/pricehist/test_fetch.py @@ -36,7 +36,7 @@ def output(mocker): @pytest.fixture -def fmt(mocker): +def fmt(): return Format() From 799aaf37ccfecf72e023ee06a685c2402023db7e Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 20 Aug 2021 18:08:40 +0200 Subject: [PATCH 057/149] Support use via bean-price. --- README.md | 56 +++++++++ src/pricehist/beanprice/__init__.py | 77 +++++++++++++ src/pricehist/beanprice/alphavantage.py | 4 + src/pricehist/beanprice/coindesk.py | 4 + src/pricehist/beanprice/coinmarketcap.py | 4 + src/pricehist/beanprice/ecb.py | 4 + src/pricehist/beanprice/yahoo.py | 4 + tests/pricehist/test_beanprice.py | 141 +++++++++++++++++++++++ 8 files changed, 294 insertions(+) create mode 100644 src/pricehist/beanprice/__init__.py create mode 100644 src/pricehist/beanprice/alphavantage.py create mode 100644 src/pricehist/beanprice/coindesk.py create mode 100644 src/pricehist/beanprice/coinmarketcap.py create mode 100644 src/pricehist/beanprice/ecb.py create mode 100644 src/pricehist/beanprice/yahoo.py create mode 100644 tests/pricehist/test_beanprice.py diff --git a/README.md b/README.md index b5095ea..ddc647e 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,62 @@ pricehist fetch coindesk BTC/USD -s 2021-01-01 -e 2021-01-05 -vvv 2>&1 \ } ``` +### Use via `bean-price` + +Beancount users may wish to use `pricehist` sources via `bean-price`. To do so, +ensure the `pricehist` package is installed in an accessible location. + +You can fetch the latest price directly from the command line. + +``` +bean-price -e "USD:pricehist.beanprice.coindesk/BTC:USD" +``` +``` +2021-08-18 price BTC:USD 44725.12 USD +``` + +You can fetch a series of prices by providing a Beancount file as input. + +``` +; input.beancount +2021-08-14 commodity BTC + price: "USD:pricehist.beanprice.coindesk/BTC:USD:close" +``` + +``` +bean-price input.beancount --update --update-rate daily --inactive --clear-cache +``` +``` +2021-08-14 price BTC 47098.2633 USD +2021-08-15 price BTC 47018.9017 USD +2021-08-16 price BTC 45927.405 USD +2021-08-17 price BTC 44686.3333 USD +2021-08-18 price BTC 44725.12 USD +``` + +Adding `-v` will print progress information, `-vv` will print debug information, +including that from `pricehist`. + +A source map specification for `bean-price` has the form +`:/[^]`. Additional `/[^]` parts can +be appended, separated by commas. + +The module name will be of the form `pricehist.beanprice.`. + +The ticker symbol will be of the form `BASE:QUOTE:TYPE`. + +Any non-alphanumeric characters except the equals sign (`=`), hyphen (`-`), +period (`.`), or parentheses (`(` or `)`) are special characters that need to +be encoded as their a two-digit hexadecimal code prefixed with an underscore, +because `bean-price` ticker symbols don't allow all the characters used by +`pricehist` pairs. +[This page](https://replit.com/@chrisberkhout/bpticker) will do it for you. + +For example, the Yahoo! Finance symbol for the Dow Jones Industrial Average is +`^DJI`, and would have the source map specification +`USD:pricehist.beanprice.yahoo/_5eDJI`, or for the daily high price +`USD:pricehist.beanprice.yahoo/_5eDJI::high`. + ### Use as a library You may find `pricehist`'s source classes useful in your own scripts. diff --git a/src/pricehist/beanprice/__init__.py b/src/pricehist/beanprice/__init__.py new file mode 100644 index 0000000..151cfb2 --- /dev/null +++ b/src/pricehist/beanprice/__init__.py @@ -0,0 +1,77 @@ +import re +from datetime import date, datetime, timedelta, timezone +from decimal import Decimal +from typing import List, NamedTuple, Optional + +from pricehist import exceptions +from pricehist.series import Series + +SourcePrice = NamedTuple( + "SourcePrice", + [ + ("price", Decimal), + ("time", Optional[datetime]), + ("quote_currency", Optional[str]), + ], +) + + +def source(pricehist_source): + class Source: + def get_latest_price(self, ticker: str) -> Optional[SourcePrice]: + time_end = datetime.combine(date.today(), datetime.min.time()) + time_begin = time_end - timedelta(days=7) + prices = self.get_prices_series(ticker, time_begin, time_end) + if prices: + return prices[-1] + else: + return None + + def get_historical_price( + self, ticker: str, time: datetime + ) -> Optional[SourcePrice]: + prices = self.get_prices_series(ticker, time, time) + if prices: + return prices[-1] + else: + return None + + def get_prices_series( + self, + ticker: str, + time_begin: datetime, + time_end: datetime, + ) -> Optional[List[SourcePrice]]: + base, quote, type = self._decode(ticker) + + start = time_begin.date().isoformat() + end = time_end.date().isoformat() + + local_tz = datetime.now(timezone.utc).astimezone().tzinfo + user_tz = time_begin.tzinfo or local_tz + + try: + series = pricehist_source.fetch(Series(base, quote, type, start, end)) + except exceptions.SourceError: + return None + + return [ + SourcePrice( + price.amount, + datetime.fromisoformat(price.date).replace(tzinfo=user_tz), + series.quote, + ) + for price in series.prices + ] + + def _decode(self, ticker): + # https://github.com/beancount/beanprice/blob/b05203/beanprice/price.py#L166 + parts = [ + re.sub(r"_[0-9a-fA-F]{2}", lambda m: chr(int(m.group(0)[1:], 16)), part) + for part in ticker.split(":") + ] + base, quote, candidate_type = (parts + [""] * 3)[0:3] + type = candidate_type or pricehist_source.types()[0] + return (base, quote, type) + + return Source diff --git a/src/pricehist/beanprice/alphavantage.py b/src/pricehist/beanprice/alphavantage.py new file mode 100644 index 0000000..1f17a80 --- /dev/null +++ b/src/pricehist/beanprice/alphavantage.py @@ -0,0 +1,4 @@ +from pricehist import beanprice +from pricehist.sources.alphavantage import AlphaVantage + +Source = beanprice.source(AlphaVantage()) diff --git a/src/pricehist/beanprice/coindesk.py b/src/pricehist/beanprice/coindesk.py new file mode 100644 index 0000000..8936456 --- /dev/null +++ b/src/pricehist/beanprice/coindesk.py @@ -0,0 +1,4 @@ +from pricehist import beanprice +from pricehist.sources.coindesk import CoinDesk + +Source = beanprice.source(CoinDesk()) diff --git a/src/pricehist/beanprice/coinmarketcap.py b/src/pricehist/beanprice/coinmarketcap.py new file mode 100644 index 0000000..ae87a12 --- /dev/null +++ b/src/pricehist/beanprice/coinmarketcap.py @@ -0,0 +1,4 @@ +from pricehist import beanprice +from pricehist.sources.coinmarketcap import CoinMarketCap + +Source = beanprice.source(CoinMarketCap()) diff --git a/src/pricehist/beanprice/ecb.py b/src/pricehist/beanprice/ecb.py new file mode 100644 index 0000000..76109c9 --- /dev/null +++ b/src/pricehist/beanprice/ecb.py @@ -0,0 +1,4 @@ +from pricehist import beanprice +from pricehist.sources.ecb import ECB + +Source = beanprice.source(ECB()) diff --git a/src/pricehist/beanprice/yahoo.py b/src/pricehist/beanprice/yahoo.py new file mode 100644 index 0000000..43d479c --- /dev/null +++ b/src/pricehist/beanprice/yahoo.py @@ -0,0 +1,4 @@ +from pricehist import beanprice +from pricehist.sources.yahoo import Yahoo + +Source = beanprice.source(Yahoo()) diff --git a/tests/pricehist/test_beanprice.py b/tests/pricehist/test_beanprice.py new file mode 100644 index 0000000..9793752 --- /dev/null +++ b/tests/pricehist/test_beanprice.py @@ -0,0 +1,141 @@ +import importlib +from datetime import date, datetime, timedelta, timezone +from decimal import Decimal + +import pytest + +from pricehist import beanprice, exceptions, sources +from pricehist.price import Price +from pricehist.series import Series + + +@pytest.fixture +def series(): + series = Series( + "BTC", + "USD", + "high", + "2021-01-01", + "2021-01-03", + prices=[ + Price("2021-01-01", Decimal("1.1")), + Price("2021-01-02", Decimal("1.2")), + Price("2021-01-03", Decimal("1.3")), + ], + ) + return series + + +@pytest.fixture +def pricehist_source(mocker, series): + mock = mocker.MagicMock() + mock.types = mocker.MagicMock(return_value=["close", "high", "low"]) + mock.fetch = mocker.MagicMock(return_value=series) + return mock + + +@pytest.fixture +def source(pricehist_source): + return beanprice.source(pricehist_source)() + + +@pytest.fixture +def ltz(): + return datetime.now(timezone.utc).astimezone().tzinfo + + +def test_get_prices_series(pricehist_source, source, ltz): + ticker = "BTC:USD:high" + begin = datetime(2021, 1, 1, tzinfo=ltz) + end = datetime(2021, 1, 3, tzinfo=ltz) + result = source.get_prices_series(ticker, begin, end) + + pricehist_source.fetch.assert_called_once_with( + Series("BTC", "USD", "high", "2021-01-01", "2021-01-03") + ) + + assert result == [ + beanprice.SourcePrice(Decimal("1.1"), datetime(2021, 1, 1, tzinfo=ltz), "USD"), + beanprice.SourcePrice(Decimal("1.2"), datetime(2021, 1, 2, tzinfo=ltz), "USD"), + beanprice.SourcePrice(Decimal("1.3"), datetime(2021, 1, 3, tzinfo=ltz), "USD"), + ] + + +def test_get_prices_series_exception(pricehist_source, source, ltz, mocker): + pricehist_source.fetch = mocker.MagicMock( + side_effect=exceptions.RequestError("Message") + ) + ticker = "_5eDJI::low" + begin = datetime(2021, 1, 1, tzinfo=ltz) + end = datetime(2021, 1, 3, tzinfo=ltz) + result = source.get_prices_series(ticker, begin, end) + assert result is None + + +def test_get_prices_series_special_chars(pricehist_source, source, ltz): + ticker = "_5eDJI::low" + begin = datetime(2021, 1, 1, tzinfo=ltz) + end = datetime(2021, 1, 3, tzinfo=ltz) + source.get_prices_series(ticker, begin, end) + pricehist_source.fetch.assert_called_once_with( + Series("^DJI", "", "low", "2021-01-01", "2021-01-03") + ) + + +def test_get_prices_series_price_type(pricehist_source, source, ltz): + ticker = "TSLA" + begin = datetime(2021, 1, 1, tzinfo=ltz) + end = datetime(2021, 1, 3, tzinfo=ltz) + source.get_prices_series(ticker, begin, end) + pricehist_source.fetch.assert_called_once_with( + Series("TSLA", "", "close", "2021-01-01", "2021-01-03") + ) + + +def test_get_historical_price(pricehist_source, source, ltz): + ticker = "BTC:USD:high" + time = datetime(2021, 1, 3, tzinfo=ltz) + result = source.get_historical_price(ticker, time) + pricehist_source.fetch.assert_called_once_with( + Series("BTC", "USD", "high", "2021-01-03", "2021-01-03") + ) + assert result == beanprice.SourcePrice( + Decimal("1.3"), datetime(2021, 1, 3, tzinfo=ltz), "USD" + ) + + +def test_get_historical_price_none_available(pricehist_source, source, ltz, mocker): + pricehist_source.fetch = mocker.MagicMock( + return_value=Series("BTC", "USD", "high", "2021-01-03", "2021-01-03", prices=[]) + ) + ticker = "BTC:USD:high" + time = datetime(2021, 1, 3, tzinfo=ltz) + result = source.get_historical_price(ticker, time) + assert result is None + + +def test_get_latest_price(pricehist_source, source, ltz): + ticker = "BTC:USD:high" + start = datetime.combine((date.today() - timedelta(days=7)), datetime.min.time()) + today = datetime.combine(date.today(), datetime.min.time()) + result = source.get_latest_price(ticker) + pricehist_source.fetch.assert_called_once_with( + Series("BTC", "USD", "high", start.date().isoformat(), today.date().isoformat()) + ) + assert result == beanprice.SourcePrice( + Decimal("1.3"), datetime(2021, 1, 3, tzinfo=ltz), "USD" + ) + + +def test_get_latest_price_none_available(pricehist_source, source, ltz, mocker): + pricehist_source.fetch = mocker.MagicMock( + return_value=Series("BTC", "USD", "high", "2021-01-01", "2021-01-03", prices=[]) + ) + ticker = "BTC:USD:high" + result = source.get_latest_price(ticker) + assert result is None + + +def test_all_sources_available_for_beanprice(): + for identifier in sources.by_id.keys(): + importlib.import_module(f"pricehist.beanprice.{identifier}").Source() From ca63a435bd47f645b6799c8ec6142207b8106f10 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 20 Aug 2021 18:09:48 +0200 Subject: [PATCH 058/149] Version 1.1.0. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c0cb56a..a483d8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.0.1" +version = "1.1.0" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 5c4105c..6849410 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.0.1" +__version__ = "1.1.0" From 7b53204bcf342b77bd9c1feda86b4fd43916bf2d Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 23 Aug 2021 18:40:35 +0200 Subject: [PATCH 059/149] Support Coinbase Pro. --- README.md | 1 + src/pricehist/beanprice/coinbasepro.py | 4 + src/pricehist/sources/__init__.py | 10 +- src/pricehist/sources/coinbasepro.py | 164 +++++++++ tests/live.sh | 12 + tests/pricehist/sources/test_coinbasepro.py | 334 ++++++++++++++++++ .../2020-01-01--2020-10-16.json | 18 + .../2020-10-17--2021-01-07.json | 18 + .../test_coinbasepro/currencies-partial.json | 141 ++++++++ .../test_coinbasepro/products-partial.json | 62 ++++ .../sources/test_coinbasepro/recent.json | 58 +++ 11 files changed, 821 insertions(+), 1 deletion(-) create mode 100644 src/pricehist/beanprice/coinbasepro.py create mode 100644 src/pricehist/sources/coinbasepro.py create mode 100644 tests/pricehist/sources/test_coinbasepro.py create mode 100644 tests/pricehist/sources/test_coinbasepro/2020-01-01--2020-10-16.json create mode 100644 tests/pricehist/sources/test_coinbasepro/2020-10-17--2021-01-07.json create mode 100644 tests/pricehist/sources/test_coinbasepro/currencies-partial.json create mode 100644 tests/pricehist/sources/test_coinbasepro/products-partial.json create mode 100644 tests/pricehist/sources/test_coinbasepro/recent.json diff --git a/README.md b/README.md index ddc647e..f339b38 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ pipx install pricehist ## Sources - **`alphavantage`**: [Alpha Vantage](https://www.alphavantage.co/) +- **`coinbasepro`**: [Coinbase Pro](https://pro.coinbase.com/) - **`coindesk`**: [CoinDesk Bitcoin Price Index](https://www.coindesk.com/coindesk-api) - **`coinmarketcap`**: [CoinMarketCap](https://coinmarketcap.com/) - **`ecb`**: [European Central Bank Euro foreign exchange reference rates](https://www.ecb.europa.eu/stats/exchange/eurofxref/html/index.en.html) diff --git a/src/pricehist/beanprice/coinbasepro.py b/src/pricehist/beanprice/coinbasepro.py new file mode 100644 index 0000000..cb1a64a --- /dev/null +++ b/src/pricehist/beanprice/coinbasepro.py @@ -0,0 +1,4 @@ +from pricehist import beanprice +from pricehist.sources.coinbasepro import CoinbasePro + +Source = beanprice.source(CoinbasePro()) diff --git a/src/pricehist/sources/__init__.py b/src/pricehist/sources/__init__.py index 892de97..36bf676 100644 --- a/src/pricehist/sources/__init__.py +++ b/src/pricehist/sources/__init__.py @@ -1,4 +1,5 @@ from .alphavantage import AlphaVantage +from .coinbasepro import CoinbasePro from .coindesk import CoinDesk from .coinmarketcap import CoinMarketCap from .ecb import ECB @@ -6,7 +7,14 @@ from .yahoo import Yahoo by_id = { source.id(): source - for source in [AlphaVantage(), CoinDesk(), CoinMarketCap(), ECB(), Yahoo()] + for source in [ + AlphaVantage(), + CoinbasePro(), + CoinDesk(), + CoinMarketCap(), + ECB(), + Yahoo(), + ] } diff --git a/src/pricehist/sources/coinbasepro.py b/src/pricehist/sources/coinbasepro.py new file mode 100644 index 0000000..c56efbd --- /dev/null +++ b/src/pricehist/sources/coinbasepro.py @@ -0,0 +1,164 @@ +import dataclasses +import json +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +import requests + +from pricehist import exceptions +from pricehist.price import Price + +from .basesource import BaseSource + + +class CoinbasePro(BaseSource): + def id(self): + return "coinbasepro" + + def name(self): + return "Coinbase Pro" + + def description(self): + return "The Coinbase Pro feed API provides market data to the public." + + def source_url(self): + return "https://docs.pro.coinbase.com/" + + def start(self): + return "2015-07-20" + + def types(self): + return ["mid", "open", "high", "low", "close"] + + def notes(self): + return ( + "This source uses Coinbase's Pro APIs, not the v2 API.\n" + "No key or other authentication is requried because it only uses " + "the feed APIs that provide market data and are public." + ) + + def symbols(self): + products_url = "https://api.pro.coinbase.com/products" + currencies_url = "https://api.pro.coinbase.com/currencies" + + try: + products_response = self.log_curl(requests.get(products_url)) + currencies_response = self.log_curl(requests.get(currencies_url)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + try: + products_response.raise_for_status() + currencies_response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + products_data = json.loads(products_response.content) + currencies_data = json.loads(currencies_response.content) + currencies = {c["id"]: c for c in currencies_data} + + results = [] + for i in sorted(products_data, key=lambda i: i["id"]): + base = i["base_currency"] + quote = i["quote_currency"] + base_name = currencies[base]["name"] if currencies[base] else base + quote_name = currencies[quote]["name"] if currencies[quote] else quote + results.append((f"{base}/{quote}", f"{base_name} against {quote_name}")) + + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if not results: + raise exceptions.ResponseParsingError("Expected data not found") + else: + return results + + def fetch(self, series): + data = [] + for seg_start, seg_end in self._segments(series.start, series.end): + data.extend(self._data(series.base, series.quote, seg_start, seg_end)) + + prices = [] + for item in data: + prices.append(Price(item["date"], self._amount(item, series.type))) + + return dataclasses.replace(series, prices=prices) + + def _segments(self, start, end, length=290): + start = datetime.fromisoformat(start).date() + end = max(datetime.fromisoformat(end).date(), start) + + segments = [] + seg_start = start + while seg_start <= end: + seg_end = min(seg_start + timedelta(days=length - 1), end) + segments.append((seg_start.isoformat(), seg_end.isoformat())) + seg_start = seg_end + timedelta(days=1) + + return segments + + def _data(self, base, quote, start, end): + product = f"{base}-{quote}" + url = f"https://api.pro.coinbase.com/products/{product}/candles" + params = { + "start": start, + "end": end, + "granularity": "86400", + } + + try: + response = self.log_curl(requests.get(url, params=params)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + code = response.status_code + text = response.text + if code == 400 and "aggregations requested exceeds" in text: + raise exceptions.BadResponse("Too many data points requested.") + elif code == 400 and "start must be before end" in text: + raise exceptions.BadResponse("The end can't preceed the start.") + elif code == 400 and "is too old" in text: + raise exceptions.BadResponse("The requested interval is too early.") + elif code == 404 and "NotFound" in text: + raise exceptions.InvalidPair(base, quote, self) + elif code == 429: + raise exceptions.RateLimit( + "The rate limit has been exceeded. For more information see " + "https://docs.pro.coinbase.com/#rate-limit." + ) + else: + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + result = reversed( + [ + { + "date": self._ts_to_date(candle[0]), + "low": candle[1], + "high": candle[2], + "open": candle[3], + "close": candle[4], + } + for candle in json.loads(response.content) + if start <= self._ts_to_date(candle[0]) <= end + ] + ) + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + return result + + def _ts_to_date(self, ts): + return datetime.fromtimestamp(ts, tz=timezone.utc).date().isoformat() + + def _amount(self, item, type): + if type in ["mid"]: + high = Decimal(str(item["high"])) + low = Decimal(str(item["low"])) + return sum([high, low]) / 2 + else: + return Decimal(str(item[type])) diff --git a/tests/live.sh b/tests/live.sh index 268111e..021eee8 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -80,6 +80,18 @@ date,base,quote,amount,source,type END run_test "$name" "$cmd" "$expected" +name="Coinbase Pro" +cmd="pricehist fetch coinbasepro BTC/EUR -s 2021-01-04 -e 2021-01-08" +read -r -d '' expected < 0 + + assert isinstance(src.name(), str) + assert len(src.name()) > 0 + + assert isinstance(src.description(), str) + assert len(src.description()) > 0 + + assert isinstance(src.source_url(), str) + assert src.source_url().startswith("http") + + assert datetime.strptime(src.start(), "%Y-%m-%d") + + assert isinstance(src.types(), list) + assert len(src.types()) > 0 + assert isinstance(src.types()[0], str) + assert len(src.types()[0]) > 0 + + assert isinstance(src.notes(), str) + + +def test_symbols(src, products_response_ok, currencies_response_ok): + syms = src.symbols() + assert ("BTC/EUR", "Bitcoin against Euro") in syms + assert len(syms) > 2 + + +def test_symbols_requests_logged( + src, products_response_ok, currencies_response_ok, caplog +): + with caplog.at_level(logging.DEBUG): + src.symbols() + matching = filter( + lambda r: "DEBUG" == r.levelname and "curl " in r.message, + caplog.records, + ) + assert len(list(matching)) == 2 + + +def test_symbols_not_found(src, requests_mock, products_url, currencies_response_ok): + requests_mock.add(responses.GET, products_url, body="[]", status=200) + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "data not found" in str(e.value) + + +def test_symbols_network_issue( + src, requests_mock, products_response_ok, currencies_url +): + requests_mock.add( + responses.GET, + currencies_url, + body=requests.exceptions.ConnectionError("Network issue"), + ) + with pytest.raises(exceptions.RequestError) as e: + src.symbols() + assert "Network issue" in str(e.value) + + +def test_symbols_bad_status(src, requests_mock, products_url, currencies_response_ok): + requests_mock.add(responses.GET, products_url, status=500) + with pytest.raises(exceptions.BadResponse) as e: + src.symbols() + assert "Server Error" in str(e.value) + + +def test_symbols_parsing_error( + src, requests_mock, products_response_ok, currencies_url +): + requests_mock.add(responses.GET, currencies_url, body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "while parsing data" in str(e.value) + + +def test_fetch_known_pair(src, type, recent_response_ok): + series = src.fetch(Series("BTC", "EUR", type, "2021-01-01", "2021-01-07")) + req = recent_response_ok.calls[0].request + assert req.params["granularity"] == "86400" + assert req.params["start"] == "2021-01-01" + assert req.params["end"] == "2021-01-07" + assert series.prices[0] == Price("2021-01-01", Decimal("23881.35")) + assert series.prices[-1] == Price("2021-01-07", Decimal("31208.49")) + assert len(series.prices) == 7 + + +def test_fetch_types_all_available(src, recent_response_ok): + mid = src.fetch(Series("BTC", "EUR", "mid", "2021-01-01", "2021-01-07")) + opn = src.fetch(Series("BTC", "EUR", "open", "2021-01-01", "2021-01-07")) + hgh = src.fetch(Series("BTC", "EUR", "high", "2021-01-01", "2021-01-07")) + low = src.fetch(Series("BTC", "EUR", "low", "2021-01-01", "2021-01-07")) + cls = src.fetch(Series("BTC", "EUR", "close", "2021-01-01", "2021-01-07")) + assert mid.prices[0].amount == Decimal("23881.35") + assert opn.prices[0].amount == Decimal("23706.73") + assert hgh.prices[0].amount == Decimal("24250") + assert low.prices[0].amount == Decimal("23512.7") + assert cls.prices[0].amount == Decimal("24070.97") + + +def test_fetch_type_mid_is_mean_of_low_and_high(src, recent_response_ok): + mid = src.fetch(Series("BTC", "EUR", "mid", "2021-01-01", "2021-01-07")).prices + low = src.fetch(Series("BTC", "EUR", "low", "2021-01-01", "2021-01-07")).prices + hgh = src.fetch(Series("BTC", "EUR", "high", "2021-01-01", "2021-01-07")).prices + assert all( + [ + mid[i].amount == (sum([low[i].amount, hgh[i].amount]) / 2) + for i in range(0, 7) + ] + ) + + +def test_fetch_requests_logged(src, type, recent_response_ok, caplog): + with caplog.at_level(logging.DEBUG): + src.fetch(Series("BTC", "EUR", type, "2021-01-01", "2021-01-07")) + assert any( + ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] + ) + + +def test_fetch_long_hist_multi_segment(src, type, multi_response_ok): + series = src.fetch(Series("BTC", "EUR", type, "2020-01-01", "2021-01-07")) + assert series.prices[0] == Price("2020-01-01", Decimal("6430.175")) + assert series.prices[-1] == Price("2021-01-07", Decimal("31208.49")) + assert len(series.prices) > 3 + + +def test_fetch_from_before_start(src, type, requests_mock): + body = '{"message":"End is too old"}' + requests_mock.add(responses.GET, product_url("BTC", "EUR"), status=400, body=body) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("BTC", "EUR", type, "1960-01-01", "1960-01-07")) + assert "too early" in str(e.value) + + +def test_fetch_in_future(src, type, response_empty): + series = src.fetch(Series("BTC", "EUR", type, "2100-01-01", "2100-01-07")) + assert len(series.prices) == 0 + + +def test_fetch_wrong_dates_order_alledged(src, type, requests_mock): + # Is actually prevented in argument parsing and inside the source. + body = '{"message":"start must be before end"}' + requests_mock.add(responses.GET, product_url("BTC", "EUR"), status=400, body=body) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("BTC", "EUR", type, "2021-01-07", "2021-01-01")) + assert "end can't preceed" in str(e.value) + + +def test_fetch_too_many_data_points_alledged(src, type, requests_mock): + # Should only happen if limit is reduced or calculated segments lengthened + body = "aggregations requested exceeds" + requests_mock.add(responses.GET, product_url("BTC", "EUR"), status=400, body=body) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("BTC", "EUR", type, "2021-01-07", "2021-01-01")) + assert "Too many data points" in str(e.value) + + +def test_fetch_rate_limit(src, type, requests_mock): + body = "Too many requests" + requests_mock.add(responses.GET, product_url("BTC", "EUR"), status=429, body=body) + with pytest.raises(exceptions.RateLimit) as e: + src.fetch(Series("BTC", "EUR", type, "2021-01-07", "2021-01-01")) + assert "rate limit has been exceeded" in str(e.value) + + +def test_fetch_empty(src, type, response_empty): + series = src.fetch(Series("BTC", "EUR", type, "2000-01-01", "2000-01-07")) + assert len(series.prices) == 0 + + +def test_fetch_unknown_base(src, type, requests_mock): + body = '{"message":"NotFound"}' + requests_mock.add( + responses.GET, product_url("UNKNOWN", "EUR"), status=404, body=body + ) + with pytest.raises(exceptions.InvalidPair): + src.fetch(Series("UNKNOWN", "EUR", type, "2021-01-01", "2021-01-07")) + + +def test_fetch_unknown_quote(src, type, requests_mock): + body = '{"message":"NotFound"}' + requests_mock.add(responses.GET, product_url("BTC", "XZY"), status=404, body=body) + with pytest.raises(exceptions.InvalidPair): + src.fetch(Series("BTC", "XZY", type, "2021-01-01", "2021-01-07")) + + +def test_fetch_no_quote(src, type, requests_mock): + body = '{"message":"NotFound"}' + requests_mock.add(responses.GET, product_url("BTC", ""), status=404, body=body) + with pytest.raises(exceptions.InvalidPair): + src.fetch(Series("BTC", "", type, "2021-01-01", "2021-01-07")) + + +def test_fetch_unknown_pair(src, type, requests_mock): + body = '{"message":"NotFound"}' + requests_mock.add(responses.GET, product_url("ABC", "XZY"), status=404, body=body) + with pytest.raises(exceptions.InvalidPair): + src.fetch(Series("ABC", "XZY", type, "2021-01-01", "2021-01-07")) + + +def test_fetch_network_issue(src, type, requests_mock): + body = requests.exceptions.ConnectionError("Network issue") + requests_mock.add(responses.GET, product_url("BTC", "EUR"), body=body) + with pytest.raises(exceptions.RequestError) as e: + src.fetch(Series("BTC", "EUR", type, "2021-01-01", "2021-01-07")) + assert "Network issue" in str(e.value) + + +def test_fetch_bad_status(src, type, requests_mock): + requests_mock.add( + responses.GET, product_url("BTC", "EUR"), status=500, body="Some other reason" + ) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("BTC", "EUR", type, "2021-01-01", "2021-01-07")) + assert "Internal Server Error" in str(e.value) + + +def test_fetch_parsing_error(src, type, requests_mock): + requests_mock.add(responses.GET, product_url("BTC", "EUR"), body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("BTC", "EUR", type, "2021-01-01", "2021-01-07")) + assert "while parsing data" in str(e.value) diff --git a/tests/pricehist/sources/test_coinbasepro/2020-01-01--2020-10-16.json b/tests/pricehist/sources/test_coinbasepro/2020-01-01--2020-10-16.json new file mode 100644 index 0000000..0d6a2d4 --- /dev/null +++ b/tests/pricehist/sources/test_coinbasepro/2020-01-01--2020-10-16.json @@ -0,0 +1,18 @@ +[ + [ + 1602806400, + 9588, + 9860, + 9828.84, + 9672.41, + 1068.08144123 + ], + [ + 1577836800, + 6388.91, + 6471.44, + 6400.02, + 6410.22, + 491.94797816 + ] +] diff --git a/tests/pricehist/sources/test_coinbasepro/2020-10-17--2021-01-07.json b/tests/pricehist/sources/test_coinbasepro/2020-10-17--2021-01-07.json new file mode 100644 index 0000000..6641080 --- /dev/null +++ b/tests/pricehist/sources/test_coinbasepro/2020-10-17--2021-01-07.json @@ -0,0 +1,18 @@ +[ + [ + 1609977600, + 29516.98, + 32900, + 29818.73, + 32120.19, + 5957.46980324 + ], + [ + 1602892800, + 9630.1, + 9742.61, + 9675.29, + 9706.33, + 385.03505036 + ] +] diff --git a/tests/pricehist/sources/test_coinbasepro/currencies-partial.json b/tests/pricehist/sources/test_coinbasepro/currencies-partial.json new file mode 100644 index 0000000..cb73b65 --- /dev/null +++ b/tests/pricehist/sources/test_coinbasepro/currencies-partial.json @@ -0,0 +1,141 @@ +[ + { + "id": "BTC", + "name": "Bitcoin", + "min_size": "0.00000001", + "status": "online", + "message": "", + "max_precision": "0.00000001", + "convertible_to": [], + "details": { + "type": "crypto", + "symbol": "₿", + "network_confirmations": 3, + "sort_order": 20, + "crypto_address_link": "https://live.blockcypher.com/btc/address/{{address}}", + "crypto_transaction_link": "https://live.blockcypher.com/btc/tx/{{txId}}", + "push_payment_methods": [ + "crypto" + ], + "group_types": [ + "btc", + "crypto" + ], + "display_name": "", + "processing_time_seconds": 0, + "min_withdrawal_amount": 0.0001, + "max_withdrawal_amount": 2400 + } + }, + { + "id": "DOGE", + "name": "Dogecoin", + "min_size": "1", + "status": "online", + "message": "", + "max_precision": "0.1", + "convertible_to": [], + "details": { + "type": "crypto", + "symbol": "", + "network_confirmations": 60, + "sort_order": 29, + "crypto_address_link": "https://dogechain.info/address/{{address}}", + "crypto_transaction_link": "", + "push_payment_methods": [ + "crypto" + ], + "group_types": [], + "display_name": "", + "processing_time_seconds": 0, + "min_withdrawal_amount": 1, + "max_withdrawal_amount": 17391300 + } + }, + { + "id": "ETH", + "name": "Ether", + "min_size": "0.00000001", + "status": "online", + "message": "", + "max_precision": "0.00000001", + "convertible_to": [], + "details": { + "type": "crypto", + "symbol": "Ξ", + "network_confirmations": 35, + "sort_order": 25, + "crypto_address_link": "https://etherscan.io/address/{{address}}", + "crypto_transaction_link": "https://etherscan.io/tx/0x{{txId}}", + "push_payment_methods": [ + "crypto" + ], + "group_types": [ + "eth", + "crypto" + ], + "display_name": "", + "processing_time_seconds": 0, + "min_withdrawal_amount": 0.001, + "max_withdrawal_amount": 7450 + } + }, + { + "id": "EUR", + "name": "Euro", + "min_size": "0.01", + "status": "online", + "message": "", + "max_precision": "0.01", + "convertible_to": [], + "details": { + "type": "fiat", + "symbol": "€", + "network_confirmations": 0, + "sort_order": 2, + "crypto_address_link": "", + "crypto_transaction_link": "", + "push_payment_methods": [ + "sepa_bank_account" + ], + "group_types": [ + "fiat", + "eur" + ], + "display_name": "", + "processing_time_seconds": 0, + "min_withdrawal_amount": 0, + "max_withdrawal_amount": 0 + } + }, + { + "id": "GBP", + "name": "British Pound", + "min_size": "0.01", + "status": "online", + "message": "", + "max_precision": "0.01", + "convertible_to": [], + "details": { + "type": "fiat", + "symbol": "£", + "network_confirmations": 0, + "sort_order": 3, + "crypto_address_link": "", + "crypto_transaction_link": "", + "push_payment_methods": [ + "uk_bank_account", + "swift_lhv", + "swift" + ], + "group_types": [ + "fiat", + "gbp" + ], + "display_name": "", + "processing_time_seconds": 0, + "min_withdrawal_amount": 0, + "max_withdrawal_amount": 0 + } + } +] diff --git a/tests/pricehist/sources/test_coinbasepro/products-partial.json b/tests/pricehist/sources/test_coinbasepro/products-partial.json new file mode 100644 index 0000000..b241803 --- /dev/null +++ b/tests/pricehist/sources/test_coinbasepro/products-partial.json @@ -0,0 +1,62 @@ +[ + { + "id": "BTC-EUR", + "base_currency": "BTC", + "quote_currency": "EUR", + "base_min_size": "0.0001", + "base_max_size": "200", + "quote_increment": "0.01", + "base_increment": "0.00000001", + "display_name": "BTC/EUR", + "min_market_funds": "10", + "max_market_funds": "600000", + "margin_enabled": false, + "fx_stablecoin": false, + "post_only": false, + "limit_only": false, + "cancel_only": false, + "trading_disabled": false, + "status": "online", + "status_message": "" + }, + { + "id": "ETH-GBP", + "base_currency": "ETH", + "quote_currency": "GBP", + "base_min_size": "0.001", + "base_max_size": "1400", + "quote_increment": "0.01", + "base_increment": "0.00000001", + "display_name": "ETH/GBP", + "min_market_funds": "10", + "max_market_funds": "1000000", + "margin_enabled": false, + "fx_stablecoin": false, + "post_only": false, + "limit_only": false, + "cancel_only": false, + "trading_disabled": false, + "status": "online", + "status_message": "" + }, + { + "id": "DOGE-EUR", + "base_currency": "DOGE", + "quote_currency": "EUR", + "base_min_size": "1", + "base_max_size": "690000", + "quote_increment": "0.0001", + "base_increment": "0.1", + "display_name": "DOGE/EUR", + "min_market_funds": "5.0", + "max_market_funds": "100000", + "margin_enabled": false, + "fx_stablecoin": false, + "post_only": false, + "limit_only": false, + "cancel_only": false, + "trading_disabled": false, + "status": "online", + "status_message": "" + } +] diff --git a/tests/pricehist/sources/test_coinbasepro/recent.json b/tests/pricehist/sources/test_coinbasepro/recent.json new file mode 100644 index 0000000..fab4821 --- /dev/null +++ b/tests/pricehist/sources/test_coinbasepro/recent.json @@ -0,0 +1,58 @@ +[ + [ + 1609977600, + 29516.98, + 32900, + 29818.73, + 32120.19, + 5957.46980324 + ], + [ + 1609891200, + 27105.01, + 29949, + 27655.04, + 29838.52, + 4227.05067035 + ], + [ + 1609804800, + 24413.62, + 27989, + 26104.4, + 27654.01, + 4036.27720179 + ], + [ + 1609718400, + 22055, + 26199, + 25624.7, + 26115.94, + 6304.41029978 + ], + [ + 1609632000, + 24500, + 27195.46, + 25916.75, + 25644.41, + 4975.13927959 + ], + [ + 1609545600, + 22000, + 27000, + 24071.26, + 25907.35, + 7291.88538639 + ], + [ + 1609459200, + 23512.7, + 24250, + 23706.73, + 24070.97, + 1830.04655405 + ] +] From 1468e1f64bfc833dd7f0d032ac967d88a58c3843 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 23 Aug 2021 18:41:34 +0200 Subject: [PATCH 060/149] Version 1.2.0. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a483d8e..612d0e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.1.0" +version = "1.2.0" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 6849410..c68196d 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.1.0" +__version__ = "1.2.0" From 89e8bc9964b85434edd4e5ed38802f10604cf61e Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 23 Aug 2021 20:59:45 +0200 Subject: [PATCH 061/149] Loosen requirement for Alpha Vantage api key. --- src/pricehist/sources/alphavantage.py | 42 ++++++++++++-------- tests/pricehist/sources/test_alphavantage.py | 18 ++++++++- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index a74eeda..07aaae2 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -8,7 +8,7 @@ from decimal import Decimal import requests -from pricehist import exceptions +from pricehist import __version__, exceptions from pricehist.price import Price from .basesource import BaseSource @@ -16,6 +16,7 @@ from .basesource import BaseSource class AlphaVantage(BaseSource): QUERY_URL = "https://www.alphavantage.co/query" + API_KEY_NAME = "ALPHAVANTAGE_API_KEY" def id(self): return "alphavantage" @@ -36,14 +37,14 @@ class AlphaVantage(BaseSource): return ["close", "open", "high", "low", "adjclose", "mid"] def notes(self): - keystatus = "already set" if self._apikey(require=False) else "NOT YET set" + keystatus = "already set" if self._apikey(require=False) else "not yet set" return ( "Alpha Vantage has data on digital (crypto) currencies, physical " "(fiat) currencies and stocks.\n" - "An API key is required. One can be obtained for free from " - "https://www.alphavantage.co/support/#api-key and should be made " - "available in the ALPHAVANTAGE_API_KEY environment variable " - f"({keystatus}).\n" + "You should obtain a free API key from " + "https://www.alphavantage.co/support/#api-key and set it in " + f"the {self.API_KEY_NAME} environment variable ({keystatus}), " + "otherise, pricehist will attempt to use a generic key.\n" "The PAIR for currencies should be in BASE/QUOTE form. The quote " "symbol must always be for a physical currency. The --symbols option " "will list all digital and physical currency symbols.\n" @@ -165,8 +166,7 @@ class AlphaVantage(BaseSource): except Exception as e: raise exceptions.ResponseParsingError(str(e)) from e - if type(data) == dict and "Note" in data and "call frequency" in data["Note"]: - raise exceptions.RateLimit(data["Note"]) + self._raise_for_generic_errors(data) expected_keys = ["1. symbol", "2. name", "3. type", "4. region", "8. currency"] if ( @@ -204,8 +204,7 @@ class AlphaVantage(BaseSource): except Exception as e: raise exceptions.ResponseParsingError(str(e)) from e - if type(data) == dict and "Note" in data and "call frequency" in data["Note"]: - raise exceptions.RateLimit(data["Note"]) + self._raise_for_generic_errors(data) if "Error Message" in data: if output_quote == "UNKNOWN": @@ -255,8 +254,7 @@ class AlphaVantage(BaseSource): except Exception as e: raise exceptions.ResponseParsingError(str(e)) from e - if type(data) == dict and "Note" in data and "call frequency" in data["Note"]: - raise exceptions.RateLimit(data["Note"]) + self._raise_for_generic_errors(data) if type(data) != dict or "Time Series FX (Daily)" not in data: raise exceptions.ResponseParsingError("Unexpected content.") @@ -297,8 +295,7 @@ class AlphaVantage(BaseSource): except Exception as e: raise exceptions.ResponseParsingError(str(e)) from e - if type(data) == dict and "Note" in data and "call frequency" in data["Note"]: - raise exceptions.RateLimit(data["Note"]) + self._raise_for_generic_errors(data) if type(data) != dict or "Time Series (Digital Currency Daily)" not in data: raise exceptions.ResponseParsingError("Unexpected content.") @@ -317,12 +314,23 @@ class AlphaVantage(BaseSource): return normalized_data def _apikey(self, require=True): - key_name = "ALPHAVANTAGE_API_KEY" - key = os.getenv(key_name) + key = os.getenv(self.API_KEY_NAME) if require and not key: - raise exceptions.CredentialsError([key_name], self) + generic_key = f"pricehist_{__version__}" + logging.debug( + f"{self.API_KEY_NAME} not set. " + f"Defaulting to generic key '{generic_key}'." + ) + return generic_key return key + def _raise_for_generic_errors(self, data): + if type(data) == dict: + if "Note" in data and "call frequency" in data["Note"]: + raise exceptions.RateLimit(data["Note"]) + if "Error Message" in data and "apikey " in data["Error Message"]: + raise exceptions.CredentialsError([self.API_KEY_NAME], self) + def _physical_symbols(self) -> list[(str, str)]: url = "https://www.alphavantage.co/physical_currency_list/" return self._get_symbols(url, "Physical: ") diff --git a/tests/pricehist/sources/test_alphavantage.py b/tests/pricehist/sources/test_alphavantage.py index 237f8e9..334bbd4 100644 --- a/tests/pricehist/sources/test_alphavantage.py +++ b/tests/pricehist/sources/test_alphavantage.py @@ -9,7 +9,7 @@ import pytest import requests import responses -from pricehist import exceptions +from pricehist import __version__, exceptions from pricehist.price import Price from pricehist.series import Series from pricehist.sources.alphavantage import AlphaVantage @@ -625,8 +625,22 @@ def test_fetch_bad_pair_quote_non_physical(src, type, physical_list_ok): assert "quote must be a physical currency" in str(e.value) -def test_fetch_api_key_missing(src, type, physical_list_ok, monkeypatch): +def test_fetch_api_key_defaults_to_generic( + src, type, physical_list_ok, euraud_ok, monkeypatch +): monkeypatch.delenv(api_key_name) + src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) + req = euraud_ok.calls[-1].request + assert req.params["apikey"] == f"pricehist_{__version__}" + + +def test_fetch_api_key_invalid(src, type, physical_list_ok, requests_mock): + body = ( + '{ "Error Message": "the parameter apikey is invalid or missing. Please ' + "claim your free API key on (https://www.alphavantage.co/support/#api-key). " + 'It should take less than 20 seconds." }' + ) + requests_mock.add(responses.GET, physical_url, body=body) with pytest.raises(exceptions.CredentialsError) as e: src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) assert "unavailable or invalid" in str(e.value) From 65f88361534d75d00012fe5d4fce1a69771ba58d Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 23 Aug 2021 21:00:23 +0200 Subject: [PATCH 062/149] Version 1.2.1. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 612d0e1..e5024f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.2.0" +version = "1.2.1" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index c68196d..a955fda 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.2.0" +__version__ = "1.2.1" From 216ab193851eb9769381f5851754d5e20e357fb0 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 24 Aug 2021 13:49:35 +0200 Subject: [PATCH 063/149] For gnucash-sql, show the summary at the end so it doesn't scroll off screen. --- src/pricehist/resources/gnucash.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pricehist/resources/gnucash.sql b/src/pricehist/resources/gnucash.sql index e22671a..aa2a39d 100644 --- a/src/pricehist/resources/gnucash.sql +++ b/src/pricehist/resources/gnucash.sql @@ -35,10 +35,10 @@ WHERE tp.base = g1.mnemonic AND tp.guid NOT IN (SELECT guid FROM prices) ; --- Show the summary. -SELECT * FROM summary; - -- Show the final relevant rows of the main prices table SELECT 'final' AS status, p.* FROM prices p WHERE p.guid IN (SELECT guid FROM new_prices) ORDER BY p.date; +-- Show the summary. +SELECT * FROM summary; + COMMIT; From 7becc4c0c566812d558995101906f827f7b9d851 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 24 Aug 2021 13:50:10 +0200 Subject: [PATCH 064/149] Version 1.2.2. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e5024f3..15d7f88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.2.1" +version = "1.2.2" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index a955fda..bc86c94 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.2.1" +__version__ = "1.2.2" From 7325ff61872b446e2dbfa9e70f5e03baed84c3a4 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 24 Aug 2021 16:52:38 +0200 Subject: [PATCH 065/149] Update gnucash-sql section of README. --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f339b38..be051e1 100644 --- a/README.md +++ b/README.md @@ -153,10 +153,18 @@ pricehist fetch ecb EUR/AUD -s 2021-01-01 -o gnucash-sql | psql -U username -d d Beware that the GnuCash project itself does not support integration at the database level, so there is a risk that the SQL generated by `pricehist` will -be ineffective or even damaging for some version of GnuCash. +be ineffective or even damaging for some version of GnuCash. In practice, this +strategy has been used successfully by other projects. Reading the SQL and +keeping regular database backups is recommended. -In practice, this strategy has been used successfully by other projects. -Reading the SQL and keeping regular database backups is recommended. +The GnuCash database must already contain commodities with mnemonics matching +the base and quote of new prices, otherwise the SQL will fail without making +changes. + +Each price entry is given a GUID based on its content (date, base, quote, +source, type and amount) and existing GUIDs are skipped in the final insert, so +you can apply identical or overlapping SQL files multiple times without +creating duplicate entries in the database. ### Show source information From c012af38816409dea5b6365f9ed07633f0bfadab Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 24 Aug 2021 17:26:47 +0200 Subject: [PATCH 066/149] Note about alphavantage not using historical rates for converting crypto quotes from USD. --- src/pricehist/sources/alphavantage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index 07aaae2..d0eac8c 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -51,6 +51,9 @@ class AlphaVantage(BaseSource): "The PAIR for stocks is the stock symbol only. The quote currency " f"will be determined automatically. {self._stock_symbols_message()}\n" "The price type 'adjclose' is only available for stocks.\n" + "Beware that digital currencies quoted in non-USD currencies may " + "be converted from USD data at one recent exchange rate rather " + "than using historical rates.\n" "Alpha Vantage's standard API call frequency limits is 5 calls per " "minute and 500 per day, so you may need to pause between successive " "commands. Note that retrieving prices for one stock requires two " From b9bd3d694d91c032011e462e4c6a0d32e7981274 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 24 Aug 2021 17:43:54 +0200 Subject: [PATCH 067/149] Fix make lint to avoid the dist directory. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1a26a11..a37b466 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ format: ## Format source code .PHONY: lint lint: ## Lint source code - poetry run flake8 + poetry run flake8 src tests .PHONY: test test: ## Run tests From b2a5b4c5c95597834605da5d149a98136ecc5f51 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 24 Aug 2021 17:44:10 +0200 Subject: [PATCH 068/149] Add a how to contribute section to the README. --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index be051e1..6db9850 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,20 @@ Type "help", "copyright", "credits" or "license" for more information. A subclass of `pricehist.exceptions.SourceError` will be raised for any error. +### Contribute + +Contributions are welcome! If you discover a bug or want to work on a +non-trivial change, please open a +[GitLab issue](https://gitlab.com/chrisberkhout/pricehist/-/issues) +to discuss it. + +Run `make install-pre-commit-hook` set up local pre-commit checks. +Set up your editor to run +[isort](https://pycqa.github.io/isort/), +[Black](https://black.readthedocs.io/en/stable/) and +[Flake8](https://flake8.pycqa.org/en/latest/), +or run them manually via `make format lint`. + ## Terminology A **source** is an upstream service that can provide a series of prices. From 38beaef3be2e7bc5484872a0dc6971507b32b35d Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 24 Aug 2021 17:45:45 +0200 Subject: [PATCH 069/149] Version 1.2.3. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 15d7f88..eb12cfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.2.2" +version = "1.2.3" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index bc86c94..10aa336 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.2.2" +__version__ = "1.2.3" From 77a77e76c89301d7967cd699dd11c02e2766e2ee Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 14 Sep 2021 09:07:11 +0200 Subject: [PATCH 070/149] More compatible type hints. --- src/pricehist/isocurrencies.py | 3 ++- src/pricehist/series.py | 3 ++- src/pricehist/sources/alphavantage.py | 7 ++++--- src/pricehist/sources/basesource.py | 7 ++++--- tests/pricehist/sources/test_basesource.py | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/pricehist/isocurrencies.py b/src/pricehist/isocurrencies.py index f7079c4..fe02678 100644 --- a/src/pricehist/isocurrencies.py +++ b/src/pricehist/isocurrencies.py @@ -25,6 +25,7 @@ Functions: from dataclasses import dataclass, field from importlib.resources import read_binary +from typing import List from lxml import etree @@ -36,7 +37,7 @@ class ISOCurrency: minor_units: int = None name: str = None is_fund: bool = False - countries: list[str] = field(default_factory=list) + countries: List[str] = field(default_factory=list) historical: bool = False withdrawal_date: str = None diff --git a/src/pricehist/series.py b/src/pricehist/series.py index d063b90..88a1de3 100644 --- a/src/pricehist/series.py +++ b/src/pricehist/series.py @@ -1,5 +1,6 @@ from dataclasses import dataclass, field, replace from decimal import Decimal, getcontext +from typing import List from pricehist.price import Price @@ -11,7 +12,7 @@ class Series: type: str start: str end: str - prices: list[Price] = field(default_factory=list) + prices: List[Price] = field(default_factory=list) def invert(self): return replace( diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index d0eac8c..75cda5d 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -5,6 +5,7 @@ import logging import os from datetime import datetime, timedelta from decimal import Decimal +from typing import List, Tuple import requests @@ -334,15 +335,15 @@ class AlphaVantage(BaseSource): if "Error Message" in data and "apikey " in data["Error Message"]: raise exceptions.CredentialsError([self.API_KEY_NAME], self) - def _physical_symbols(self) -> list[(str, str)]: + def _physical_symbols(self) -> List[Tuple[str, str]]: url = "https://www.alphavantage.co/physical_currency_list/" return self._get_symbols(url, "Physical: ") - def _digital_symbols(self) -> list[(str, str)]: + def _digital_symbols(self) -> List[Tuple[str, str]]: url = "https://www.alphavantage.co/digital_currency_list/" return self._get_symbols(url, "Digital: ") - def _get_symbols(self, url, prefix) -> list[(str, str)]: + def _get_symbols(self, url, prefix) -> List[Tuple[str, str]]: try: response = self.log_curl(requests.get(url)) except Exception as e: diff --git a/src/pricehist/sources/basesource.py b/src/pricehist/sources/basesource.py index 4273678..cab423f 100644 --- a/src/pricehist/sources/basesource.py +++ b/src/pricehist/sources/basesource.py @@ -1,6 +1,7 @@ import logging from abc import ABC, abstractmethod from textwrap import TextWrapper +from typing import List, Tuple import curlify @@ -30,7 +31,7 @@ class BaseSource(ABC): pass # pragma: nocover @abstractmethod - def types(self) -> list[str]: + def types(self) -> List[str]: pass # pragma: nocover @abstractmethod @@ -41,10 +42,10 @@ class BaseSource(ABC): return str.upper() @abstractmethod - def symbols(self) -> list[(str, str)]: + def symbols(self) -> List[Tuple[str, str]]: pass # pragma: nocover - def search(self, query) -> list[(str, str)]: + def search(self, query) -> List[Tuple[str, str]]: pass # pragma: nocover @abstractmethod diff --git a/tests/pricehist/sources/test_basesource.py b/tests/pricehist/sources/test_basesource.py index cfd5e80..478d5c2 100644 --- a/tests/pricehist/sources/test_basesource.py +++ b/tests/pricehist/sources/test_basesource.py @@ -1,4 +1,5 @@ import logging +from typing import List, Tuple import pytest @@ -22,13 +23,13 @@ class TestSource(BaseSource): def start(self) -> str: return "" - def types(self) -> list[str]: + def types(self) -> List[str]: return [] def notes(self) -> str: return "" - def symbols(self) -> list[(str, str)]: + def symbols(self) -> List[Tuple[str, str]]: return [] def fetch(self, series: Series) -> Series: From 336b2c3461ba9d856169cf08e9f6463eaf77bc80 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 5 Oct 2021 12:50:18 +0200 Subject: [PATCH 071/149] Update ISO 4217 data. --- src/pricehist/isocurrencies.py | 5 ++++- src/pricehist/resources/list_one.xml | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/pricehist/isocurrencies.py b/src/pricehist/isocurrencies.py index fe02678..550db3a 100644 --- a/src/pricehist/isocurrencies.py +++ b/src/pricehist/isocurrencies.py @@ -24,6 +24,7 @@ Functions: """ from dataclasses import dataclass, field +from datetime import datetime from importlib.resources import read_binary from typing import List @@ -44,7 +45,9 @@ class ISOCurrency: def current_data_date(): one = etree.fromstring(read_binary("pricehist.resources", "list_one.xml")) - return one.cssselect("ISO_4217")[0].attrib["Pblshd"] + pblshd = one.cssselect("ISO_4217")[0].attrib["Pblshd"] + date = datetime.strptime(pblshd, "%B %d, %Y").date().isoformat() + return date def historical_data_date(): diff --git a/src/pricehist/resources/list_one.xml b/src/pricehist/resources/list_one.xml index 20be53b..7bc46c2 100644 --- a/src/pricehist/resources/list_one.xml +++ b/src/pricehist/resources/list_one.xml @@ -1,5 +1,5 @@ - + AFGHANISTAN @@ -1819,6 +1819,13 @@ 928 2 + + VENEZUELA (BOLIVARIAN REPUBLIC OF) + Bolívar Soberano + VED + 926 + 2 + VIET NAM Dong From 2249917494df3bbfbd6f244a6fa0226cdf2c13a2 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 5 Oct 2021 13:28:48 +0200 Subject: [PATCH 072/149] Allow installation on Python 3.8. --- poetry.lock | 317 ++++++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 2 files changed, 169 insertions(+), 150 deletions(-) diff --git a/poetry.lock b/poetry.lock index f5f3ffb..1de5b1a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -16,17 +16,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "20.3.0" +version = "21.2.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] -docs = ["furo", "sphinx", "zope.interface"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] [[package]] name = "black" @@ -52,27 +52,33 @@ d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] name = "certifi" -version = "2020.12.5" +version = "2021.5.30" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = "*" [[package]] -name = "chardet" -version = "4.0.0" -description = "Universal encoding detector for Python 2 and 3" +name = "charset-normalizer" +version = "2.0.6" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] [[package]] name = "click" -version = "7.1.2" +version = "8.0.1" description = "Composable command line interface toolkit" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" @@ -114,7 +120,7 @@ requests = "*" [[package]] name = "flake8" -version = "3.9.1" +version = "3.9.2" description = "the modular source code checker: pep8 pyflakes and co" category = "dev" optional = false @@ -127,11 +133,11 @@ pyflakes = ">=2.3.0,<2.4.0" [[package]] name = "idna" -version = "2.10" +version = "3.2" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.5" [[package]] name = "iniconfig" @@ -143,20 +149,21 @@ python-versions = "*" [[package]] name = "isort" -version = "5.8.0" +version = "5.9.3" description = "A Python utility / library to sort Python imports." category = "dev" optional = false -python-versions = ">=3.6,<4.0" +python-versions = ">=3.6.1,<4.0" [package.extras] pipfile_deprecated_finder = ["pipreqs", "requirementslib"] requirements_deprecated_finder = ["pipreqs", "pip-api"] colors = ["colorama (>=0.4.3,<0.5.0)"] +plugins = ["setuptools"] [[package]] name = "lxml" -version = "4.6.2" +version = "4.6.3" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." category = "main" optional = false @@ -186,33 +193,34 @@ python-versions = "*" [[package]] name = "packaging" -version = "20.9" +version = "21.0" description = "Core utilities for Python packages" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.6" [package.dependencies] pyparsing = ">=2.0.2" [[package]] name = "pathspec" -version = "0.8.1" +version = "0.9.0" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [[package]] name = "pluggy" -version = "0.13.1" +version = "1.0.0" description = "plugin and hook calling mechanisms for python" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.6" [package.extras] dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "py" @@ -248,7 +256,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "pytest" -version = "6.2.2" +version = "6.2.5" description = "pytest: simple powerful testing with Python" category = "dev" optional = false @@ -260,7 +268,7 @@ attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0.0a1" +pluggy = ">=0.12,<2.0" py = ">=1.8.2" toml = "*" @@ -283,7 +291,7 @@ dev = ["pre-commit", "tox", "pytest-asyncio"] [[package]] name = "regex" -version = "2021.4.4" +version = "2021.9.30" description = "Alternative regular expression module, to replace re." category = "dev" optional = false @@ -291,25 +299,25 @@ python-versions = "*" [[package]] name = "requests" -version = "2.25.1" +version = "2.26.0" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] certifi = ">=2017.4.17" -chardet = ">=3.0.2,<5" -idna = ">=2.5,<3" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} urllib3 = ">=1.21.1,<1.27" [package.extras] -security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] [[package]] name = "responses" -version = "0.13.3" +version = "0.13.4" description = "A utility library for mocking out the `requests` Python library." category = "dev" optional = false @@ -321,7 +329,7 @@ six = "*" urllib3 = ">=1.25.10" [package.extras] -tests = ["coverage (>=3.7.1,<6.0.0)", "pytest-cov", "pytest-localserver", "flake8", "pytest (>=4.6,<5.0)", "pytest (>=4.6)", "mypy"] +tests = ["coverage (>=3.7.1,<6.0.0)", "pytest-cov", "pytest-localserver", "flake8", "types-mock", "types-requests", "types-six", "pytest (>=4.6,<5.0)", "pytest (>=4.6)", "mypy"] [[package]] name = "six" @@ -349,7 +357,7 @@ python-versions = "*" [[package]] name = "typing-extensions" -version = "3.7.4.3" +version = "3.10.0.2" description = "Backported and Experimental Type Hints for Python 3.5+" category = "dev" optional = false @@ -357,7 +365,7 @@ python-versions = "*" [[package]] name = "urllib3" -version = "1.26.3" +version = "1.26.7" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -370,8 +378,8 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" -python-versions = "^3.9" -content-hash = "4434fa4fdfb1a7f4d9b833dd611330228497cdd37c0502de9d83f64752dd6480" +python-versions = "^3.8" +content-hash = "252531625354ed4b7e8f14839d769020ebcce296da72fcd27c1eb1290dd9ca63" [metadata.files] appdirs = [ @@ -383,23 +391,23 @@ atomicwrites = [ {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, - {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, + {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, + {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, ] black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, ] certifi = [ - {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"}, - {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, + {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, + {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, ] -chardet = [ - {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, - {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, +charset-normalizer = [ + {file = "charset-normalizer-2.0.6.tar.gz", hash = "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f"}, + {file = "charset_normalizer-2.0.6-py3-none-any.whl", hash = "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6"}, ] click = [ - {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, - {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, + {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, + {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, @@ -467,59 +475,70 @@ curlify = [ {file = "curlify-2.2.1.tar.gz", hash = "sha256:0d3f02e7235faf952de8ef45ef469845196d30632d5838bcd5aee217726ddd6d"}, ] flake8 = [ - {file = "flake8-3.9.1-py2.py3-none-any.whl", hash = "sha256:3b9f848952dddccf635be78098ca75010f073bfe14d2c6bda867154bea728d2a"}, - {file = "flake8-3.9.1.tar.gz", hash = "sha256:1aa8990be1e689d96c745c5682b687ea49f2e05a443aff1f8251092b0014e378"}, + {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, + {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] idna = [ - {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, - {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, + {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, + {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] isort = [ - {file = "isort-5.8.0-py3-none-any.whl", hash = "sha256:2bb1680aad211e3c9944dbce1d4ba09a989f04e238296c87fe2139faa26d655d"}, - {file = "isort-5.8.0.tar.gz", hash = "sha256:0a943902919f65c5684ac4e0154b1ad4fac6dcaa5d9f3426b732f1c8b5419be6"}, + {file = "isort-5.9.3-py3-none-any.whl", hash = "sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2"}, + {file = "isort-5.9.3.tar.gz", hash = "sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899"}, ] lxml = [ - {file = "lxml-4.6.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a9d6bc8642e2c67db33f1247a77c53476f3a166e09067c0474facb045756087f"}, - {file = "lxml-4.6.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:791394449e98243839fa822a637177dd42a95f4883ad3dec2a0ce6ac99fb0a9d"}, - {file = "lxml-4.6.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:68a5d77e440df94011214b7db907ec8f19e439507a70c958f750c18d88f995d2"}, - {file = "lxml-4.6.2-cp27-cp27m-win32.whl", hash = "sha256:fc37870d6716b137e80d19241d0e2cff7a7643b925dfa49b4c8ebd1295eb506e"}, - {file = "lxml-4.6.2-cp27-cp27m-win_amd64.whl", hash = "sha256:69a63f83e88138ab7642d8f61418cf3180a4d8cd13995df87725cb8b893e950e"}, - {file = "lxml-4.6.2-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:42ebca24ba2a21065fb546f3e6bd0c58c3fe9ac298f3a320147029a4850f51a2"}, - {file = "lxml-4.6.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:f83d281bb2a6217cd806f4cf0ddded436790e66f393e124dfe9731f6b3fb9afe"}, - {file = "lxml-4.6.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:535f067002b0fd1a4e5296a8f1bf88193080ff992a195e66964ef2a6cfec5388"}, - {file = "lxml-4.6.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:366cb750140f221523fa062d641393092813b81e15d0e25d9f7c6025f910ee80"}, - {file = "lxml-4.6.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:97db258793d193c7b62d4e2586c6ed98d51086e93f9a3af2b2034af01450a74b"}, - {file = "lxml-4.6.2-cp35-cp35m-win32.whl", hash = "sha256:648914abafe67f11be7d93c1a546068f8eff3c5fa938e1f94509e4a5d682b2d8"}, - {file = "lxml-4.6.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4e751e77006da34643ab782e4a5cc21ea7b755551db202bc4d3a423b307db780"}, - {file = "lxml-4.6.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:681d75e1a38a69f1e64ab82fe4b1ed3fd758717bed735fb9aeaa124143f051af"}, - {file = "lxml-4.6.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:127f76864468d6630e1b453d3ffbbd04b024c674f55cf0a30dc2595137892d37"}, - {file = "lxml-4.6.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4fb85c447e288df535b17ebdebf0ec1cf3a3f1a8eba7e79169f4f37af43c6b98"}, - {file = "lxml-4.6.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:5be4a2e212bb6aa045e37f7d48e3e1e4b6fd259882ed5a00786f82e8c37ce77d"}, - {file = "lxml-4.6.2-cp36-cp36m-win32.whl", hash = "sha256:8c88b599e226994ad4db29d93bc149aa1aff3dc3a4355dd5757569ba78632bdf"}, - {file = "lxml-4.6.2-cp36-cp36m-win_amd64.whl", hash = "sha256:6e4183800f16f3679076dfa8abf2db3083919d7e30764a069fb66b2b9eff9939"}, - {file = "lxml-4.6.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d8d3d4713f0c28bdc6c806a278d998546e8efc3498949e3ace6e117462ac0a5e"}, - {file = "lxml-4.6.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:8246f30ca34dc712ab07e51dc34fea883c00b7ccb0e614651e49da2c49a30711"}, - {file = "lxml-4.6.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:923963e989ffbceaa210ac37afc9b906acebe945d2723e9679b643513837b089"}, - {file = "lxml-4.6.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:1471cee35eba321827d7d53d104e7b8c593ea3ad376aa2df89533ce8e1b24a01"}, - {file = "lxml-4.6.2-cp37-cp37m-win32.whl", hash = "sha256:2363c35637d2d9d6f26f60a208819e7eafc4305ce39dc1d5005eccc4593331c2"}, - {file = "lxml-4.6.2-cp37-cp37m-win_amd64.whl", hash = "sha256:f4822c0660c3754f1a41a655e37cb4dbbc9be3d35b125a37fab6f82d47674ebc"}, - {file = "lxml-4.6.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0448576c148c129594d890265b1a83b9cd76fd1f0a6a04620753d9a6bcfd0a4d"}, - {file = "lxml-4.6.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:60a20bfc3bd234d54d49c388950195d23a5583d4108e1a1d47c9eef8d8c042b3"}, - {file = "lxml-4.6.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2e5cc908fe43fe1aa299e58046ad66981131a66aea3129aac7770c37f590a644"}, - {file = "lxml-4.6.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:50c348995b47b5a4e330362cf39fc503b4a43b14a91c34c83b955e1805c8e308"}, - {file = "lxml-4.6.2-cp38-cp38-win32.whl", hash = "sha256:94d55bd03d8671686e3f012577d9caa5421a07286dd351dfef64791cf7c6c505"}, - {file = "lxml-4.6.2-cp38-cp38-win_amd64.whl", hash = "sha256:7a7669ff50f41225ca5d6ee0a1ec8413f3a0d8aa2b109f86d540887b7ec0d72a"}, - {file = "lxml-4.6.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e0bfe9bb028974a481410432dbe1b182e8191d5d40382e5b8ff39cdd2e5c5931"}, - {file = "lxml-4.6.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:6fd8d5903c2e53f49e99359b063df27fdf7acb89a52b6a12494208bf61345a03"}, - {file = "lxml-4.6.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7e9eac1e526386df7c70ef253b792a0a12dd86d833b1d329e038c7a235dfceb5"}, - {file = "lxml-4.6.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:7ee8af0b9f7de635c61cdd5b8534b76c52cd03536f29f51151b377f76e214a1a"}, - {file = "lxml-4.6.2-cp39-cp39-win32.whl", hash = "sha256:2e6fd1b8acd005bd71e6c94f30c055594bbd0aa02ef51a22bbfa961ab63b2d75"}, - {file = "lxml-4.6.2-cp39-cp39-win_amd64.whl", hash = "sha256:535332fe9d00c3cd455bd3dd7d4bacab86e2d564bdf7606079160fa6251caacf"}, - {file = "lxml-4.6.2.tar.gz", hash = "sha256:cd11c7e8d21af997ee8079037fff88f16fda188a9776eb4b81c7e4c9c0a7d7fc"}, + {file = "lxml-4.6.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:df7c53783a46febb0e70f6b05df2ba104610f2fb0d27023409734a3ecbb78fb2"}, + {file = "lxml-4.6.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1b7584d421d254ab86d4f0b13ec662a9014397678a7c4265a02a6d7c2b18a75f"}, + {file = "lxml-4.6.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:079f3ae844f38982d156efce585bc540c16a926d4436712cf4baee0cce487a3d"}, + {file = "lxml-4.6.3-cp27-cp27m-win32.whl", hash = "sha256:bc4313cbeb0e7a416a488d72f9680fffffc645f8a838bd2193809881c67dd106"}, + {file = "lxml-4.6.3-cp27-cp27m-win_amd64.whl", hash = "sha256:8157dadbb09a34a6bd95a50690595e1fa0af1a99445e2744110e3dca7831c4ee"}, + {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7728e05c35412ba36d3e9795ae8995e3c86958179c9770e65558ec3fdfd3724f"}, + {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4bff24dfeea62f2e56f5bab929b4428ae6caba2d1eea0c2d6eb618e30a71e6d4"}, + {file = "lxml-4.6.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:64812391546a18896adaa86c77c59a4998f33c24788cadc35789e55b727a37f4"}, + {file = "lxml-4.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c1a40c06fd5ba37ad39caa0b3144eb3772e813b5fb5b084198a985431c2f1e8d"}, + {file = "lxml-4.6.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:74f7d8d439b18fa4c385f3f5dfd11144bb87c1da034a466c5b5577d23a1d9b51"}, + {file = "lxml-4.6.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586"}, + {file = "lxml-4.6.3-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:4c61b3a0db43a1607d6264166b230438f85bfed02e8cff20c22e564d0faff354"}, + {file = "lxml-4.6.3-cp35-cp35m-manylinux2014_x86_64.whl", hash = "sha256:5c8c163396cc0df3fd151b927e74f6e4acd67160d6c33304e805b84293351d16"}, + {file = "lxml-4.6.3-cp35-cp35m-win32.whl", hash = "sha256:f2380a6376dfa090227b663f9678150ef27543483055cc327555fb592c5967e2"}, + {file = "lxml-4.6.3-cp35-cp35m-win_amd64.whl", hash = "sha256:c4f05c5a7c49d2fb70223d0d5bcfbe474cf928310ac9fa6a7c6dddc831d0b1d4"}, + {file = "lxml-4.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d2e35d7bf1c1ac8c538f88d26b396e73dd81440d59c1ef8522e1ea77b345ede4"}, + {file = "lxml-4.6.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:289e9ca1a9287f08daaf796d96e06cb2bc2958891d7911ac7cae1c5f9e1e0ee3"}, + {file = "lxml-4.6.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:bccbfc27563652de7dc9bdc595cb25e90b59c5f8e23e806ed0fd623755b6565d"}, + {file = "lxml-4.6.3-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:d916d31fd85b2f78c76400d625076d9124de3e4bda8b016d25a050cc7d603f24"}, + {file = "lxml-4.6.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:820628b7b3135403540202e60551e741f9b6d3304371712521be939470b454ec"}, + {file = "lxml-4.6.3-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:c47ff7e0a36d4efac9fd692cfa33fbd0636674c102e9e8d9b26e1b93a94e7617"}, + {file = "lxml-4.6.3-cp36-cp36m-win32.whl", hash = "sha256:5a0a14e264069c03e46f926be0d8919f4105c1623d620e7ec0e612a2e9bf1c04"}, + {file = "lxml-4.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:92e821e43ad382332eade6812e298dc9701c75fe289f2a2d39c7960b43d1e92a"}, + {file = "lxml-4.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:efd7a09678fd8b53117f6bae4fa3825e0a22b03ef0a932e070c0bdbb3a35e654"}, + {file = "lxml-4.6.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:efac139c3f0bf4f0939f9375af4b02c5ad83a622de52d6dfa8e438e8e01d0eb0"}, + {file = "lxml-4.6.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:0fbcf5565ac01dff87cbfc0ff323515c823081c5777a9fc7703ff58388c258c3"}, + {file = "lxml-4.6.3-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:36108c73739985979bf302006527cf8a20515ce444ba916281d1c43938b8bb96"}, + {file = "lxml-4.6.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:122fba10466c7bd4178b07dba427aa516286b846b2cbd6f6169141917283aae2"}, + {file = "lxml-4.6.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:cdaf11d2bd275bf391b5308f86731e5194a21af45fbaaaf1d9e8147b9160ea92"}, + {file = "lxml-4.6.3-cp37-cp37m-win32.whl", hash = "sha256:3439c71103ef0e904ea0a1901611863e51f50b5cd5e8654a151740fde5e1cade"}, + {file = "lxml-4.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:4289728b5e2000a4ad4ab8da6e1db2e093c63c08bdc0414799ee776a3f78da4b"}, + {file = "lxml-4.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b007cbb845b28db4fb8b6a5cdcbf65bacb16a8bd328b53cbc0698688a68e1caa"}, + {file = "lxml-4.6.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:76fa7b1362d19f8fbd3e75fe2fb7c79359b0af8747e6f7141c338f0bee2f871a"}, + {file = "lxml-4.6.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:26e761ab5b07adf5f555ee82fb4bfc35bf93750499c6c7614bd64d12aaa67927"}, + {file = "lxml-4.6.3-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:e1cbd3f19a61e27e011e02f9600837b921ac661f0c40560eefb366e4e4fb275e"}, + {file = "lxml-4.6.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:66e575c62792c3f9ca47cb8b6fab9e35bab91360c783d1606f758761810c9791"}, + {file = "lxml-4.6.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:1b38116b6e628118dea5b2186ee6820ab138dbb1e24a13e478490c7db2f326ae"}, + {file = "lxml-4.6.3-cp38-cp38-win32.whl", hash = "sha256:89b8b22a5ff72d89d48d0e62abb14340d9e99fd637d046c27b8b257a01ffbe28"}, + {file = "lxml-4.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:2a9d50e69aac3ebee695424f7dbd7b8c6d6eb7de2a2eb6b0f6c7db6aa41e02b7"}, + {file = "lxml-4.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ce256aaa50f6cc9a649c51be3cd4ff142d67295bfc4f490c9134d0f9f6d58ef0"}, + {file = "lxml-4.6.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:7610b8c31688f0b1be0ef882889817939490a36d0ee880ea562a4e1399c447a1"}, + {file = "lxml-4.6.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f8380c03e45cf09f8557bdaa41e1fa7c81f3ae22828e1db470ab2a6c96d8bc23"}, + {file = "lxml-4.6.3-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:3082c518be8e97324390614dacd041bb1358c882d77108ca1957ba47738d9d59"}, + {file = "lxml-4.6.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:884ab9b29feaca361f7f88d811b1eea9bfca36cf3da27768d28ad45c3ee6f969"}, + {file = "lxml-4.6.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:6f12e1427285008fd32a6025e38e977d44d6382cf28e7201ed10d6c1698d2a9a"}, + {file = "lxml-4.6.3-cp39-cp39-win32.whl", hash = "sha256:33bb934a044cf32157c12bfcfbb6649807da20aa92c062ef51903415c704704f"}, + {file = "lxml-4.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:542d454665a3e277f76954418124d67516c5f88e51a900365ed54a9806122b83"}, + {file = "lxml-4.6.3.tar.gz", hash = "sha256:39b78571b3b30645ac77b95f7c69d1bffc4cf8c3b157c435a34da72e78c82468"}, ] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, @@ -530,16 +549,16 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] packaging = [ - {file = "packaging-20.9-py2.py3-none-any.whl", hash = "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"}, - {file = "packaging-20.9.tar.gz", hash = "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5"}, + {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, + {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, ] pathspec = [ - {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"}, - {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"}, + {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, + {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, ] pluggy = [ - {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, - {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] py = [ {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, @@ -558,63 +577,63 @@ pyparsing = [ {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] pytest = [ - {file = "pytest-6.2.2-py3-none-any.whl", hash = "sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839"}, - {file = "pytest-6.2.2.tar.gz", hash = "sha256:9d1edf9e7d0b84d72ea3dbcdfd22b35fb543a5e8f2a60092dd578936bf63d7f9"}, + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] pytest-mock = [ {file = "pytest-mock-3.6.1.tar.gz", hash = "sha256:40217a058c52a63f1042f0784f62009e976ba824c418cced42e88d5f40ab0e62"}, {file = "pytest_mock-3.6.1-py3-none-any.whl", hash = "sha256:30c2f2cc9759e76eee674b81ea28c9f0b94f8f0445a1b87762cadf774f0df7e3"}, ] regex = [ - {file = "regex-2021.4.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:619d71c59a78b84d7f18891fe914446d07edd48dc8328c8e149cbe0929b4e000"}, - {file = "regex-2021.4.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:47bf5bf60cf04d72bf6055ae5927a0bd9016096bf3d742fa50d9bf9f45aa0711"}, - {file = "regex-2021.4.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:281d2fd05555079448537fe108d79eb031b403dac622621c78944c235f3fcf11"}, - {file = "regex-2021.4.4-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:bd28bc2e3a772acbb07787c6308e00d9626ff89e3bfcdebe87fa5afbfdedf968"}, - {file = "regex-2021.4.4-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:7c2a1af393fcc09e898beba5dd59196edaa3116191cc7257f9224beaed3e1aa0"}, - {file = "regex-2021.4.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c38c71df845e2aabb7fb0b920d11a1b5ac8526005e533a8920aea97efb8ec6a4"}, - {file = "regex-2021.4.4-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:96fcd1888ab4d03adfc9303a7b3c0bd78c5412b2bfbe76db5b56d9eae004907a"}, - {file = "regex-2021.4.4-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:ade17eb5d643b7fead300a1641e9f45401c98eee23763e9ed66a43f92f20b4a7"}, - {file = "regex-2021.4.4-cp36-cp36m-win32.whl", hash = "sha256:e8e5b509d5c2ff12f8418006d5a90e9436766133b564db0abaec92fd27fcee29"}, - {file = "regex-2021.4.4-cp36-cp36m-win_amd64.whl", hash = "sha256:11d773d75fa650cd36f68d7ca936e3c7afaae41b863b8c387a22aaa78d3c5c79"}, - {file = "regex-2021.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d3029c340cfbb3ac0a71798100ccc13b97dddf373a4ae56b6a72cf70dfd53bc8"}, - {file = "regex-2021.4.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:18c071c3eb09c30a264879f0d310d37fe5d3a3111662438889ae2eb6fc570c31"}, - {file = "regex-2021.4.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:4c557a7b470908b1712fe27fb1ef20772b78079808c87d20a90d051660b1d69a"}, - {file = "regex-2021.4.4-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:01afaf2ec48e196ba91b37451aa353cb7eda77efe518e481707e0515025f0cd5"}, - {file = "regex-2021.4.4-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:3a9cd17e6e5c7eb328517969e0cb0c3d31fd329298dd0c04af99ebf42e904f82"}, - {file = "regex-2021.4.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:90f11ff637fe8798933fb29f5ae1148c978cccb0452005bf4c69e13db951e765"}, - {file = "regex-2021.4.4-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:919859aa909429fb5aa9cf8807f6045592c85ef56fdd30a9a3747e513db2536e"}, - {file = "regex-2021.4.4-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:339456e7d8c06dd36a22e451d58ef72cef293112b559010db3d054d5560ef439"}, - {file = "regex-2021.4.4-cp37-cp37m-win32.whl", hash = "sha256:67bdb9702427ceddc6ef3dc382455e90f785af4c13d495f9626861763ee13f9d"}, - {file = "regex-2021.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:32e65442138b7b76dd8173ffa2cf67356b7bc1768851dded39a7a13bf9223da3"}, - {file = "regex-2021.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1e1c20e29358165242928c2de1482fb2cf4ea54a6a6dea2bd7a0e0d8ee321500"}, - {file = "regex-2021.4.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:314d66636c494ed9c148a42731b3834496cc9a2c4251b1661e40936814542b14"}, - {file = "regex-2021.4.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6d1b01031dedf2503631d0903cb563743f397ccaf6607a5e3b19a3d76fc10480"}, - {file = "regex-2021.4.4-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:741a9647fcf2e45f3a1cf0e24f5e17febf3efe8d4ba1281dcc3aa0459ef424dc"}, - {file = "regex-2021.4.4-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:4c46e22a0933dd783467cf32b3516299fb98cfebd895817d685130cc50cd1093"}, - {file = "regex-2021.4.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:e512d8ef5ad7b898cdb2d8ee1cb09a8339e4f8be706d27eaa180c2f177248a10"}, - {file = "regex-2021.4.4-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:980d7be47c84979d9136328d882f67ec5e50008681d94ecc8afa8a65ed1f4a6f"}, - {file = "regex-2021.4.4-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:ce15b6d103daff8e9fee13cf7f0add05245a05d866e73926c358e871221eae87"}, - {file = "regex-2021.4.4-cp38-cp38-win32.whl", hash = "sha256:a91aa8619b23b79bcbeb37abe286f2f408d2f2d6f29a17237afda55bb54e7aac"}, - {file = "regex-2021.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:c0502c0fadef0d23b128605d69b58edb2c681c25d44574fc673b0e52dce71ee2"}, - {file = "regex-2021.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:598585c9f0af8374c28edd609eb291b5726d7cbce16be6a8b95aa074d252ee17"}, - {file = "regex-2021.4.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:ee54ff27bf0afaf4c3b3a62bcd016c12c3fdb4ec4f413391a90bd38bc3624605"}, - {file = "regex-2021.4.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7d9884d86dd4dd489e981d94a65cd30d6f07203d90e98f6f657f05170f6324c9"}, - {file = "regex-2021.4.4-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:bf5824bfac591ddb2c1f0a5f4ab72da28994548c708d2191e3b87dd207eb3ad7"}, - {file = "regex-2021.4.4-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:563085e55b0d4fb8f746f6a335893bda5c2cef43b2f0258fe1020ab1dd874df8"}, - {file = "regex-2021.4.4-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b9c3db21af35e3b3c05764461b262d6f05bbca08a71a7849fd79d47ba7bc33ed"}, - {file = "regex-2021.4.4-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:3916d08be28a1149fb97f7728fca1f7c15d309a9f9682d89d79db75d5e52091c"}, - {file = "regex-2021.4.4-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:fd45ff9293d9274c5008a2054ecef86a9bfe819a67c7be1afb65e69b405b3042"}, - {file = "regex-2021.4.4-cp39-cp39-win32.whl", hash = "sha256:fa4537fb4a98fe8fde99626e4681cc644bdcf2a795038533f9f711513a862ae6"}, - {file = "regex-2021.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:97f29f57d5b84e73fbaf99ab3e26134e6687348e95ef6b48cfd2c06807005a07"}, - {file = "regex-2021.4.4.tar.gz", hash = "sha256:52ba3d3f9b942c49d7e4bc105bb28551c44065f139a65062ab7912bef10c9afb"}, + {file = "regex-2021.9.30-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:66696c8336a1b5d1182464f3af3427cc760118f26d0b09a2ddc16a976a4d2637"}, + {file = "regex-2021.9.30-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d87459ad3ab40cd8493774f8a454b2e490d8e729e7e402a0625867a983e4e02"}, + {file = "regex-2021.9.30-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78cf6a1e023caf5e9a982f5377414e1aeac55198831b852835732cfd0a0ca5ff"}, + {file = "regex-2021.9.30-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:255791523f80ea8e48e79af7120b4697ef3b74f6886995dcdb08c41f8e516be0"}, + {file = "regex-2021.9.30-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e502f8d4e5ef714bcc2c94d499684890c94239526d61fdf1096547db91ca6aa6"}, + {file = "regex-2021.9.30-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4907fb0f9b9309a5bded72343e675a252c2589a41871874feace9a05a540241e"}, + {file = "regex-2021.9.30-cp310-cp310-win32.whl", hash = "sha256:3be40f720af170a6b20ddd2ad7904c58b13d2b56f6734ee5d09bbdeed2fa4816"}, + {file = "regex-2021.9.30-cp310-cp310-win_amd64.whl", hash = "sha256:c2b180ed30856dfa70cfe927b0fd38e6b68198a03039abdbeb1f2029758d87e7"}, + {file = "regex-2021.9.30-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e6f2d2f93001801296fe3ca86515eb04915472b5380d4d8752f09f25f0b9b0ed"}, + {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fa7ba9ab2eba7284e0d7d94f61df7af86015b0398e123331362270d71fab0b9"}, + {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28040e89a04b60d579c69095c509a4f6a1a5379cd865258e3a186b7105de72c6"}, + {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f588209d3e4797882cd238195c175290dbc501973b10a581086b5c6bcd095ffb"}, + {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42952d325439ef223e4e9db7ee6d9087b5c68c5c15b1f9de68e990837682fc7b"}, + {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cae4099031d80703954c39680323dabd87a69b21262303160776aa0e55970ca0"}, + {file = "regex-2021.9.30-cp36-cp36m-win32.whl", hash = "sha256:0de8ad66b08c3e673b61981b9e3626f8784d5564f8c3928e2ad408c0eb5ac38c"}, + {file = "regex-2021.9.30-cp36-cp36m-win_amd64.whl", hash = "sha256:b345ecde37c86dd7084c62954468a4a655fd2d24fd9b237949dd07a4d0dd6f4c"}, + {file = "regex-2021.9.30-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6f08187136f11e430638c2c66e1db091105d7c2e9902489f0dbc69b44c222b4"}, + {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b55442650f541d195a535ccec33078c78a9521973fb960923da7515e9ed78fa6"}, + {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87e9c489aa98f50f367fb26cc9c8908d668e9228d327644d7aa568d47e456f47"}, + {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e2cb7d4909ed16ed35729d38af585673f1f0833e73dfdf0c18e5be0061107b99"}, + {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0861e7f6325e821d5c40514c551fd538b292f8cc3960086e73491b9c5d8291d"}, + {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:81fdc90f999b2147fc62e303440c424c47e5573a9b615ed5d43a5b832efcca9e"}, + {file = "regex-2021.9.30-cp37-cp37m-win32.whl", hash = "sha256:8c1ad61fa024195136a6b7b89538030bd00df15f90ac177ca278df9b2386c96f"}, + {file = "regex-2021.9.30-cp37-cp37m-win_amd64.whl", hash = "sha256:e3770781353a4886b68ef10cec31c1f61e8e3a0be5f213c2bb15a86efd999bc4"}, + {file = "regex-2021.9.30-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9c065d95a514a06b92a5026766d72ac91bfabf581adb5b29bc5c91d4b3ee9b83"}, + {file = "regex-2021.9.30-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9925985be05d54b3d25fd6c1ea8e50ff1f7c2744c75bdc4d3b45c790afa2bcb3"}, + {file = "regex-2021.9.30-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:470f2c882f2672d8eeda8ab27992aec277c067d280b52541357e1acd7e606dae"}, + {file = "regex-2021.9.30-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ad0517df22a97f1da20d8f1c8cb71a5d1997fa383326b81f9cf22c9dadfbdf34"}, + {file = "regex-2021.9.30-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9e30838df7bfd20db6466fd309d9b580d32855f8e2c2e6d74cf9da27dcd9b63"}, + {file = "regex-2021.9.30-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b34d2335d6aedec7dcadd3f8283b9682fadad8b9b008da8788d2fce76125ebe"}, + {file = "regex-2021.9.30-cp38-cp38-win32.whl", hash = "sha256:e07049cece3462c626d650e8bf42ddbca3abf4aa08155002c28cb6d9a5a281e2"}, + {file = "regex-2021.9.30-cp38-cp38-win_amd64.whl", hash = "sha256:37868075eda024470bd0feab872c692ac4ee29db1e14baec103257bf6cc64346"}, + {file = "regex-2021.9.30-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d331f238a7accfbbe1c4cd1ba610d4c087b206353539331e32a8f05345c74aec"}, + {file = "regex-2021.9.30-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6348a7ab2a502cbdd0b7fd0496d614007489adb7361956b38044d1d588e66e04"}, + {file = "regex-2021.9.30-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce7b1cca6c23f19bee8dc40228d9c314d86d1e51996b86f924aca302fc8f8bf9"}, + {file = "regex-2021.9.30-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1f1125bc5172ab3a049bc6f4b9c0aae95a2a2001a77e6d6e4239fa3653e202b5"}, + {file = "regex-2021.9.30-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:638e98d069b14113e8afba6a54d1ca123f712c0d105e67c1f9211b2a825ef926"}, + {file = "regex-2021.9.30-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9a0b0db6b49da7fa37ca8eddf9f40a8dbc599bad43e64f452284f37b6c34d91c"}, + {file = "regex-2021.9.30-cp39-cp39-win32.whl", hash = "sha256:9910869c472e5a6728680ca357b5846546cbbd2ab3ad5bef986ef0bc438d0aa6"}, + {file = "regex-2021.9.30-cp39-cp39-win_amd64.whl", hash = "sha256:3b71213ec3bad9a5a02e049f2ec86b3d7c3e350129ae0f4e2f99c12b5da919ed"}, + {file = "regex-2021.9.30.tar.gz", hash = "sha256:81e125d9ba54c34579e4539a967e976a3c56150796674aec318b1b2f49251be7"}, ] requests = [ - {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, - {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, + {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, + {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, ] responses = [ - {file = "responses-0.13.3-py2.py3-none-any.whl", hash = "sha256:b54067596f331786f5ed094ff21e8d79e6a1c68ef625180a7d34808d6f36c11b"}, - {file = "responses-0.13.3.tar.gz", hash = "sha256:18a5b88eb24143adbf2b4100f328a2f5bfa72fbdacf12d97d41f07c26c45553d"}, + {file = "responses-0.13.4-py2.py3-none-any.whl", hash = "sha256:d8d0f655710c46fd3513b9202a7f0dcedd02ca0f8cf4976f27fa8ab5b81e656d"}, + {file = "responses-0.13.4.tar.gz", hash = "sha256:9476775d856d3c24ae660bbebe29fb6d789d4ad16acd723efbfb6ee20990b899"}, ] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, @@ -657,11 +676,11 @@ typed-ast = [ {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, ] typing-extensions = [ - {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, - {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, - {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, + {file = "typing_extensions-3.10.0.2-py2-none-any.whl", hash = "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7"}, + {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"}, + {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, ] urllib3 = [ - {file = "urllib3-1.26.3-py2.py3-none-any.whl", hash = "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80"}, - {file = "urllib3-1.26.3.tar.gz", hash = "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"}, + {file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"}, + {file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"}, ] diff --git a/pyproject.toml b/pyproject.toml index eb12cfd..6ef0c87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ include = [ ] [tool.poetry.dependencies] -python = "^3.9" +python = "^3.8" requests = "^2.25.1" lxml = "^4.6.2" cssselect = "^1.1.0" From 15a39bb8a0c9460df73fe01833703cbac363ead0 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 5 Oct 2021 13:30:53 +0200 Subject: [PATCH 073/149] Add tox for running tests on python 3.8. --- .gitignore | 1 + Makefile | 4 ++ poetry.lock | 111 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + tox.ini | 9 ++++ 5 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index ddf4605..4411e41 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist/ .coverage htmlcov/ +.tox/ diff --git a/Makefile b/Makefile index a37b466..83c4c2c 100644 --- a/Makefile +++ b/Makefile @@ -35,3 +35,7 @@ pre-commit: ## Checks to run before each commit poetry run isort src tests --check poetry run black src tests --check poetry run flake8 src tests + +.PHONY: tox +tox: ## Run tests via tox + poetry run tox diff --git a/poetry.lock b/poetry.lock index 1de5b1a..5632fbc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -28,6 +28,18 @@ docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] +[[package]] +name = "backports.entry-points-selectable" +version = "1.1.0" +description = "Compatibility shim providing selectable entry points for older implementations" +category = "dev" +optional = false +python-versions = ">=2.7" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-flake8", "pytest-cov", "pytest-black (>=0.3.7)", "pytest-mypy", "pytest-checkdocs (>=2.4)", "pytest-enabler (>=1.0.1)"] + [[package]] name = "black" version = "20.8b1" @@ -118,6 +130,26 @@ python-versions = "*" [package.dependencies] requests = "*" +[[package]] +name = "distlib" +version = "0.3.3" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "filelock" +version = "3.3.0" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] +testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] + [[package]] name = "flake8" version = "3.9.2" @@ -210,6 +242,18 @@ category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +[[package]] +name = "platformdirs" +version = "2.4.0" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] +test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] + [[package]] name = "pluggy" version = "1.0.0" @@ -347,6 +391,28 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tox" +version = "3.24.4" +description = "tox is a generic virtualenv management and test command line tool" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +colorama = {version = ">=0.4.1", markers = "platform_system == \"Windows\""} +filelock = ">=3.0.0" +packaging = ">=14" +pluggy = ">=0.12.0" +py = ">=1.4.17" +six = ">=1.14.0" +toml = ">=0.9.4" +virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2,<20.0.3 || >20.0.3,<20.0.4 || >20.0.4,<20.0.5 || >20.0.5,<20.0.6 || >20.0.6,<20.0.7 || >20.0.7" + +[package.extras] +docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"] +testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)", "pytest-xdist (>=1.22.2)", "pathlib2 (>=2.3.3)"] + [[package]] name = "typed-ast" version = "1.4.3" @@ -376,10 +442,29 @@ brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "virtualenv" +version = "20.8.1" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +"backports.entry-points-selectable" = ">=1.0.4" +distlib = ">=0.3.1,<1" +filelock = ">=3.0.0,<4" +platformdirs = ">=2,<3" +six = ">=1.9.0,<2" + +[package.extras] +docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)"] +testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] + [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "252531625354ed4b7e8f14839d769020ebcce296da72fcd27c1eb1290dd9ca63" +content-hash = "94082325c885ed7e0cf8cf137f9f1d8ced4b84e746adc192a8cdaa2a61e22fac" [metadata.files] appdirs = [ @@ -394,6 +479,10 @@ attrs = [ {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, ] +"backports.entry-points-selectable" = [ + {file = "backports.entry_points_selectable-1.1.0-py2.py3-none-any.whl", hash = "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc"}, + {file = "backports.entry_points_selectable-1.1.0.tar.gz", hash = "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a"}, +] black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, ] @@ -474,6 +563,14 @@ cssselect = [ curlify = [ {file = "curlify-2.2.1.tar.gz", hash = "sha256:0d3f02e7235faf952de8ef45ef469845196d30632d5838bcd5aee217726ddd6d"}, ] +distlib = [ + {file = "distlib-0.3.3-py2.py3-none-any.whl", hash = "sha256:c8b54e8454e5bf6237cc84c20e8264c3e991e824ef27e8f1e81049867d861e31"}, + {file = "distlib-0.3.3.zip", hash = "sha256:d982d0751ff6eaaab5e2ec8e691d949ee80eddf01a62eaa96ddb11531fe16b05"}, +] +filelock = [ + {file = "filelock-3.3.0-py3-none-any.whl", hash = "sha256:bbc6a0382fe8ec4744ecdf6683a2e07f65eb10ff1aff53fc02a202565446cde0"}, + {file = "filelock-3.3.0.tar.gz", hash = "sha256:8c7eab13dc442dc249e95158bcc12dec724465919bdc9831fdbf0660f03d1785"}, +] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, @@ -556,6 +653,10 @@ pathspec = [ {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, ] +platformdirs = [ + {file = "platformdirs-2.4.0-py3-none-any.whl", hash = "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d"}, + {file = "platformdirs-2.4.0.tar.gz", hash = "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2"}, +] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, @@ -643,6 +744,10 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +tox = [ + {file = "tox-3.24.4-py2.py3-none-any.whl", hash = "sha256:5e274227a53dc9ef856767c21867377ba395992549f02ce55eb549f9fb9a8d10"}, + {file = "tox-3.24.4.tar.gz", hash = "sha256:c30b57fa2477f1fb7c36aa1d83292d5c2336cd0018119e1b1c17340e2c2708ca"}, +] typed-ast = [ {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, @@ -684,3 +789,7 @@ urllib3 = [ {file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"}, {file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"}, ] +virtualenv = [ + {file = "virtualenv-20.8.1-py2.py3-none-any.whl", hash = "sha256:10062e34c204b5e4ec5f62e6ef2473f8ba76513a9a617e873f1f8fb4a519d300"}, + {file = "virtualenv-20.8.1.tar.gz", hash = "sha256:bcc17f0b3a29670dd777d6f0755a4c04f28815395bca279cdcb213b97199a6b8"}, +] diff --git a/pyproject.toml b/pyproject.toml index 6ef0c87..7b42598 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ isort = "^5.8.0" responses = "^0.13.3" coverage = "^5.5" pytest-mock = "^3.6.1" +tox = "^3.24.3" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..15bf882 --- /dev/null +++ b/tox.ini @@ -0,0 +1,9 @@ +[tox] +isolated_build = True +envlist = py38,py39 + +[testenv] +deps = poetry +commands = + poetry install + poetry run make test From 2c7ac5f084c94824d4a349353331de8e8afeb0db Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 5 Oct 2021 13:32:42 +0200 Subject: [PATCH 074/149] Version 1.2.4. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7b42598..2569d53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.2.3" +version = "1.2.4" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 10aa336..b3f9ac7 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.2.3" +__version__ = "1.2.4" From 249ea0b2dbecd4d9f47d92dfa3ce7f380a505d40 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 5 Oct 2021 13:39:47 +0200 Subject: [PATCH 075/149] Update live test to match (unexpectedly) new Alphavantage results. --- tests/live.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/live.sh b/tests/live.sh index 021eee8..93b9ba0 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -64,7 +64,7 @@ date,base,quote,amount,source,type 2021-01-05,AUD,EUR,0.63086,alphavantage,close 2021-01-06,AUD,EUR,0.63306,alphavantage,close 2021-01-07,AUD,EUR,0.63284,alphavantage,close -2021-01-08,AUD,EUR,0.63530,alphavantage,close +2021-01-08,AUD,EUR,0.63360,alphavantage,close END run_test "$name" "$cmd" "$expected" From afd41da6ef2c7138dadccee3fdcfe80f5931e950 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 12 Nov 2021 13:40:46 +0100 Subject: [PATCH 076/149] Fix handling of Yahoo date rows with nulls. --- src/pricehist/sources/yahoo.py | 4 +++- tests/pricehist/sources/test_yahoo.py | 14 ++++++++++++++ .../sources/test_yahoo/ibm-date-with-nulls.csv | 4 ++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index ae3179a..f93f53f 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -84,8 +84,10 @@ class Yahoo(BaseSource): def _amount(self, row, type): if type == "mid" and row["high"] != "null" and row["low"] != "null": return sum([Decimal(row["high"]), Decimal(row["low"])]) / 2 - else: + elif row[type] != "null": return Decimal(row[type]) + else: + return None def _data(self, series) -> (dict, csv.DictReader): base_url = "https://query1.finance.yahoo.com/v7/finance" diff --git a/tests/pricehist/sources/test_yahoo.py b/tests/pricehist/sources/test_yahoo.py index 51f6ca9..e1954dc 100644 --- a/tests/pricehist/sources/test_yahoo.py +++ b/tests/pricehist/sources/test_yahoo.py @@ -64,6 +64,13 @@ def long_ok(requests_mock): yield requests_mock +@pytest.fixture +def date_with_nulls_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "ibm-date-with-nulls.csv").read_text() + requests_mock.add(responses.GET, history_url("IBM"), body=json, status=200) + yield requests_mock + + def test_normalizesymbol(src): assert src.normalizesymbol("tsla") == "TSLA" @@ -163,6 +170,13 @@ def test_fetch_from_before_start(src, type, spark_ok, long_ok): assert len(series.prices) > 9 +def test_fetch_skips_dates_with_nulls(src, type, spark_ok, date_with_nulls_ok): + series = src.fetch(Series("IBM", "", type, "2021-01-05", "2021-01-07")) + assert series.prices[0] == Price("2021-01-05", Decimal("123.101204")) + assert series.prices[1] == Price("2021-01-07", Decimal("125.882545")) + assert len(series.prices) == 2 + + def test_fetch_to_future(src, type, spark_ok, recent_ok): series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2100-01-08")) assert len(series.prices) > 0 diff --git a/tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv b/tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv new file mode 100644 index 0000000..601b395 --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv @@ -0,0 +1,4 @@ +Date,Open,High,Low,Close,Adj Close,Volume +2021-01-05,125.010002,126.680000,124.610001,126.139999,123.101204,6114600 +2021-01-06,null,null,null,null,null,null +2021-01-07,130.039993,130.460007,128.259995,128.990005,125.882545,4507400 From a1b87c36f501cfc2f5695a694994244e2f81e8e0 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 12 Nov 2021 13:49:59 +0100 Subject: [PATCH 077/149] Fix assertion for changed label for optional arguments. --- tests/pricehist/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pricehist/test_cli.py b/tests/pricehist/test_cli.py index 16bd383..0c4d8ef 100644 --- a/tests/pricehist/test_cli.py +++ b/tests/pricehist/test_cli.py @@ -54,7 +54,7 @@ def test_cli_no_args_shows_usage(capfd): cli.cli(w("pricehist")) out, err = capfd.readouterr() assert "usage: pricehist" in out - assert "optional arguments:" in out + assert "optional arguments:" in out or "options:" in out assert "commands:" in out @@ -64,7 +64,7 @@ def test_cli_help_shows_usage_and_exits(capfd): assert e.value.code == 0 out, err = capfd.readouterr() assert "usage: pricehist" in out - assert "optional arguments:" in out + assert "optional arguments:" in out or "options:" in out assert "commands:" in out From 039d7fb809fe5dd4d43685952ec7993e61f0f20f Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 12 Nov 2021 13:55:06 +0100 Subject: [PATCH 078/149] Version 1.2.5. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2569d53..8a1d3b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.2.4" +version = "1.2.5" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index b3f9ac7..b7e1990 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.2.4" +__version__ = "1.2.5" From 947eaacd299453ca3ec22b5d77f52c956cb6439c Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 27 Dec 2021 08:31:55 +1100 Subject: [PATCH 079/149] Support Bank of Canada daily exchange rates. --- README.md | 1 + poetry.lock | 53 ++-- src/pricehist/beanprice/bankofcanada.py | 4 + src/pricehist/sources/__init__.py | 2 + src/pricehist/sources/bankofcanada.py | 118 ++++++++ tests/live.sh | 12 + tests/pricehist/sources/test_bankofcanada.py | 246 ++++++++++++++++ .../test_bankofcanada/all-partial.json | 101 +++++++ .../sources/test_bankofcanada/recent.json | 41 +++ .../test_bankofcanada/series-partial.json | 272 ++++++++++++++++++ 10 files changed, 818 insertions(+), 32 deletions(-) create mode 100644 src/pricehist/beanprice/bankofcanada.py create mode 100644 src/pricehist/sources/bankofcanada.py create mode 100644 tests/pricehist/sources/test_bankofcanada.py create mode 100644 tests/pricehist/sources/test_bankofcanada/all-partial.json create mode 100644 tests/pricehist/sources/test_bankofcanada/recent.json create mode 100644 tests/pricehist/sources/test_bankofcanada/series-partial.json diff --git a/README.md b/README.md index 6db9850..cdb9bd9 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ pipx install pricehist ## Sources - **`alphavantage`**: [Alpha Vantage](https://www.alphavantage.co/) +- **`bankofcanada`**: [Bank of Canada daily exchange rates](https://www.bankofcanada.ca/valet/docs) - **`coinbasepro`**: [Coinbase Pro](https://pro.coinbase.com/) - **`coindesk`**: [CoinDesk Bitcoin Price Index](https://www.coindesk.com/coindesk-api) - **`coinmarketcap`**: [CoinMarketCap](https://coinmarketcap.com/) diff --git a/poetry.lock b/poetry.lock index 5632fbc..e71f93b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -415,11 +415,11 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "psutil (>=5.6.1)", "pytes [[package]] name = "typed-ast" -version = "1.4.3" +version = "1.5.1" description = "a fork of Python 2 and 3 ast modules with type comment support" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "typing-extensions" @@ -749,36 +749,25 @@ tox = [ {file = "tox-3.24.4.tar.gz", hash = "sha256:c30b57fa2477f1fb7c36aa1d83292d5c2336cd0018119e1b1c17340e2c2708ca"}, ] typed-ast = [ - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528"}, - {file = "typed_ast-1.4.3-cp35-cp35m-win32.whl", hash = "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428"}, - {file = "typed_ast-1.4.3-cp35-cp35m-win_amd64.whl", hash = "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3"}, - {file = "typed_ast-1.4.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f"}, - {file = "typed_ast-1.4.3-cp36-cp36m-win32.whl", hash = "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363"}, - {file = "typed_ast-1.4.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7"}, - {file = "typed_ast-1.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899"}, - {file = "typed_ast-1.4.3-cp37-cp37m-win32.whl", hash = "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c"}, - {file = "typed_ast-1.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805"}, - {file = "typed_ast-1.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39"}, - {file = "typed_ast-1.4.3-cp38-cp38-win32.whl", hash = "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927"}, - {file = "typed_ast-1.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40"}, - {file = "typed_ast-1.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3"}, - {file = "typed_ast-1.4.3-cp39-cp39-win32.whl", hash = "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808"}, - {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, - {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, + {file = "typed_ast-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d8314c92414ce7481eee7ad42b353943679cf6f30237b5ecbf7d835519e1212"}, + {file = "typed_ast-1.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b53ae5de5500529c76225d18eeb060efbcec90ad5e030713fe8dab0fb4531631"}, + {file = "typed_ast-1.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:24058827d8f5d633f97223f5148a7d22628099a3d2efe06654ce872f46f07cdb"}, + {file = "typed_ast-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:a6d495c1ef572519a7bac9534dbf6d94c40e5b6a608ef41136133377bba4aa08"}, + {file = "typed_ast-1.5.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:de4ecae89c7d8b56169473e08f6bfd2df7f95015591f43126e4ea7865928677e"}, + {file = "typed_ast-1.5.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:256115a5bc7ea9e665c6314ed6671ee2c08ca380f9d5f130bd4d2c1f5848d695"}, + {file = "typed_ast-1.5.1-cp36-cp36m-win_amd64.whl", hash = "sha256:7c42707ab981b6cf4b73490c16e9d17fcd5227039720ca14abe415d39a173a30"}, + {file = "typed_ast-1.5.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:71dcda943a471d826ea930dd449ac7e76db7be778fcd722deb63642bab32ea3f"}, + {file = "typed_ast-1.5.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4f30a2bcd8e68adbb791ce1567fdb897357506f7ea6716f6bbdd3053ac4d9471"}, + {file = "typed_ast-1.5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ca9e8300d8ba0b66d140820cf463438c8e7b4cdc6fd710c059bfcfb1531d03fb"}, + {file = "typed_ast-1.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9caaf2b440efb39ecbc45e2fabde809cbe56272719131a6318fd9bf08b58e2cb"}, + {file = "typed_ast-1.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c9bcad65d66d594bffab8575f39420fe0ee96f66e23c4d927ebb4e24354ec1af"}, + {file = "typed_ast-1.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:591bc04e507595887160ed7aa8d6785867fb86c5793911be79ccede61ae96f4d"}, + {file = "typed_ast-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:a80d84f535642420dd17e16ae25bb46c7f4c16ee231105e7f3eb43976a89670a"}, + {file = "typed_ast-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:38cf5c642fa808300bae1281460d4f9b7617cf864d4e383054a5ef336e344d32"}, + {file = "typed_ast-1.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b6ab14c56bc9c7e3c30228a0a0b54b915b1579613f6e463ba6f4eb1382e7fd4"}, + {file = "typed_ast-1.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a2b8d7007f6280e36fa42652df47087ac7b0a7d7f09f9468f07792ba646aac2d"}, + {file = "typed_ast-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:b6d17f37f6edd879141e64a5db17b67488cfeffeedad8c5cec0392305e9bc775"}, + {file = "typed_ast-1.5.1.tar.gz", hash = "sha256:484137cab8ecf47e137260daa20bafbba5f4e3ec7fda1c1e69ab299b75fa81c5"}, ] typing-extensions = [ {file = "typing_extensions-3.10.0.2-py2-none-any.whl", hash = "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7"}, diff --git a/src/pricehist/beanprice/bankofcanada.py b/src/pricehist/beanprice/bankofcanada.py new file mode 100644 index 0000000..7c09ba7 --- /dev/null +++ b/src/pricehist/beanprice/bankofcanada.py @@ -0,0 +1,4 @@ +from pricehist import beanprice +from pricehist.sources.bankofcanada import BankOfCanada + +Source = beanprice.source(BankOfCanada()) diff --git a/src/pricehist/sources/__init__.py b/src/pricehist/sources/__init__.py index 36bf676..dfde1be 100644 --- a/src/pricehist/sources/__init__.py +++ b/src/pricehist/sources/__init__.py @@ -1,4 +1,5 @@ from .alphavantage import AlphaVantage +from .bankofcanada import BankOfCanada from .coinbasepro import CoinbasePro from .coindesk import CoinDesk from .coinmarketcap import CoinMarketCap @@ -9,6 +10,7 @@ by_id = { source.id(): source for source in [ AlphaVantage(), + BankOfCanada(), CoinbasePro(), CoinDesk(), CoinMarketCap(), diff --git a/src/pricehist/sources/bankofcanada.py b/src/pricehist/sources/bankofcanada.py new file mode 100644 index 0000000..67b3b59 --- /dev/null +++ b/src/pricehist/sources/bankofcanada.py @@ -0,0 +1,118 @@ +import dataclasses +import json +from decimal import Decimal + +import requests + +from pricehist import exceptions +from pricehist.price import Price + +from .basesource import BaseSource + + +class BankOfCanada(BaseSource): + def id(self): + return "bankofcanada" + + def name(self): + return "Bank of Canada" + + def description(self): + return "Daily exchange rates of the Canadian dollar from the Bank of Canada" + + def source_url(self): + return "https://www.bankofcanada.ca/valet/docs" + + def start(self): + return "2017-01-03" + + def types(self): + return ["default"] + + def notes(self): + return ( + "Currently, only daily exchange rates are supported. They are " + "published once each business day by 16:30 ET. " + "All Bank of Canada exchange rates are indicative rates only.\n" + "To request support for other data provided by the " + "Bank of Canada Valet Web Services, please open an " + "issue in pricehist's Gitlab project. " + ) + + def symbols(self): + url = "https://www.bankofcanada.ca/valet/lists/series/json" + + try: + response = self.log_curl(requests.get(url)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + data = json.loads(response.content) + series_names = data["series"].keys() + fx_series_names = [ + n for n in series_names if len(n) == 8 and n[0:2] == "FX" + ] + results = [ + (f"{n[2:5]}/{n[5:9]}", data["series"][n]["description"]) + for n in sorted(fx_series_names) + ] + + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if not results: + raise exceptions.ResponseParsingError("Expected data not found") + else: + return results + + def fetch(self, series): + if len(series.base) != 3 or len(series.quote) != 3: + raise exceptions.InvalidPair(series.base, series.quote, self) + + series_name = f"FX{series.base}{series.quote}" + data = self._data(series, series_name) + + prices = [] + for o in data.get("observations", []): + prices.append(Price(o["d"], Decimal(o[series_name]["v"]))) + + return dataclasses.replace(series, prices=prices) + + def _data(self, series, series_name): + url = f"https://www.bankofcanada.ca/valet/observations/{series_name}/json" + params = { + "start_date": series.start, + "end_date": series.end, + "order_dir": "asc", + } + + try: + response = self.log_curl(requests.get(url, params=params)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + code = response.status_code + text = response.text + + try: + result = json.loads(response.content) + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if code == 404 and "not found" in text: + raise exceptions.InvalidPair(series.base, series.quote, self) + elif code == 400 and "End date must be greater than the Start date" in text: + raise exceptions.BadResponse(result["message"]) + else: + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + return result diff --git a/tests/live.sh b/tests/live.sh index 93b9ba0..543d634 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -80,6 +80,18 @@ date,base,quote,amount,source,type END run_test "$name" "$cmd" "$expected" +name="Bank of Canada" +cmd="pricehist fetch bankofcanada CAD/USD -s 2021-01-04 -e 2021-01-08" +read -r -d '' expected < 0 + + assert isinstance(src.name(), str) + assert len(src.name()) > 0 + + assert isinstance(src.description(), str) + assert len(src.description()) > 0 + + assert isinstance(src.source_url(), str) + assert src.source_url().startswith("http") + + assert datetime.strptime(src.start(), "%Y-%m-%d") + + assert isinstance(src.types(), list) + assert len(src.types()) > 0 + assert isinstance(src.types()[0], str) + assert len(src.types()[0]) > 0 + + assert isinstance(src.notes(), str) + + +def test_symbols(src, series_list_response_ok): + syms = src.symbols() + assert ("CAD/USD", "Canadian dollar to US dollar daily exchange rate") in syms + assert len(syms) > 3 + + +def test_symbols_requests_logged(src, series_list_response_ok, caplog): + with caplog.at_level(logging.DEBUG): + src.symbols() + assert any( + ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] + ) + + +def test_symbols_not_found(src, requests_mock, series_list_url): + requests_mock.add(responses.GET, series_list_url, body='{"series":{}}', status=200) + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "data not found" in str(e.value) + + +def test_symbols_network_issue(src, requests_mock, series_list_url): + requests_mock.add( + responses.GET, + series_list_url, + body=requests.exceptions.ConnectionError("Network issue"), + ) + with pytest.raises(exceptions.RequestError) as e: + src.symbols() + assert "Network issue" in str(e.value) + + +def test_symbols_bad_status(src, requests_mock, series_list_url): + requests_mock.add(responses.GET, series_list_url, status=500) + with pytest.raises(exceptions.BadResponse) as e: + src.symbols() + assert "Server Error" in str(e.value) + + +def test_symbols_parsing_error(src, requests_mock, series_list_url): + requests_mock.add(responses.GET, series_list_url, body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.symbols() + assert "while parsing data" in str(e.value) + + +def test_fetch_known_pair(src, type, recent_response_ok): + series = src.fetch(Series("CAD", "USD", type, "2021-01-01", "2021-01-07")) + req = recent_response_ok.calls[0].request + assert req.params["order_dir"] == "asc" + assert req.params["start_date"] == "2021-01-01" + assert req.params["end_date"] == "2021-01-07" + assert series.prices[0] == Price("2021-01-04", Decimal("0.7843")) + assert series.prices[-1] == Price("2021-01-07", Decimal("0.7870")) + assert len(series.prices) == 4 + + +def test_fetch_requests_logged(src, type, recent_response_ok, caplog): + with caplog.at_level(logging.DEBUG): + src.fetch(Series("CAD", "USD", type, "2021-01-01", "2021-01-07")) + assert any( + ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] + ) + + +def test_fetch_long_hist_from_start(src, type, all_response_ok): + series = src.fetch(Series("CAD", "USD", type, src.start(), "2021-01-07")) + assert series.prices[0] == Price("2017-01-03", Decimal("0.7443")) + assert series.prices[-1] == Price("2021-01-07", Decimal("0.7870")) + assert len(series.prices) > 13 + + +def test_fetch_from_before_start(src, type, requests_mock): + body = """{ "observations": [] }""" + requests_mock.add(responses.GET, fetch_url("FXCADUSD"), status=200, body=body) + series = src.fetch(Series("CAD", "USD", type, "2000-01-01", "2017-01-01")) + assert len(series.prices) == 0 + + +def test_fetch_to_future(src, type, all_response_ok): + series = src.fetch(Series("CAD", "USD", type, "2021-01-01", "2100-01-01")) + assert len(series.prices) > 0 + + +def test_wrong_dates_order(src, type, requests_mock): + body = """{ "message": "The End date must be greater than the Start date." }""" + requests_mock.add(responses.GET, fetch_url("FXCADUSD"), status=400, body=body) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("CAD", "USD", type, "2021-01-07", "2021-01-01")) + assert "End date must be greater" in str(e.value) + + +def test_fetch_in_future(src, type, requests_mock): + body = """{ "observations": [] }""" + requests_mock.add(responses.GET, fetch_url("FXCADUSD"), status=200, body=body) + series = src.fetch(Series("CAD", "USD", type, "2030-01-01", "2030-01-07")) + assert len(series.prices) == 0 + + +def test_fetch_empty(src, type, requests_mock): + requests_mock.add( + responses.GET, fetch_url("FXCADUSD"), body="""{"observations":{}}""" + ) + series = src.fetch(Series("CAD", "USD", type, "2021-01-03", "2021-01-03")) + assert len(series.prices) == 0 + + +def test_fetch_no_quote(src, type): + with pytest.raises(exceptions.InvalidPair): + src.fetch(Series("CAD", "", type, "2021-01-01", "2021-01-07")) + + +def test_fetch_unknown_pair(src, type, requests_mock): + requests_mock.add( + responses.GET, + fetch_url("FXCADAFN"), + status=404, + body="""{ + "message": "Series FXCADAFN not found.", + "docs": "https://www.bankofcanada.ca/valet/docs" + }""", + ) + with pytest.raises(exceptions.InvalidPair): + src.fetch(Series("CAD", "AFN", type, "2021-01-01", "2021-01-07")) + + +def test_fetch_network_issue(src, type, requests_mock): + body = requests.exceptions.ConnectionError("Network issue") + requests_mock.add(responses.GET, fetch_url("FXCADUSD"), body=body) + with pytest.raises(exceptions.RequestError) as e: + src.fetch(Series("CAD", "USD", type, "2021-01-01", "2021-01-07")) + assert "Network issue" in str(e.value) + + +def test_fetch_bad_status(src, type, requests_mock): + requests_mock.add( + responses.GET, + fetch_url("FXCADUSD"), + status=500, + body="""{"message": "Some other reason"}""", + ) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("CAD", "USD", type, "2021-01-01", "2021-01-07")) + assert "Internal Server Error" in str(e.value) + + +def test_fetch_parsing_error(src, type, requests_mock): + requests_mock.add(responses.GET, fetch_url("FXCADUSD"), body="NOT JSON") + with pytest.raises(exceptions.ResponseParsingError) as e: + src.fetch(Series("CAD", "USD", type, "2021-01-01", "2021-01-07")) + assert "while parsing data" in str(e.value) diff --git a/tests/pricehist/sources/test_bankofcanada/all-partial.json b/tests/pricehist/sources/test_bankofcanada/all-partial.json new file mode 100644 index 0000000..3604707 --- /dev/null +++ b/tests/pricehist/sources/test_bankofcanada/all-partial.json @@ -0,0 +1,101 @@ +{ + "terms": { + "url": "https://www.bankofcanada.ca/terms/" + }, + "seriesDetail": { + "FXCADUSD": { + "label": "CAD/USD", + "description": "Canadian dollar to US dollar daily exchange rate", + "dimension": { + "key": "d", + "name": "date" + } + } + }, + "observations": [ + { + "d": "2017-01-03", + "FXCADUSD": { + "v": "0.7443" + } + }, + { + "d": "2017-01-04", + "FXCADUSD": { + "v": "0.7510" + } + }, + { + "d": "2017-01-05", + "FXCADUSD": { + "v": "0.7551" + } + }, + { + "d": "2017-01-06", + "FXCADUSD": { + "v": "0.7568" + } + }, + { + "d": "2017-01-09", + "FXCADUSD": { + "v": "0.7553" + } + }, + { + "d": "2017-01-10", + "FXCADUSD": { + "v": "0.7568" + } + }, + { + "d": "2017-01-11", + "FXCADUSD": { + "v": "0.7547" + } + }, + { + "d": "2020-12-29", + "FXCADUSD": { + "v": "0.7809" + } + }, + { + "d": "2020-12-30", + "FXCADUSD": { + "v": "0.7831" + } + }, + { + "d": "2020-12-31", + "FXCADUSD": { + "v": "0.7854" + } + }, + { + "d": "2021-01-04", + "FXCADUSD": { + "v": "0.7843" + } + }, + { + "d": "2021-01-05", + "FXCADUSD": { + "v": "0.7870" + } + }, + { + "d": "2021-01-06", + "FXCADUSD": { + "v": "0.7883" + } + }, + { + "d": "2021-01-07", + "FXCADUSD": { + "v": "0.7870" + } + } + ] +} diff --git a/tests/pricehist/sources/test_bankofcanada/recent.json b/tests/pricehist/sources/test_bankofcanada/recent.json new file mode 100644 index 0000000..46ec248 --- /dev/null +++ b/tests/pricehist/sources/test_bankofcanada/recent.json @@ -0,0 +1,41 @@ +{ + "terms": { + "url": "https://www.bankofcanada.ca/terms/" + }, + "seriesDetail": { + "FXCADUSD": { + "label": "CAD/USD", + "description": "Canadian dollar to US dollar daily exchange rate", + "dimension": { + "key": "d", + "name": "date" + } + } + }, + "observations": [ + { + "d": "2021-01-04", + "FXCADUSD": { + "v": "0.7843" + } + }, + { + "d": "2021-01-05", + "FXCADUSD": { + "v": "0.7870" + } + }, + { + "d": "2021-01-06", + "FXCADUSD": { + "v": "0.7883" + } + }, + { + "d": "2021-01-07", + "FXCADUSD": { + "v": "0.7870" + } + } + ] +} diff --git a/tests/pricehist/sources/test_bankofcanada/series-partial.json b/tests/pricehist/sources/test_bankofcanada/series-partial.json new file mode 100644 index 0000000..68bfb4c --- /dev/null +++ b/tests/pricehist/sources/test_bankofcanada/series-partial.json @@ -0,0 +1,272 @@ +{ + "terms": { + "url": "https://www.bankofcanada.ca/terms/" + }, + "series": { + "FXAUDCAD": { + "label": "AUD/CAD", + "description": "Australian dollar to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXAUDCAD" + }, + "FXBRLCAD": { + "label": "BRL/CAD", + "description": "Brazilian real to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXBRLCAD" + }, + "FXCNYCAD": { + "label": "CNY/CAD", + "description": "Chinese renminbi to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCNYCAD" + }, + "FXEURCAD": { + "label": "EUR/CAD", + "description": "European euro to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXEURCAD" + }, + "FXHKDCAD": { + "label": "HKD/CAD", + "description": "Hong Kong dollar to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXHKDCAD" + }, + "FXINRCAD": { + "label": "INR/CAD", + "description": "Indian rupee to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXINRCAD" + }, + "FXIDRCAD": { + "label": "IDR/CAD", + "description": "Indonesian rupiah to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXIDRCAD" + }, + "FXJPYCAD": { + "label": "JPY/CAD", + "description": "Japanese yen to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXJPYCAD" + }, + "FXMYRCAD": { + "label": "MYR/CAD", + "description": "Malaysian ringgit to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXMYRCAD" + }, + "FXMXNCAD": { + "label": "MXN/CAD", + "description": "Mexican peso to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXMXNCAD" + }, + "FXNZDCAD": { + "label": "NZD/CAD", + "description": "New Zealand dollar to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXNZDCAD" + }, + "FXNOKCAD": { + "label": "NOK/CAD", + "description": "Norwegian krone to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXNOKCAD" + }, + "FXPENCAD": { + "label": "PEN/CAD", + "description": "Peruvian new sol to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXPENCAD" + }, + "FXRUBCAD": { + "label": "RUB/CAD", + "description": "Russian ruble to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXRUBCAD" + }, + "FXSARCAD": { + "label": "SAR/CAD", + "description": "Saudi riyal to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXSARCAD" + }, + "FXSGDCAD": { + "label": "SGD/CAD", + "description": "Singapore dollar to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXSGDCAD" + }, + "FXZARCAD": { + "label": "ZAR/CAD", + "description": "South African rand to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXZARCAD" + }, + "FXKRWCAD": { + "label": "KRW/CAD", + "description": "South Korean won to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXKRWCAD" + }, + "FXSEKCAD": { + "label": "SEK/CAD", + "description": "Swedish krona to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXSEKCAD" + }, + "FXCHFCAD": { + "label": "CHF/CAD", + "description": "Swiss franc to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCHFCAD" + }, + "FXTWDCAD": { + "label": "TWD/CAD", + "description": "Taiwanese dollar to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXTWDCAD" + }, + "FXTHBCAD": { + "label": "THB/CAD", + "description": "Thai baht to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXTHBCAD" + }, + "FXTRYCAD": { + "label": "TRY/CAD", + "description": "Turkish lira to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXTRYCAD" + }, + "FXGBPCAD": { + "label": "GBP/CAD", + "description": "UK pound sterling to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXGBPCAD" + }, + "FXUSDCAD": { + "label": "USD/CAD", + "description": "US dollar to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXUSDCAD" + }, + "FXVNDCAD": { + "label": "VND/CAD", + "description": "Vietnamese dong to Canadian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXVNDCAD" + }, + "FXCADAUD": { + "label": "CAD/AUD", + "description": "Canadian dollar to Australian dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADAUD" + }, + "FXCADBRL": { + "label": "CAD/BRL", + "description": "Canadian dollar to Brazilian real daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADBRL" + }, + "FXCADCNY": { + "label": "CAD/CNY", + "description": "Canadian dollar to Chinese renminbi daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADCNY" + }, + "FXCADEUR": { + "label": "CAD/EUR", + "description": "Canadian dollar to European euro daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADEUR" + }, + "FXCADHKD": { + "label": "CAD/HKD", + "description": "Canadian dollar to Hong Kong dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADHKD" + }, + "FXCADINR": { + "label": "CAD/INR", + "description": "Canadian dollar to Indian rupee daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADINR" + }, + "FXCADIDR": { + "label": "CAD/IDR", + "description": "Canadian dollar to Indonesian rupiah daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADIDR" + }, + "FXCADJPY": { + "label": "CAD/JPY", + "description": "Canadian dollar to Japanese yen daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADJPY" + }, + "FXCADMYR": { + "label": "CAD/MYR", + "description": "Canadian dollar to Malaysian ringgit daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADMYR" + }, + "FXCADMXN": { + "label": "CAD/MXN", + "description": "Canadian dollar to Mexican peso daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADMXN" + }, + "FXCADNZD": { + "label": "CAD/NZD", + "description": "Canadian dollar to New Zealand dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADNZD" + }, + "FXCADNOK": { + "label": "CAD/NOK", + "description": "Canadian dollar to Norwegian krone daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADNOK" + }, + "FXCADPEN": { + "label": "CAD/PEN", + "description": "Canadian dollar to Peruvian new sol daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADPEN" + }, + "FXCADRUB": { + "label": "CAD/RUB", + "description": "Canadian dollar to Russian ruble daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADRUB" + }, + "FXCADSAR": { + "label": "CAD/SAR", + "description": "Canadian dollar to Saudi riyal daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADSAR" + }, + "FXCADSGD": { + "label": "CAD/SGD", + "description": "Canadian dollar to Singapore dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADSGD" + }, + "FXCADZAR": { + "label": "CAD/ZAR", + "description": "Canadian dollar to South African rand daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADZAR" + }, + "FXCADKRW": { + "label": "CAD/KRW", + "description": "Canadian dollar to South Korean won daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADKRW" + }, + "FXCADSEK": { + "label": "CAD/SEK", + "description": "Canadian dollar to Swedish krona daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADSEK" + }, + "FXCADCHF": { + "label": "CAD/CHF", + "description": "Canadian dollar to Swiss franc daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADCHF" + }, + "FXCADTWD": { + "label": "CAD/TWD", + "description": "Canadian dollar to Taiwanese dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADTWD" + }, + "FXCADTHB": { + "label": "CAD/THB", + "description": "Canadian dollar to Thai baht daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADTHB" + }, + "FXCADTRY": { + "label": "CAD/TRY", + "description": "Canadian dollar to Turkish lira daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADTRY" + }, + "FXCADGBP": { + "label": "CAD/GBP", + "description": "Canadian dollar to UK pound sterling daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADGBP" + }, + "FXCADUSD": { + "label": "CAD/USD", + "description": "Canadian dollar to US dollar daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADUSD" + }, + "FXCADVND": { + "label": "CAD/VND", + "description": "Canadian dollar to Vietnamese dong daily exchange rate", + "link": "https://www.bankofcanada.ca/valet/series/FXCADVND" + }, + "INDINF_GRACE_Q": { + "label": "Foreign demand for Canadian non-commodity exports (GRACE) (2007=100)", + "description": "Foreign demand for Canadian non-commodity exports (GRACE) (2007=100)", + "link": "https://www.bankofcanada.ca/valet/series/INDINF_GRACE_Q" + } + } +} From 2787c212d23e73db2973843a6f367df057ebcc71 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 27 Dec 2021 11:25:55 +1100 Subject: [PATCH 080/149] Fix AlphaVantage to handle adjusted endpoint being premium. --- src/pricehist/exceptions.py | 4 +- src/pricehist/sources/alphavantage.py | 19 ++++- tests/live.sh | 12 +-- tests/pricehist/sources/test_alphavantage.py | 39 ++++++++- .../test_alphavantage/ibm-partial-adj.json | 81 +++++++++++++++++++ .../test_alphavantage/ibm-partial.json | 42 ++-------- 6 files changed, 149 insertions(+), 48 deletions(-) create mode 100644 tests/pricehist/sources/test_alphavantage/ibm-partial-adj.json diff --git a/src/pricehist/exceptions.py b/src/pricehist/exceptions.py index 5ac7aa7..e207537 100644 --- a/src/pricehist/exceptions.py +++ b/src/pricehist/exceptions.py @@ -52,7 +52,7 @@ class InvalidType(SourceError, ValueError): class CredentialsError(SourceError): """Access credentials are unavailable or invalid.""" - def __init__(self, keys, source): + def __init__(self, keys, source, msg=""): self.keys = keys self.source = source message = ( @@ -61,6 +61,8 @@ class CredentialsError(SourceError): f"correctly. Run 'pricehist source {source.id()}' for more " f"information about credentials." ) + if msg: + message += f" {msg}" super(CredentialsError, self).__init__(message) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index 75cda5d..84df4d9 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -51,7 +51,8 @@ class AlphaVantage(BaseSource): "will list all digital and physical currency symbols.\n" "The PAIR for stocks is the stock symbol only. The quote currency " f"will be determined automatically. {self._stock_symbols_message()}\n" - "The price type 'adjclose' is only available for stocks.\n" + "The price type 'adjclose' is only available for stocks, and " + "requires an access key for which premium endpoints are unlocked.\n" "Beware that digital currencies quoted in non-USD currencies may " "be converted from USD data at one recent exchange rate rather " "than using historical rates.\n" @@ -186,8 +187,13 @@ class AlphaVantage(BaseSource): def _stock_data(self, series): output_quote = self._stock_currency(series.base) or "UNKNOWN" + if series.type == "adjclose": + function = "TIME_SERIES_DAILY_ADJUSTED" + else: + function = "TIME_SERIES_DAILY" + params = { - "function": "TIME_SERIES_DAILY_ADJUSTED", + "function": function, "symbol": series.base, "outputsize": self._outputsize(series.start), "apikey": self._apikey(), @@ -225,7 +231,8 @@ class AlphaVantage(BaseSource): "high": entries["2. high"], "low": entries["3. low"], "close": entries["4. close"], - "adjclose": entries["5. adjusted close"], + "adjclose": "5. adjusted close" in entries + and entries["5. adjusted close"], } for day, entries in reversed(data["Time Series (Daily)"].items()) } @@ -332,6 +339,12 @@ class AlphaVantage(BaseSource): if type(data) == dict: if "Note" in data and "call frequency" in data["Note"]: raise exceptions.RateLimit(data["Note"]) + if ( + "Information" in data + and "ways to unlock premium" in data["Information"] + ): + msg = "You were denied access to a premium endpoint." + raise exceptions.CredentialsError([self.API_KEY_NAME], self, msg) if "Error Message" in data and "apikey " in data["Error Message"]: raise exceptions.CredentialsError([self.API_KEY_NAME], self) diff --git a/tests/live.sh b/tests/live.sh index 543d634..33e66c3 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -47,11 +47,11 @@ name="Alpha Vantage stocks" cmd="pricehist fetch alphavantage TSLA -s 2021-01-04 -e 2021-01-08" read -r -d '' expected < Date: Mon, 27 Dec 2021 11:26:37 +1100 Subject: [PATCH 081/149] Remove old note. --- tests/pricehist/sources/test_alphavantage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pricehist/sources/test_alphavantage.py b/tests/pricehist/sources/test_alphavantage.py index 615f4fb..8ff70cc 100644 --- a/tests/pricehist/sources/test_alphavantage.py +++ b/tests/pricehist/sources/test_alphavantage.py @@ -426,7 +426,6 @@ def test_fetch_stock_rate_limit(src, type, search_ok, requests_mock): assert "rate limit" in str(e.value) -# TODO def test_fetch_stock_premium(src, search_ok, requests_mock): requests_mock.add(responses.GET, adj_stock_url, body=premium_json) with pytest.raises(exceptions.CredentialsError) as e: From 486d4097d7868d3921d0c6c64e5d2920a49c8065 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 27 Dec 2021 11:30:15 +1100 Subject: [PATCH 082/149] Version 1.3.0. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8a1d3b9..dbe409d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.2.5" +version = "1.3.0" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index b7e1990..67bc602 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.2.5" +__version__ = "1.3.0" From 66c9f42ef8b330d3e2114adcd7ddc1aae446da87 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 20 Jan 2022 11:51:38 +1100 Subject: [PATCH 083/149] Add hits badge to README. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index cdb9bd9..ff8c521 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ support for multiple data sources and output formats. [![Downloads](https://pepy.tech/badge/pricehist)](https://pepy.tech/project/pricehist) [![License](https://img.shields.io/pypi/l/pricehist)](https://gitlab.com/chrisberkhout/pricehist/-/blob/master/LICENSE) [![Code style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Hits](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fgitlab.com%2Fchrisberkhout%2Fpricehist&count_bg=%2379C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=hits&edge_flat=false)](https://hits.seeyoufarm.com) ## Installation From 5f2b96a5bbd23cbc45607580fe8a27185a7c7e95 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 24 Jan 2022 19:17:39 +1100 Subject: [PATCH 084/149] Add reactions to README. --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index ff8c521..b72158b 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,14 @@ pipx install pricehist - **`gnucash-sql`**: [GnuCash](https://www.gnucash.org/) SQL - **`ledger`**: [Ledger](https://www.ledger-cli.org/) and [hledger](https://hledger.org/) +## Reactions + +> This is my new favourite price fetcher, by far. +> -- _Simon Michael, creator of [hledger](https://hledger.org/) ([ref](https://groups.google.com/g/hledger/c/SCLbNiKl9D8/m/0ReYmDppAAAJ))_ + +> This is great! +> -- _Martin Blais, creator of [Beancount](https://beancount.github.io/) ([ref](https://groups.google.com/g/beancount/c/cCJc9OhIlNg/m/QGRvNowcAwAJ))_ + ## How to ### Fetch prices From aceb0f09d1878fa4423e096b79d141d009df55ad Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 4 Apr 2022 12:48:39 +0200 Subject: [PATCH 085/149] Minor doc fixes. --- README.md | 4 ++-- src/pricehist/sources/alphavantage.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b72158b..1e7f35a 100644 --- a/README.md +++ b/README.md @@ -393,8 +393,8 @@ pricehist fetch coindesk BTC/USD --type close - **`close`** is the price type for the last price of each day. A BTC/USD price of the amount 29,391.775 can be written as -"BTC/USD = 29391.78" or "BTC 29391.78 USD", and means that one Bitcoin is -worth 29,391.78 United States Dollars. +"BTC/USD = 29391.775" or "BTC 29391.775 USD", and means that one Bitcoin is +worth 29,391.775 United States Dollars. ## Initial design choices diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index 84df4d9..707ba50 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -45,7 +45,7 @@ class AlphaVantage(BaseSource): "You should obtain a free API key from " "https://www.alphavantage.co/support/#api-key and set it in " f"the {self.API_KEY_NAME} environment variable ({keystatus}), " - "otherise, pricehist will attempt to use a generic key.\n" + "otherwise, pricehist will attempt to use a generic key.\n" "The PAIR for currencies should be in BASE/QUOTE form. The quote " "symbol must always be for a physical currency. The --symbols option " "will list all digital and physical currency symbols.\n" @@ -58,8 +58,8 @@ class AlphaVantage(BaseSource): "than using historical rates.\n" "Alpha Vantage's standard API call frequency limits is 5 calls per " "minute and 500 per day, so you may need to pause between successive " - "commands. Note that retrieving prices for one stock requires two " - "calls." + "commands. Note that retrieving prices for one stock consumes two " + "API calls." ) def _stock_symbols_message(self): From 7a9d3d3e8f2cc13b144629e7bc3d49bf980c01d0 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 4 Apr 2022 13:04:26 +0200 Subject: [PATCH 086/149] Update ISO 4217 currency data for ISO 4217 amendment number 171. --- Makefile | 7 +++++++ src/pricehist/isocurrencies.py | 5 +---- src/pricehist/resources/list_one.xml | 9 ++++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 83c4c2c..eb13631 100644 --- a/Makefile +++ b/Makefile @@ -39,3 +39,10 @@ pre-commit: ## Checks to run before each commit .PHONY: tox tox: ## Run tests via tox poetry run tox + +.PHONY: fetch-iso-data +fetch-iso-data: ## Fetch the latest copy of the ISO 4217 currency data + wget -O src/pricehist/resources/list_one.xml \ + https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list_one.xml + wget -O src/pricehist/resources/list_three.xml \ + https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list_three.xml diff --git a/src/pricehist/isocurrencies.py b/src/pricehist/isocurrencies.py index 550db3a..fe02678 100644 --- a/src/pricehist/isocurrencies.py +++ b/src/pricehist/isocurrencies.py @@ -24,7 +24,6 @@ Functions: """ from dataclasses import dataclass, field -from datetime import datetime from importlib.resources import read_binary from typing import List @@ -45,9 +44,7 @@ class ISOCurrency: def current_data_date(): one = etree.fromstring(read_binary("pricehist.resources", "list_one.xml")) - pblshd = one.cssselect("ISO_4217")[0].attrib["Pblshd"] - date = datetime.strptime(pblshd, "%B %d, %Y").date().isoformat() - return date + return one.cssselect("ISO_4217")[0].attrib["Pblshd"] def historical_data_date(): diff --git a/src/pricehist/resources/list_one.xml b/src/pricehist/resources/list_one.xml index 7bc46c2..0e8dd7a 100644 --- a/src/pricehist/resources/list_one.xml +++ b/src/pricehist/resources/list_one.xml @@ -1,5 +1,5 @@ - + AFGHANISTAN @@ -1493,6 +1493,13 @@ 694 2 + + SIERRA LEONE + Leone + SLE + 925 + 2 + SINGAPORE Singapore Dollar From 46ebdfe074eb410968764f39b38b10ec145d3d13 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 4 Apr 2022 13:40:04 +0200 Subject: [PATCH 087/149] Add JSON and JSONL output formats. --- README.md | 8 +++-- src/pricehist/outputs/__init__.py | 3 ++ src/pricehist/outputs/json.py | 54 +++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 src/pricehist/outputs/json.py diff --git a/README.md b/README.md index 1e7f35a..05f6492 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ pipx install pricehist - **`beancount`**: [Beancount](http://furius.ca/beancount/) - **`csv`**: [Comma-separated values](https://en.wikipedia.org/wiki/Comma-separated_values) +- **`json`**: [JSON](https://en.wikipedia.org/wiki/JSON) +- **`jsonl`**: [JSON lines](https://en.wikipedia.org/wiki/JSON_streaming) - **`gnucash-sql`**: [GnuCash](https://www.gnucash.org/) SQL - **`ledger`**: [Ledger](https://www.ledger-cli.org/) and [hledger](https://hledger.org/) @@ -91,7 +93,7 @@ pricehist fetch -h ``` ``` usage: pricehist fetch SOURCE PAIR [-h] [-vvv] [-t TYPE] [-s DATE | -sx DATE] [-e DATE | -ex DATE] -[-o beancount|csv|gnucash-sql|ledger] [--invert] [--quantize INT] +[-o beancount|csv|json|jsonl|gnucash-sql|ledger] [--invert] [--quantize INT] [--fmt-base SYM] [--fmt-quote SYM] [--fmt-time TIME] [--fmt-decimal CHAR] [--fmt-thousands CHAR] [--fmt-symbol rightspace|right|leftspace|left] [--fmt-datesep CHAR] [--fmt-csvdelim CHAR] @@ -122,8 +124,8 @@ optional arguments: ### Choose and customize the output format -As the output format you can choose one of `beancount`, `csv`, `ledger` or -`gnucash-sql`. +As the output format you can choose one of `beancount`, `csv`, `json`, `jsonl`, +`ledger` or `gnucash-sql`. ``` pricehist fetch ecb EUR/AUD -s 2021-01-04 -e 2021-01-08 -o ledger diff --git a/src/pricehist/outputs/__init__.py b/src/pricehist/outputs/__init__.py index 98e9547..4b015c8 100644 --- a/src/pricehist/outputs/__init__.py +++ b/src/pricehist/outputs/__init__.py @@ -1,6 +1,7 @@ from .beancount import Beancount from .csv import CSV from .gnucashsql import GnuCashSQL +from .json import JSON from .ledger import Ledger default = "csv" @@ -8,6 +9,8 @@ default = "csv" by_type = { "beancount": Beancount(), "csv": CSV(), + "json": JSON(), + "jsonl": JSON(jsonl=True), "gnucash-sql": GnuCashSQL(), "ledger": Ledger(), } diff --git a/src/pricehist/outputs/json.py b/src/pricehist/outputs/json.py new file mode 100644 index 0000000..7e0f674 --- /dev/null +++ b/src/pricehist/outputs/json.py @@ -0,0 +1,54 @@ +""" +JSON output + +Date, number and base/quote formatting options will be respected. + +Classes: + + JSON + +""" + +import io +import json + +from pricehist.format import Format + +from .baseoutput import BaseOutput + + +class JSON(BaseOutput): + def __init__(self, jsonl=False): + self.jsonl = jsonl + + def format(self, series, source, fmt=Format()): + data = [] + output = io.StringIO() + + base = fmt.base or series.base + quote = fmt.quote or series.quote + + for price in series.prices: + date = fmt.format_date(price.date) + amount = fmt.format_num(price.amount) + + data.append( + { + "date": date, + "base": base, + "quote": quote, + "amount": amount, + "source": source.id(), + "type": series.type, + } + ) + + if self.jsonl: + for row in data: + json.dump(row, output) + output.write("\n") + else: + json.dump(data, output, indent=2) + output.write("\n") + + return output.getvalue() From 99aeb6bbc7a747e183543b8526cdeffdffdcab10 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 4 Apr 2022 15:33:40 +0200 Subject: [PATCH 088/149] Test json and jsonl output formats. --- src/pricehist/outputs/json.py | 4 +- tests/pricehist/outputs/test_json.py | 127 +++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 tests/pricehist/outputs/test_json.py diff --git a/src/pricehist/outputs/json.py b/src/pricehist/outputs/json.py index 7e0f674..9deb90e 100644 --- a/src/pricehist/outputs/json.py +++ b/src/pricehist/outputs/json.py @@ -45,10 +45,10 @@ class JSON(BaseOutput): if self.jsonl: for row in data: - json.dump(row, output) + json.dump(row, output, ensure_ascii=False) output.write("\n") else: - json.dump(data, output, indent=2) + json.dump(data, output, ensure_ascii=False, indent=2) output.write("\n") return output.getvalue() diff --git a/tests/pricehist/outputs/test_json.py b/tests/pricehist/outputs/test_json.py new file mode 100644 index 0000000..608007e --- /dev/null +++ b/tests/pricehist/outputs/test_json.py @@ -0,0 +1,127 @@ +from decimal import Decimal +from textwrap import dedent + +import pytest + +from pricehist.format import Format +from pricehist.outputs.json import JSON +from pricehist.price import Price +from pricehist.series import Series + + +@pytest.fixture +def json_out(): + return JSON() + + +@pytest.fixture +def jsonl_out(): + return JSON(jsonl=True) + + +@pytest.fixture +def series(): + prices = [ + Price("2021-01-01", Decimal("24139.4648")), + Price("2021-01-02", Decimal("26533.576")), + Price("2021-01-03", Decimal("27001.2846")), + ] + return Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03", prices) + + +def test_format_basics(json_out, series, mocker): + source = mocker.MagicMock() + source.id = mocker.MagicMock(return_value="sourceid") + result = json_out.format(series, source, Format()) + assert ( + result + == dedent( + """ + [ + { + "date": "2021-01-01", + "base": "BTC", + "quote": "EUR", + "amount": "24139.4648", + "source": "sourceid", + "type": "close" + }, + { + "date": "2021-01-02", + "base": "BTC", + "quote": "EUR", + "amount": "26533.576", + "source": "sourceid", + "type": "close" + }, + { + "date": "2021-01-03", + "base": "BTC", + "quote": "EUR", + "amount": "27001.2846", + "source": "sourceid", + "type": "close" + } + ] + """ + ).strip() + + "\n" + ) + + +def test_format_basic_jsonl(jsonl_out, series, mocker): + source = mocker.MagicMock() + source.id = mocker.MagicMock(return_value="sourceid") + result = jsonl_out.format(series, source, Format()) + assert ( + result + == dedent( + """ + {"date": "2021-01-01", "base": "BTC", "quote": "EUR", "amount": "24139.4648", "source": "sourceid", "type": "close"} + {"date": "2021-01-02", "base": "BTC", "quote": "EUR", "amount": "26533.576", "source": "sourceid", "type": "close"} + {"date": "2021-01-03", "base": "BTC", "quote": "EUR", "amount": "27001.2846", "source": "sourceid", "type": "close"} + """ # noqa + ).strip() + + "\n" + ) + + +def test_format_custom(json_out, series, mocker): + source = mocker.MagicMock() + source.id = mocker.MagicMock(return_value="sourceid") + fmt = Format(base="XBT", quote="€", thousands=".", decimal=",", datesep="/") + result = json_out.format(series, source, fmt) + assert ( + result + == dedent( + """ + [ + { + "date": "2021/01/01", + "base": "XBT", + "quote": "€", + "amount": "24.139,4648", + "source": "sourceid", + "type": "close" + }, + { + "date": "2021/01/02", + "base": "XBT", + "quote": "€", + "amount": "26.533,576", + "source": "sourceid", + "type": "close" + }, + { + "date": "2021/01/03", + "base": "XBT", + "quote": "€", + "amount": "27.001,2846", + "source": "sourceid", + "type": "close" + } + ] + """ + ).strip() + + "\n" + ) From dace604129418ff0ea8a91b6f16cd82b7c1c1450 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 4 Apr 2022 15:52:43 +0200 Subject: [PATCH 089/149] Note about fetching new prices only. --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 05f6492..ca74284 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,17 @@ P 2021/01/07 € $1.5836 P 2021/01/08 € $1.5758 ``` +### Fetch new prices only + +You can update an existing file without refetching the prices you already have. +First find the date of the last price, then fetch from there, drop the header +line if present and append the rest to the existing file. + +``` +last=$(tail -1 prices-eur-usd.csv | cut -d, -f1) +pricehist fetch ecb EUR/USD -sx $last -o csv | sed 1d >> prices-eur-usd.csv +``` + ### Load prices into GnuCash You can generate SQL for a GnuCash database and apply it immediately with one From bbf33df65742a52db46e97957cb9fd308d852da9 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 4 Apr 2022 15:57:33 +0200 Subject: [PATCH 090/149] Version 1.4.0. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dbe409d..6dccab0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.3.0" +version = "1.4.0" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 67bc602..3e8d9f9 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.3.0" +__version__ = "1.4.0" From 3f65a21ffd4bd6968db06563e1c8132929e04da2 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 4 Apr 2022 16:29:03 +0200 Subject: [PATCH 091/149] Turn off logging by charset_normalizer (which may be used by requests). --- src/pricehist/logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pricehist/logger.py b/src/pricehist/logger.py index 3a6143f..d9bdf58 100644 --- a/src/pricehist/logger.py +++ b/src/pricehist/logger.py @@ -23,6 +23,7 @@ def init(): handler.setFormatter(Formatter()) logging.root.addHandler(handler) logging.root.setLevel(logging.INFO) + logging.getLogger("charset_normalizer").disabled = True def show_debug(): From a3e19f9bcf2c283d899d780cbc118de296fb8625 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 4 Apr 2022 16:30:07 +0200 Subject: [PATCH 092/149] Version 1.4.1. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6dccab0..ffa4f47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.0" +version = "1.4.1" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 3e8d9f9..bf25615 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.0" +__version__ = "1.4.1" From aabce7fe6fd171e9e5a20e62ea9fdbcd0833009b Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 4 Apr 2022 17:01:09 +0200 Subject: [PATCH 093/149] Add --fmt-jsonnums option. --- README.md | 4 ++- src/pricehist/cli.py | 8 +++++- src/pricehist/format.py | 2 ++ src/pricehist/outputs/json.py | 5 +++- tests/pricehist/outputs/test_json.py | 41 ++++++++++++++++++++++++++++ tests/pricehist/test_format.py | 1 + 6 files changed, 58 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ca74284..1b6b04c 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,8 @@ pricehist fetch -h usage: pricehist fetch SOURCE PAIR [-h] [-vvv] [-t TYPE] [-s DATE | -sx DATE] [-e DATE | -ex DATE] [-o beancount|csv|json|jsonl|gnucash-sql|ledger] [--invert] [--quantize INT] [--fmt-base SYM] [--fmt-quote SYM] [--fmt-time TIME] [--fmt-decimal CHAR] [--fmt-thousands CHAR] -[--fmt-symbol rightspace|right|leftspace|left] [--fmt-datesep CHAR] [--fmt-csvdelim CHAR] +[--fmt-symbol rightspace|right|leftspace|left] [--fmt-datesep CHAR] +[--fmt-csvdelim CHAR] [--fmt-jsonnums] positional arguments: SOURCE the source identifier @@ -120,6 +121,7 @@ optional arguments: --fmt-symbol LOCATION commodity symbol placement in output (default: rightspace) --fmt-datesep CHAR date separator in output (default: '-') --fmt-csvdelim CHAR field delimiter for CSV output (default: ',') + --fmt-jsonnums numbers not strings for JSON output (default: False) ``` ### Choose and customize the output format diff --git a/src/pricehist/cli.py b/src/pricehist/cli.py index d676856..9e4a69d 100644 --- a/src/pricehist/cli.py +++ b/src/pricehist/cli.py @@ -205,7 +205,7 @@ def build_parser(): "[--fmt-base SYM] [--fmt-quote SYM] [--fmt-time TIME] " "[--fmt-decimal CHAR] [--fmt-thousands CHAR] " "[--fmt-symbol rightspace|right|leftspace|left] [--fmt-datesep CHAR] " - "[--fmt-csvdelim CHAR]" + "[--fmt-csvdelim CHAR] [--fmt-jsonnums]" ), formatter_class=formatter, ) @@ -353,5 +353,11 @@ def build_parser(): type=valid_char, help=f"field delimiter for CSV output (default: '{default_fmt.csvdelim}')", ) + fetch_parser.add_argument( + "--fmt-jsonnums", + dest="formatjsonnums", + action="store_true", + help=f"numbers not strings for JSON output (default: {default_fmt.jsonnums})", + ) return parser diff --git a/src/pricehist/format.py b/src/pricehist/format.py index fb7a8de..14207e7 100644 --- a/src/pricehist/format.py +++ b/src/pricehist/format.py @@ -11,6 +11,7 @@ class Format: symbol: str = "rightspace" datesep: str = "-" csvdelim: str = "," + jsonnums: bool = False @classmethod def fromargs(cls, args): @@ -27,6 +28,7 @@ class Format: symbol=if_not_none(args.formatsymbol, default.symbol), datesep=if_not_none(args.formatdatesep, default.datesep), csvdelim=if_not_none(args.formatcsvdelim, default.csvdelim), + jsonnums=if_not_none(args.formatjsonnums, default.jsonnums), ) def format_date(self, date): diff --git a/src/pricehist/outputs/json.py b/src/pricehist/outputs/json.py index 9deb90e..8983e8e 100644 --- a/src/pricehist/outputs/json.py +++ b/src/pricehist/outputs/json.py @@ -30,7 +30,10 @@ class JSON(BaseOutput): for price in series.prices: date = fmt.format_date(price.date) - amount = fmt.format_num(price.amount) + if fmt.jsonnums: + amount = float(price.amount) + else: + amount = fmt.format_num(price.amount) data.append( { diff --git a/tests/pricehist/outputs/test_json.py b/tests/pricehist/outputs/test_json.py index 608007e..4b3d3fd 100644 --- a/tests/pricehist/outputs/test_json.py +++ b/tests/pricehist/outputs/test_json.py @@ -125,3 +125,44 @@ def test_format_custom(json_out, series, mocker): ).strip() + "\n" ) + + +def test_format_numbers(json_out, series, mocker): + source = mocker.MagicMock() + source.id = mocker.MagicMock(return_value="sourceid") + fmt = Format(jsonnums=True) + result = json_out.format(series, source, fmt) + assert ( + result + == dedent( + """ + [ + { + "date": "2021-01-01", + "base": "BTC", + "quote": "EUR", + "amount": 24139.4648, + "source": "sourceid", + "type": "close" + }, + { + "date": "2021-01-02", + "base": "BTC", + "quote": "EUR", + "amount": 26533.576, + "source": "sourceid", + "type": "close" + }, + { + "date": "2021-01-03", + "base": "BTC", + "quote": "EUR", + "amount": 27001.2846, + "source": "sourceid", + "type": "close" + } + ] + """ + ).strip() + + "\n" + ) diff --git a/tests/pricehist/test_format.py b/tests/pricehist/test_format.py index 2795037..e816850 100644 --- a/tests/pricehist/test_format.py +++ b/tests/pricehist/test_format.py @@ -14,6 +14,7 @@ def test_fromargs(): "formatdatesep": None, "formatcsvdelim": None, "formatbase": None, + "formatjsonnums": None, } args = namedtuple("args", arg_values.keys())(**arg_values) fmt = Format.fromargs(args) From a54da85a6fae15e2f771e8612aed089407ec5c22 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 4 Apr 2022 17:02:08 +0200 Subject: [PATCH 094/149] Version 1.4.2. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ffa4f47..fcbf936 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.1" +version = "1.4.2" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index bf25615..daa50c7 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.1" +__version__ = "1.4.2" From 765e2ec77de97f87abc782943c5cdf086d293838 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 23 Sep 2022 17:05:54 +0200 Subject: [PATCH 095/149] Rename ISO 4217 data files to match SIX Group's new naming. --- Makefile | 8 ++++---- src/pricehist/isocurrencies.py | 12 ++++++------ .../resources/{list_one.xml => list-one.xml} | 0 .../resources/{list_three.xml => list-three.xml} | 0 4 files changed, 10 insertions(+), 10 deletions(-) rename src/pricehist/resources/{list_one.xml => list-one.xml} (100%) rename src/pricehist/resources/{list_three.xml => list-three.xml} (100%) diff --git a/Makefile b/Makefile index eb13631..49bbeb9 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ tox: ## Run tests via tox .PHONY: fetch-iso-data fetch-iso-data: ## Fetch the latest copy of the ISO 4217 currency data - wget -O src/pricehist/resources/list_one.xml \ - https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list_one.xml - wget -O src/pricehist/resources/list_three.xml \ - https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list_three.xml + wget -O src/pricehist/resources/list-one.xml \ + https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list-one.xml + wget -O src/pricehist/resources/list-three.xml \ + https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list-three.xml diff --git a/src/pricehist/isocurrencies.py b/src/pricehist/isocurrencies.py index fe02678..bd453dc 100644 --- a/src/pricehist/isocurrencies.py +++ b/src/pricehist/isocurrencies.py @@ -8,8 +8,8 @@ currencies are included and countries with no universal currency are ignored. The data is read from vendored copies of the XML files published by the maintainers of the standard: -* :file:`list_one.xml` (current currencies & funds) -* :file:`list_three.xml` (historical currencies & funds) +* :file:`list-one.xml` (current currencies & funds) +* :file:`list-three.xml` (historical currencies & funds) Classes: @@ -43,20 +43,20 @@ class ISOCurrency: def current_data_date(): - one = etree.fromstring(read_binary("pricehist.resources", "list_one.xml")) + one = etree.fromstring(read_binary("pricehist.resources", "list-one.xml")) return one.cssselect("ISO_4217")[0].attrib["Pblshd"] def historical_data_date(): - three = etree.fromstring(read_binary("pricehist.resources", "list_three.xml")) + three = etree.fromstring(read_binary("pricehist.resources", "list-three.xml")) return three.cssselect("ISO_4217")[0].attrib["Pblshd"] def by_code(): result = {} - one = etree.fromstring(read_binary("pricehist.resources", "list_one.xml")) - three = etree.fromstring(read_binary("pricehist.resources", "list_three.xml")) + one = etree.fromstring(read_binary("pricehist.resources", "list-one.xml")) + three = etree.fromstring(read_binary("pricehist.resources", "list-three.xml")) for entry in three.cssselect("HstrcCcyNtry") + one.cssselect("CcyNtry"): if currency := _parse(entry): diff --git a/src/pricehist/resources/list_one.xml b/src/pricehist/resources/list-one.xml similarity index 100% rename from src/pricehist/resources/list_one.xml rename to src/pricehist/resources/list-one.xml diff --git a/src/pricehist/resources/list_three.xml b/src/pricehist/resources/list-three.xml similarity index 100% rename from src/pricehist/resources/list_three.xml rename to src/pricehist/resources/list-three.xml From 42d969a3ba2d267a777724c539d0d48d0a322d32 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 23 Sep 2022 17:12:53 +0200 Subject: [PATCH 096/149] ISO data update. --- src/pricehist/resources/list-one.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pricehist/resources/list-one.xml b/src/pricehist/resources/list-one.xml index 0e8dd7a..031048f 100644 --- a/src/pricehist/resources/list-one.xml +++ b/src/pricehist/resources/list-one.xml @@ -1,5 +1,5 @@ - + AFGHANISTAN @@ -1708,7 +1708,7 @@ 3 - TURKEY + TÜRKİYE Turkish Lira TRY 949 From 2d2b4b1e02c2082c8c9751e4768bdae24e295076 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 23 Sep 2022 17:25:47 +0200 Subject: [PATCH 097/149] Update expected TSLA prices in live test. --- tests/live.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/live.sh b/tests/live.sh index 33e66c3..6b391e7 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -144,11 +144,11 @@ name="Yahoo! Finance" cmd="pricehist fetch yahoo TSLA -s 2021-01-04 -e 2021-01-08" read -r -d '' expected < Date: Thu, 24 Nov 2022 13:50:19 +0100 Subject: [PATCH 098/149] Update dev dependency black. --- poetry.lock | 515 ++++++++++++++++++++++--------------------------- pyproject.toml | 2 +- 2 files changed, 231 insertions(+), 286 deletions(-) diff --git a/poetry.lock b/poetry.lock index e71f93b..9df92df 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,14 +1,6 @@ -[[package]] -name = "appdirs" -version = "1.4.4" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "atomicwrites" -version = "1.4.0" +version = "1.4.1" description = "Atomic file writes." category = "dev" optional = false @@ -16,89 +8,77 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "21.2.0" +version = "22.1.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] - -[[package]] -name = "backports.entry-points-selectable" -version = "1.1.0" -description = "Compatibility shim providing selectable entry points for older implementations" -category = "dev" -optional = false -python-versions = ">=2.7" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-flake8", "pytest-cov", "pytest-black (>=0.3.7)", "pytest-mypy", "pytest-checkdocs (>=2.4)", "pytest-enabler (>=1.0.1)"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "black" -version = "20.8b1" +version = "22.10.0" description = "The uncompromising code formatter." category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] -appdirs = "*" -click = ">=7.1.2" +click = ">=8.0.0" mypy-extensions = ">=0.4.3" -pathspec = ">=0.6,<1" -regex = ">=2020.1.8" -toml = ">=0.10.1" -typed-ast = ">=1.4.0" -typing-extensions = ">=3.7.4" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "certifi" -version = "2021.5.30" +version = "2022.9.24" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "charset-normalizer" -version = "2.0.6" +version = "2.1.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=3.5.0" +python-versions = ">=3.6.0" [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "click" -version = "8.0.1" +version = "8.1.3" description = "Composable command line interface toolkit" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.6" description = "Cross-platform colored terminal text." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" [[package]] name = "coverage" @@ -113,11 +93,11 @@ toml = ["toml"] [[package]] name = "cssselect" -version = "1.1.0" +version = "1.2.0" description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.7" [[package]] name = "curlify" @@ -132,7 +112,7 @@ requests = "*" [[package]] name = "distlib" -version = "0.3.3" +version = "0.3.6" description = "Distribution utilities" category = "dev" optional = false @@ -140,15 +120,15 @@ python-versions = "*" [[package]] name = "filelock" -version = "3.3.0" +version = "3.8.0" description = "A platform independent file lock." category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] -testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] +docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] +testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] [[package]] name = "flake8" @@ -165,7 +145,7 @@ pyflakes = ">=2.3.0,<2.4.0" [[package]] name = "idna" -version = "3.2" +version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false @@ -181,21 +161,21 @@ python-versions = "*" [[package]] name = "isort" -version = "5.9.3" +version = "5.10.1" description = "A Python utility / library to sort Python imports." category = "dev" optional = false python-versions = ">=3.6.1,<4.0" [package.extras] -pipfile_deprecated_finder = ["pipreqs", "requirementslib"] -requirements_deprecated_finder = ["pipreqs", "pip-api"] colors = ["colorama (>=0.4.3,<0.5.0)"] +pipfile-deprecated-finder = ["pipreqs", "requirementslib"] plugins = ["setuptools"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] [[package]] name = "lxml" -version = "4.6.3" +version = "4.9.1" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." category = "main" optional = false @@ -204,7 +184,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" [package.extras] cssselect = ["cssselect (>=0.7)"] html5 = ["html5lib"] -htmlsoup = ["beautifulsoup4"] +htmlsoup = ["BeautifulSoup4"] source = ["Cython (>=0.29.7)"] [[package]] @@ -225,34 +205,34 @@ python-versions = "*" [[package]] name = "packaging" -version = "21.0" +version = "21.3" description = "Core utilities for Python packages" category = "dev" optional = false python-versions = ">=3.6" [package.dependencies] -pyparsing = ">=2.0.2" +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "pathspec" -version = "0.9.0" +version = "0.10.2" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.7" [[package]] name = "platformdirs" -version = "2.4.0" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +version = "2.5.4" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] -docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] -test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] +docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] +test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" @@ -268,11 +248,11 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "py" -version = "1.10.0" +version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pycodestyle" @@ -292,11 +272,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pyparsing" -version = "2.4.7" -description = "Python parsing module" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" category = "dev" optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = ">=3.6.8" + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" @@ -321,43 +304,35 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm [[package]] name = "pytest-mock" -version = "3.6.1" +version = "3.10.0" description = "Thin-wrapper around the mock package for easier use with pytest" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] pytest = ">=5.0" [package.extras] -dev = ["pre-commit", "tox", "pytest-asyncio"] - -[[package]] -name = "regex" -version = "2021.9.30" -description = "Alternative regular expression module, to replace re." -category = "dev" -optional = false -python-versions = "*" +dev = ["pre-commit", "pytest-asyncio", "tox"] [[package]] name = "requests" -version = "2.26.0" +version = "2.28.1" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7, <4" [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} -idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "responses" @@ -373,7 +348,7 @@ six = "*" urllib3 = ">=1.25.10" [package.extras] -tests = ["coverage (>=3.7.1,<6.0.0)", "pytest-cov", "pytest-localserver", "flake8", "types-mock", "types-requests", "types-six", "pytest (>=4.6,<5.0)", "pytest (>=4.6)", "mypy"] +tests = ["coverage (>=3.7.1,<6.0.0)", "flake8", "mypy", "pytest (>=4.6)", "pytest (>=4.6,<5.0)", "pytest-cov", "pytest-localserver", "types-mock", "types-requests", "types-six"] [[package]] name = "six" @@ -391,9 +366,17 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + [[package]] name = "tox" -version = "3.24.4" +version = "3.27.1" description = "tox is a generic virtualenv management and test command line tool" category = "dev" optional = false @@ -406,101 +389,102 @@ packaging = ">=14" pluggy = ">=0.12.0" py = ">=1.4.17" six = ">=1.14.0" -toml = ">=0.9.4" +tomli = {version = ">=2.0.1", markers = "python_version >= \"3.7\" and python_version < \"3.11\""} virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2,<20.0.3 || >20.0.3,<20.0.4 || >20.0.4,<20.0.5 || >20.0.5,<20.0.6 || >20.0.6,<20.0.7 || >20.0.7" [package.extras] docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"] -testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)", "pytest-xdist (>=1.22.2)", "pathlib2 (>=2.3.3)"] - -[[package]] -name = "typed-ast" -version = "1.5.1" -description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "dev" -optional = false -python-versions = ">=3.6" +testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)"] [[package]] name = "typing-extensions" -version = "3.10.0.2" -description = "Backported and Experimental Type Hints for Python 3.5+" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.7" +version = "1.26.13" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.extras] -brotli = ["brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.8.1" +version = "20.16.7" description = "Virtual Python Environment builder" category = "dev" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.6" [package.dependencies] -"backports.entry-points-selectable" = ">=1.0.4" -distlib = ">=0.3.1,<1" -filelock = ">=3.0.0,<4" -platformdirs = ">=2,<3" -six = ">=1.9.0,<2" +distlib = ">=0.3.6,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<3" [package.extras] -docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)"] -testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] +docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "94082325c885ed7e0cf8cf137f9f1d8ced4b84e746adc192a8cdaa2a61e22fac" +content-hash = "c8dc7901eea7c89dca425872abe275d5a1fc8d6f1a1d39ebf7aa7cd632d7dc44" [metadata.files] -appdirs = [ - {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, - {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, -] atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, ] attrs = [ - {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, - {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, -] -"backports.entry-points-selectable" = [ - {file = "backports.entry_points_selectable-1.1.0-py2.py3-none-any.whl", hash = "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc"}, - {file = "backports.entry_points_selectable-1.1.0.tar.gz", hash = "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a"}, + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, ] black = [ - {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, + {file = "black-22.10.0-1fixedarch-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:5cc42ca67989e9c3cf859e84c2bf014f6633db63d1cbdf8fdb666dcd9e77e3fa"}, + {file = "black-22.10.0-1fixedarch-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:5d8f74030e67087b219b032aa33a919fae8806d49c867846bfacde57f43972ef"}, + {file = "black-22.10.0-1fixedarch-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:197df8509263b0b8614e1df1756b1dd41be6738eed2ba9e9769f3880c2b9d7b6"}, + {file = "black-22.10.0-1fixedarch-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:2644b5d63633702bc2c5f3754b1b475378fbbfb481f62319388235d0cd104c2d"}, + {file = "black-22.10.0-1fixedarch-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:e41a86c6c650bcecc6633ee3180d80a025db041a8e2398dcc059b3afa8382cd4"}, + {file = "black-22.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2039230db3c6c639bd84efe3292ec7b06e9214a2992cd9beb293d639c6402edb"}, + {file = "black-22.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14ff67aec0a47c424bc99b71005202045dc09270da44a27848d534600ac64fc7"}, + {file = "black-22.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:819dc789f4498ecc91438a7de64427c73b45035e2e3680c92e18795a839ebb66"}, + {file = "black-22.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b9b29da4f564ba8787c119f37d174f2b69cdfdf9015b7d8c5c16121ddc054ae"}, + {file = "black-22.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8b49776299fece66bffaafe357d929ca9451450f5466e997a7285ab0fe28e3b"}, + {file = "black-22.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:21199526696b8f09c3997e2b4db8d0b108d801a348414264d2eb8eb2532e540d"}, + {file = "black-22.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e464456d24e23d11fced2bc8c47ef66d471f845c7b7a42f3bd77bf3d1789650"}, + {file = "black-22.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9311e99228ae10023300ecac05be5a296f60d2fd10fff31cf5c1fa4ca4b1988d"}, + {file = "black-22.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fba8a281e570adafb79f7755ac8721b6cf1bbf691186a287e990c7929c7692ff"}, + {file = "black-22.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:915ace4ff03fdfff953962fa672d44be269deb2eaf88499a0f8805221bc68c87"}, + {file = "black-22.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:444ebfb4e441254e87bad00c661fe32df9969b2bf224373a448d8aca2132b395"}, + {file = "black-22.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:974308c58d057a651d182208a484ce80a26dac0caef2895836a92dd6ebd725e0"}, + {file = "black-22.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72ef3925f30e12a184889aac03d77d031056860ccae8a1e519f6cbb742736383"}, + {file = "black-22.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:432247333090c8c5366e69627ccb363bc58514ae3e63f7fc75c54b1ea80fa7de"}, + {file = "black-22.10.0-py3-none-any.whl", hash = "sha256:c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458"}, + {file = "black-22.10.0.tar.gz", hash = "sha256:f513588da599943e0cde4e32cc9879e825d58720d6557062d1098c5ad80080e1"}, ] certifi = [ - {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, - {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, + {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, + {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, ] charset-normalizer = [ - {file = "charset-normalizer-2.0.6.tar.gz", hash = "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f"}, - {file = "charset_normalizer-2.0.6-py3-none-any.whl", hash = "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6"}, + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, ] click = [ - {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, - {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] coverage = [ {file = "coverage-5.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf"}, @@ -557,85 +541,107 @@ coverage = [ {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"}, ] cssselect = [ - {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"}, - {file = "cssselect-1.1.0.tar.gz", hash = "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc"}, + {file = "cssselect-1.2.0-py2.py3-none-any.whl", hash = "sha256:da1885f0c10b60c03ed5eccbb6b68d6eff248d91976fcde348f395d54c9fd35e"}, + {file = "cssselect-1.2.0.tar.gz", hash = "sha256:666b19839cfaddb9ce9d36bfe4c969132c647b92fc9088c4e23f786b30f1b3dc"}, ] curlify = [ {file = "curlify-2.2.1.tar.gz", hash = "sha256:0d3f02e7235faf952de8ef45ef469845196d30632d5838bcd5aee217726ddd6d"}, ] distlib = [ - {file = "distlib-0.3.3-py2.py3-none-any.whl", hash = "sha256:c8b54e8454e5bf6237cc84c20e8264c3e991e824ef27e8f1e81049867d861e31"}, - {file = "distlib-0.3.3.zip", hash = "sha256:d982d0751ff6eaaab5e2ec8e691d949ee80eddf01a62eaa96ddb11531fe16b05"}, + {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, + {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] filelock = [ - {file = "filelock-3.3.0-py3-none-any.whl", hash = "sha256:bbc6a0382fe8ec4744ecdf6683a2e07f65eb10ff1aff53fc02a202565446cde0"}, - {file = "filelock-3.3.0.tar.gz", hash = "sha256:8c7eab13dc442dc249e95158bcc12dec724465919bdc9831fdbf0660f03d1785"}, + {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, + {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, ] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] idna = [ - {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, - {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] isort = [ - {file = "isort-5.9.3-py3-none-any.whl", hash = "sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2"}, - {file = "isort-5.9.3.tar.gz", hash = "sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899"}, + {file = "isort-5.10.1-py3-none-any.whl", hash = "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7"}, + {file = "isort-5.10.1.tar.gz", hash = "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"}, ] lxml = [ - {file = "lxml-4.6.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:df7c53783a46febb0e70f6b05df2ba104610f2fb0d27023409734a3ecbb78fb2"}, - {file = "lxml-4.6.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1b7584d421d254ab86d4f0b13ec662a9014397678a7c4265a02a6d7c2b18a75f"}, - {file = "lxml-4.6.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:079f3ae844f38982d156efce585bc540c16a926d4436712cf4baee0cce487a3d"}, - {file = "lxml-4.6.3-cp27-cp27m-win32.whl", hash = "sha256:bc4313cbeb0e7a416a488d72f9680fffffc645f8a838bd2193809881c67dd106"}, - {file = "lxml-4.6.3-cp27-cp27m-win_amd64.whl", hash = "sha256:8157dadbb09a34a6bd95a50690595e1fa0af1a99445e2744110e3dca7831c4ee"}, - {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7728e05c35412ba36d3e9795ae8995e3c86958179c9770e65558ec3fdfd3724f"}, - {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4bff24dfeea62f2e56f5bab929b4428ae6caba2d1eea0c2d6eb618e30a71e6d4"}, - {file = "lxml-4.6.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:64812391546a18896adaa86c77c59a4998f33c24788cadc35789e55b727a37f4"}, - {file = "lxml-4.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c1a40c06fd5ba37ad39caa0b3144eb3772e813b5fb5b084198a985431c2f1e8d"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:74f7d8d439b18fa4c385f3f5dfd11144bb87c1da034a466c5b5577d23a1d9b51"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:4c61b3a0db43a1607d6264166b230438f85bfed02e8cff20c22e564d0faff354"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux2014_x86_64.whl", hash = "sha256:5c8c163396cc0df3fd151b927e74f6e4acd67160d6c33304e805b84293351d16"}, - {file = "lxml-4.6.3-cp35-cp35m-win32.whl", hash = "sha256:f2380a6376dfa090227b663f9678150ef27543483055cc327555fb592c5967e2"}, - {file = "lxml-4.6.3-cp35-cp35m-win_amd64.whl", hash = "sha256:c4f05c5a7c49d2fb70223d0d5bcfbe474cf928310ac9fa6a7c6dddc831d0b1d4"}, - {file = "lxml-4.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d2e35d7bf1c1ac8c538f88d26b396e73dd81440d59c1ef8522e1ea77b345ede4"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:289e9ca1a9287f08daaf796d96e06cb2bc2958891d7911ac7cae1c5f9e1e0ee3"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:bccbfc27563652de7dc9bdc595cb25e90b59c5f8e23e806ed0fd623755b6565d"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:d916d31fd85b2f78c76400d625076d9124de3e4bda8b016d25a050cc7d603f24"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:820628b7b3135403540202e60551e741f9b6d3304371712521be939470b454ec"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:c47ff7e0a36d4efac9fd692cfa33fbd0636674c102e9e8d9b26e1b93a94e7617"}, - {file = "lxml-4.6.3-cp36-cp36m-win32.whl", hash = "sha256:5a0a14e264069c03e46f926be0d8919f4105c1623d620e7ec0e612a2e9bf1c04"}, - {file = "lxml-4.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:92e821e43ad382332eade6812e298dc9701c75fe289f2a2d39c7960b43d1e92a"}, - {file = "lxml-4.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:efd7a09678fd8b53117f6bae4fa3825e0a22b03ef0a932e070c0bdbb3a35e654"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:efac139c3f0bf4f0939f9375af4b02c5ad83a622de52d6dfa8e438e8e01d0eb0"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:0fbcf5565ac01dff87cbfc0ff323515c823081c5777a9fc7703ff58388c258c3"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:36108c73739985979bf302006527cf8a20515ce444ba916281d1c43938b8bb96"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:122fba10466c7bd4178b07dba427aa516286b846b2cbd6f6169141917283aae2"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:cdaf11d2bd275bf391b5308f86731e5194a21af45fbaaaf1d9e8147b9160ea92"}, - {file = "lxml-4.6.3-cp37-cp37m-win32.whl", hash = "sha256:3439c71103ef0e904ea0a1901611863e51f50b5cd5e8654a151740fde5e1cade"}, - {file = "lxml-4.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:4289728b5e2000a4ad4ab8da6e1db2e093c63c08bdc0414799ee776a3f78da4b"}, - {file = "lxml-4.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b007cbb845b28db4fb8b6a5cdcbf65bacb16a8bd328b53cbc0698688a68e1caa"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:76fa7b1362d19f8fbd3e75fe2fb7c79359b0af8747e6f7141c338f0bee2f871a"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:26e761ab5b07adf5f555ee82fb4bfc35bf93750499c6c7614bd64d12aaa67927"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:e1cbd3f19a61e27e011e02f9600837b921ac661f0c40560eefb366e4e4fb275e"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:66e575c62792c3f9ca47cb8b6fab9e35bab91360c783d1606f758761810c9791"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:1b38116b6e628118dea5b2186ee6820ab138dbb1e24a13e478490c7db2f326ae"}, - {file = "lxml-4.6.3-cp38-cp38-win32.whl", hash = "sha256:89b8b22a5ff72d89d48d0e62abb14340d9e99fd637d046c27b8b257a01ffbe28"}, - {file = "lxml-4.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:2a9d50e69aac3ebee695424f7dbd7b8c6d6eb7de2a2eb6b0f6c7db6aa41e02b7"}, - {file = "lxml-4.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ce256aaa50f6cc9a649c51be3cd4ff142d67295bfc4f490c9134d0f9f6d58ef0"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:7610b8c31688f0b1be0ef882889817939490a36d0ee880ea562a4e1399c447a1"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f8380c03e45cf09f8557bdaa41e1fa7c81f3ae22828e1db470ab2a6c96d8bc23"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:3082c518be8e97324390614dacd041bb1358c882d77108ca1957ba47738d9d59"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:884ab9b29feaca361f7f88d811b1eea9bfca36cf3da27768d28ad45c3ee6f969"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:6f12e1427285008fd32a6025e38e977d44d6382cf28e7201ed10d6c1698d2a9a"}, - {file = "lxml-4.6.3-cp39-cp39-win32.whl", hash = "sha256:33bb934a044cf32157c12bfcfbb6649807da20aa92c062ef51903415c704704f"}, - {file = "lxml-4.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:542d454665a3e277f76954418124d67516c5f88e51a900365ed54a9806122b83"}, - {file = "lxml-4.6.3.tar.gz", hash = "sha256:39b78571b3b30645ac77b95f7c69d1bffc4cf8c3b157c435a34da72e78c82468"}, + {file = "lxml-4.9.1-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed"}, + {file = "lxml-4.9.1-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc"}, + {file = "lxml-4.9.1-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc"}, + {file = "lxml-4.9.1-cp27-cp27m-win32.whl", hash = "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3"}, + {file = "lxml-4.9.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627"}, + {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84"}, + {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837"}, + {file = "lxml-4.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad"}, + {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5"}, + {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8"}, + {file = "lxml-4.9.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8"}, + {file = "lxml-4.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d"}, + {file = "lxml-4.9.1-cp310-cp310-win32.whl", hash = "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7"}, + {file = "lxml-4.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b"}, + {file = "lxml-4.9.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d"}, + {file = "lxml-4.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3"}, + {file = "lxml-4.9.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29"}, + {file = "lxml-4.9.1-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d"}, + {file = "lxml-4.9.1-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318"}, + {file = "lxml-4.9.1-cp35-cp35m-win32.whl", hash = "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7"}, + {file = "lxml-4.9.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4"}, + {file = "lxml-4.9.1-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb"}, + {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067"}, + {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536"}, + {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8"}, + {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b"}, + {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf"}, + {file = "lxml-4.9.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3"}, + {file = "lxml-4.9.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391"}, + {file = "lxml-4.9.1-cp36-cp36m-win32.whl", hash = "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e"}, + {file = "lxml-4.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7"}, + {file = "lxml-4.9.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2"}, + {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc"}, + {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c"}, + {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4"}, + {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3"}, + {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca"}, + {file = "lxml-4.9.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785"}, + {file = "lxml-4.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785"}, + {file = "lxml-4.9.1-cp37-cp37m-win32.whl", hash = "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a"}, + {file = "lxml-4.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e"}, + {file = "lxml-4.9.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b"}, + {file = "lxml-4.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97"}, + {file = "lxml-4.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21"}, + {file = "lxml-4.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2"}, + {file = "lxml-4.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130"}, + {file = "lxml-4.9.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715"}, + {file = "lxml-4.9.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036"}, + {file = "lxml-4.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387"}, + {file = "lxml-4.9.1-cp38-cp38-win32.whl", hash = "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94"}, + {file = "lxml-4.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345"}, + {file = "lxml-4.9.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67"}, + {file = "lxml-4.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb"}, + {file = "lxml-4.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448"}, + {file = "lxml-4.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7"}, + {file = "lxml-4.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91"}, + {file = "lxml-4.9.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000"}, + {file = "lxml-4.9.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25"}, + {file = "lxml-4.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd"}, + {file = "lxml-4.9.1-cp39-cp39-win32.whl", hash = "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb"}, + {file = "lxml-4.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d"}, + {file = "lxml-4.9.1-pp37-pypy37_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c"}, + {file = "lxml-4.9.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b"}, + {file = "lxml-4.9.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc"}, + {file = "lxml-4.9.1-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b"}, + {file = "lxml-4.9.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2"}, + {file = "lxml-4.9.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73"}, + {file = "lxml-4.9.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c"}, + {file = "lxml-4.9.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9"}, + {file = "lxml-4.9.1.tar.gz", hash = "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f"}, ] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, @@ -646,24 +652,24 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] packaging = [ - {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, - {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] pathspec = [ - {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, - {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, + {file = "pathspec-0.10.2-py3-none-any.whl", hash = "sha256:88c2606f2c1e818b978540f73ecc908e13999c6c3a383daf3705652ae79807a5"}, + {file = "pathspec-0.10.2.tar.gz", hash = "sha256:8f6bf73e5758fd365ef5d58ce09ac7c27d2833a8d7da51712eac6e27e35141b0"}, ] platformdirs = [ - {file = "platformdirs-2.4.0-py3-none-any.whl", hash = "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d"}, - {file = "platformdirs-2.4.0.tar.gz", hash = "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2"}, + {file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"}, + {file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] py = [ - {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, - {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] pycodestyle = [ {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, @@ -674,63 +680,20 @@ pyflakes = [ {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, ] pyparsing = [ - {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, - {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] pytest = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] pytest-mock = [ - {file = "pytest-mock-3.6.1.tar.gz", hash = "sha256:40217a058c52a63f1042f0784f62009e976ba824c418cced42e88d5f40ab0e62"}, - {file = "pytest_mock-3.6.1-py3-none-any.whl", hash = "sha256:30c2f2cc9759e76eee674b81ea28c9f0b94f8f0445a1b87762cadf774f0df7e3"}, -] -regex = [ - {file = "regex-2021.9.30-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:66696c8336a1b5d1182464f3af3427cc760118f26d0b09a2ddc16a976a4d2637"}, - {file = "regex-2021.9.30-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d87459ad3ab40cd8493774f8a454b2e490d8e729e7e402a0625867a983e4e02"}, - {file = "regex-2021.9.30-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78cf6a1e023caf5e9a982f5377414e1aeac55198831b852835732cfd0a0ca5ff"}, - {file = "regex-2021.9.30-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:255791523f80ea8e48e79af7120b4697ef3b74f6886995dcdb08c41f8e516be0"}, - {file = "regex-2021.9.30-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e502f8d4e5ef714bcc2c94d499684890c94239526d61fdf1096547db91ca6aa6"}, - {file = "regex-2021.9.30-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4907fb0f9b9309a5bded72343e675a252c2589a41871874feace9a05a540241e"}, - {file = "regex-2021.9.30-cp310-cp310-win32.whl", hash = "sha256:3be40f720af170a6b20ddd2ad7904c58b13d2b56f6734ee5d09bbdeed2fa4816"}, - {file = "regex-2021.9.30-cp310-cp310-win_amd64.whl", hash = "sha256:c2b180ed30856dfa70cfe927b0fd38e6b68198a03039abdbeb1f2029758d87e7"}, - {file = "regex-2021.9.30-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e6f2d2f93001801296fe3ca86515eb04915472b5380d4d8752f09f25f0b9b0ed"}, - {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fa7ba9ab2eba7284e0d7d94f61df7af86015b0398e123331362270d71fab0b9"}, - {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28040e89a04b60d579c69095c509a4f6a1a5379cd865258e3a186b7105de72c6"}, - {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f588209d3e4797882cd238195c175290dbc501973b10a581086b5c6bcd095ffb"}, - {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42952d325439ef223e4e9db7ee6d9087b5c68c5c15b1f9de68e990837682fc7b"}, - {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cae4099031d80703954c39680323dabd87a69b21262303160776aa0e55970ca0"}, - {file = "regex-2021.9.30-cp36-cp36m-win32.whl", hash = "sha256:0de8ad66b08c3e673b61981b9e3626f8784d5564f8c3928e2ad408c0eb5ac38c"}, - {file = "regex-2021.9.30-cp36-cp36m-win_amd64.whl", hash = "sha256:b345ecde37c86dd7084c62954468a4a655fd2d24fd9b237949dd07a4d0dd6f4c"}, - {file = "regex-2021.9.30-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6f08187136f11e430638c2c66e1db091105d7c2e9902489f0dbc69b44c222b4"}, - {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b55442650f541d195a535ccec33078c78a9521973fb960923da7515e9ed78fa6"}, - {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87e9c489aa98f50f367fb26cc9c8908d668e9228d327644d7aa568d47e456f47"}, - {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e2cb7d4909ed16ed35729d38af585673f1f0833e73dfdf0c18e5be0061107b99"}, - {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0861e7f6325e821d5c40514c551fd538b292f8cc3960086e73491b9c5d8291d"}, - {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:81fdc90f999b2147fc62e303440c424c47e5573a9b615ed5d43a5b832efcca9e"}, - {file = "regex-2021.9.30-cp37-cp37m-win32.whl", hash = "sha256:8c1ad61fa024195136a6b7b89538030bd00df15f90ac177ca278df9b2386c96f"}, - {file = "regex-2021.9.30-cp37-cp37m-win_amd64.whl", hash = "sha256:e3770781353a4886b68ef10cec31c1f61e8e3a0be5f213c2bb15a86efd999bc4"}, - {file = "regex-2021.9.30-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9c065d95a514a06b92a5026766d72ac91bfabf581adb5b29bc5c91d4b3ee9b83"}, - {file = "regex-2021.9.30-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9925985be05d54b3d25fd6c1ea8e50ff1f7c2744c75bdc4d3b45c790afa2bcb3"}, - {file = "regex-2021.9.30-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:470f2c882f2672d8eeda8ab27992aec277c067d280b52541357e1acd7e606dae"}, - {file = "regex-2021.9.30-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ad0517df22a97f1da20d8f1c8cb71a5d1997fa383326b81f9cf22c9dadfbdf34"}, - {file = "regex-2021.9.30-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9e30838df7bfd20db6466fd309d9b580d32855f8e2c2e6d74cf9da27dcd9b63"}, - {file = "regex-2021.9.30-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b34d2335d6aedec7dcadd3f8283b9682fadad8b9b008da8788d2fce76125ebe"}, - {file = "regex-2021.9.30-cp38-cp38-win32.whl", hash = "sha256:e07049cece3462c626d650e8bf42ddbca3abf4aa08155002c28cb6d9a5a281e2"}, - {file = "regex-2021.9.30-cp38-cp38-win_amd64.whl", hash = "sha256:37868075eda024470bd0feab872c692ac4ee29db1e14baec103257bf6cc64346"}, - {file = "regex-2021.9.30-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d331f238a7accfbbe1c4cd1ba610d4c087b206353539331e32a8f05345c74aec"}, - {file = "regex-2021.9.30-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6348a7ab2a502cbdd0b7fd0496d614007489adb7361956b38044d1d588e66e04"}, - {file = "regex-2021.9.30-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce7b1cca6c23f19bee8dc40228d9c314d86d1e51996b86f924aca302fc8f8bf9"}, - {file = "regex-2021.9.30-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1f1125bc5172ab3a049bc6f4b9c0aae95a2a2001a77e6d6e4239fa3653e202b5"}, - {file = "regex-2021.9.30-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:638e98d069b14113e8afba6a54d1ca123f712c0d105e67c1f9211b2a825ef926"}, - {file = "regex-2021.9.30-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9a0b0db6b49da7fa37ca8eddf9f40a8dbc599bad43e64f452284f37b6c34d91c"}, - {file = "regex-2021.9.30-cp39-cp39-win32.whl", hash = "sha256:9910869c472e5a6728680ca357b5846546cbbd2ab3ad5bef986ef0bc438d0aa6"}, - {file = "regex-2021.9.30-cp39-cp39-win_amd64.whl", hash = "sha256:3b71213ec3bad9a5a02e049f2ec86b3d7c3e350129ae0f4e2f99c12b5da919ed"}, - {file = "regex-2021.9.30.tar.gz", hash = "sha256:81e125d9ba54c34579e4539a967e976a3c56150796674aec318b1b2f49251be7"}, + {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, + {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, ] requests = [ - {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, - {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, + {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, + {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] responses = [ {file = "responses-0.13.4-py2.py3-none-any.whl", hash = "sha256:d8d0f655710c46fd3513b9202a7f0dcedd02ca0f8cf4976f27fa8ab5b81e656d"}, @@ -744,41 +707,23 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] -tox = [ - {file = "tox-3.24.4-py2.py3-none-any.whl", hash = "sha256:5e274227a53dc9ef856767c21867377ba395992549f02ce55eb549f9fb9a8d10"}, - {file = "tox-3.24.4.tar.gz", hash = "sha256:c30b57fa2477f1fb7c36aa1d83292d5c2336cd0018119e1b1c17340e2c2708ca"}, +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -typed-ast = [ - {file = "typed_ast-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d8314c92414ce7481eee7ad42b353943679cf6f30237b5ecbf7d835519e1212"}, - {file = "typed_ast-1.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b53ae5de5500529c76225d18eeb060efbcec90ad5e030713fe8dab0fb4531631"}, - {file = "typed_ast-1.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:24058827d8f5d633f97223f5148a7d22628099a3d2efe06654ce872f46f07cdb"}, - {file = "typed_ast-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:a6d495c1ef572519a7bac9534dbf6d94c40e5b6a608ef41136133377bba4aa08"}, - {file = "typed_ast-1.5.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:de4ecae89c7d8b56169473e08f6bfd2df7f95015591f43126e4ea7865928677e"}, - {file = "typed_ast-1.5.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:256115a5bc7ea9e665c6314ed6671ee2c08ca380f9d5f130bd4d2c1f5848d695"}, - {file = "typed_ast-1.5.1-cp36-cp36m-win_amd64.whl", hash = "sha256:7c42707ab981b6cf4b73490c16e9d17fcd5227039720ca14abe415d39a173a30"}, - {file = "typed_ast-1.5.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:71dcda943a471d826ea930dd449ac7e76db7be778fcd722deb63642bab32ea3f"}, - {file = "typed_ast-1.5.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4f30a2bcd8e68adbb791ce1567fdb897357506f7ea6716f6bbdd3053ac4d9471"}, - {file = "typed_ast-1.5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ca9e8300d8ba0b66d140820cf463438c8e7b4cdc6fd710c059bfcfb1531d03fb"}, - {file = "typed_ast-1.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9caaf2b440efb39ecbc45e2fabde809cbe56272719131a6318fd9bf08b58e2cb"}, - {file = "typed_ast-1.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c9bcad65d66d594bffab8575f39420fe0ee96f66e23c4d927ebb4e24354ec1af"}, - {file = "typed_ast-1.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:591bc04e507595887160ed7aa8d6785867fb86c5793911be79ccede61ae96f4d"}, - {file = "typed_ast-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:a80d84f535642420dd17e16ae25bb46c7f4c16ee231105e7f3eb43976a89670a"}, - {file = "typed_ast-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:38cf5c642fa808300bae1281460d4f9b7617cf864d4e383054a5ef336e344d32"}, - {file = "typed_ast-1.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b6ab14c56bc9c7e3c30228a0a0b54b915b1579613f6e463ba6f4eb1382e7fd4"}, - {file = "typed_ast-1.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a2b8d7007f6280e36fa42652df47087ac7b0a7d7f09f9468f07792ba646aac2d"}, - {file = "typed_ast-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:b6d17f37f6edd879141e64a5db17b67488cfeffeedad8c5cec0392305e9bc775"}, - {file = "typed_ast-1.5.1.tar.gz", hash = "sha256:484137cab8ecf47e137260daa20bafbba5f4e3ec7fda1c1e69ab299b75fa81c5"}, +tox = [ + {file = "tox-3.27.1-py2.py3-none-any.whl", hash = "sha256:f52ca66eae115fcfef0e77ef81fd107133d295c97c52df337adedb8dfac6ab84"}, + {file = "tox-3.27.1.tar.gz", hash = "sha256:b2a920e35a668cc06942ffd1cf3a4fb221a4d909ca72191fb6d84b0b18a7be04"}, ] typing-extensions = [ - {file = "typing_extensions-3.10.0.2-py2-none-any.whl", hash = "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7"}, - {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"}, - {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] urllib3 = [ - {file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"}, - {file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"}, + {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, + {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, ] virtualenv = [ - {file = "virtualenv-20.8.1-py2.py3-none-any.whl", hash = "sha256:10062e34c204b5e4ec5f62e6ef2473f8ba76513a9a617e873f1f8fb4a519d300"}, - {file = "virtualenv-20.8.1.tar.gz", hash = "sha256:bcc17f0b3a29670dd777d6f0755a4c04f28815395bca279cdcb213b97199a6b8"}, + {file = "virtualenv-20.16.7-py3-none-any.whl", hash = "sha256:efd66b00386fdb7dbe4822d172303f40cd05e50e01740b19ea42425cbe653e29"}, + {file = "virtualenv-20.16.7.tar.gz", hash = "sha256:8691e3ff9387f743e00f6bb20f70121f5e4f596cae754531f2b3b3a1b1ac696e"}, ] diff --git a/pyproject.toml b/pyproject.toml index fcbf936..a0c1f84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ curlify = "^2.2.1" [tool.poetry.dev-dependencies] pytest = "^6.2.2" -black = "^20.8b1" +black = "^22.10.0" flake8 = "^3.9.1" isort = "^5.8.0" responses = "^0.13.3" From 582bf952e02a44e2d2159cd0192e6080350a3166 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 24 Nov 2022 13:55:04 +0100 Subject: [PATCH 099/149] Formatting. --- src/pricehist/outputs/gnucashsql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pricehist/outputs/gnucashsql.py b/src/pricehist/outputs/gnucashsql.py index 2034754..723fcd4 100644 --- a/src/pricehist/outputs/gnucashsql.py +++ b/src/pricehist/outputs/gnucashsql.py @@ -169,9 +169,9 @@ class GnuCashSQL(BaseOutput): denom = str(1) else: numerator = sign + "".join([str(d) for d in tup.digits]) - denom = str(10 ** -tup.exponent) + denom = str(10**-tup.exponent) fit = self._fit_in_int64(Decimal(numerator), Decimal(denom)) return (numerator, denom, fit) def _fit_in_int64(self, *numbers): - return all(n >= -(2 ** 63) and n <= (2 ** 63) - 1 for n in numbers) + return all(n >= -(2**63) and n <= (2**63) - 1 for n in numbers) From 09fbeb79cbf4ef8aee2b8a0a84e3926d5793b905 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 24 Nov 2022 14:47:51 +0100 Subject: [PATCH 100/149] Don't mock a mock (doesn't work in Python 11). --- tests/pricehist/test_fetch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pricehist/test_fetch.py b/tests/pricehist/test_fetch.py index 61af29b..30cda76 100644 --- a/tests/pricehist/test_fetch.py +++ b/tests/pricehist/test_fetch.py @@ -65,7 +65,7 @@ def test_fetch_returns_formatted_output(source, res_series, output, fmt, mocker) def test_fetch_inverts_if_requested(source, res_series, output, fmt, mocker): req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") - inv_series = mocker.MagicMock(res_series) + inv_series = mocker.MagicMock() res_series.invert = mocker.MagicMock(return_value=inv_series) fetch(req_series, source, output, invert=True, quantize=None, fmt=fmt) @@ -76,7 +76,7 @@ def test_fetch_inverts_if_requested(source, res_series, output, fmt, mocker): def test_fetch_quantizes_if_requested(source, res_series, output, fmt, mocker): req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") - qnt_series = mocker.MagicMock(res_series) + qnt_series = mocker.MagicMock() res_series.quantize = mocker.MagicMock(return_value=qnt_series) fetch(req_series, source, output, invert=False, quantize=2, fmt=fmt) From d6036c9d148b79b3fac39c0ba5c713dd88497c8b Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 24 Nov 2022 15:14:42 +0100 Subject: [PATCH 101/149] Update Alphavantage source for changes in which endpoint is premium. --- src/pricehist/sources/alphavantage.py | 12 +++++------- tests/live.sh | 10 +++++----- tests/pricehist/sources/test_alphavantage.py | 16 ++++------------ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index 707ba50..4824282 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -51,8 +51,7 @@ class AlphaVantage(BaseSource): "will list all digital and physical currency symbols.\n" "The PAIR for stocks is the stock symbol only. The quote currency " f"will be determined automatically. {self._stock_symbols_message()}\n" - "The price type 'adjclose' is only available for stocks, and " - "requires an access key for which premium endpoints are unlocked.\n" + "The price type 'adjclose' is only available for stocks.\n" "Beware that digital currencies quoted in non-USD currencies may " "be converted from USD data at one recent exchange rate rather " "than using historical rates.\n" @@ -187,10 +186,9 @@ class AlphaVantage(BaseSource): def _stock_data(self, series): output_quote = self._stock_currency(series.base) or "UNKNOWN" - if series.type == "adjclose": - function = "TIME_SERIES_DAILY_ADJUSTED" - else: - function = "TIME_SERIES_DAILY" + # As of 2022-11-24 TIME_SERIES_DAILY_ADJUSTED is no longer premium, but + # now TIME_SERIES_DAILY is. So, always use TIME_SERIES_DAILY_ADJUSTED. + function = "TIME_SERIES_DAILY_ADJUSTED" params = { "function": function, @@ -341,7 +339,7 @@ class AlphaVantage(BaseSource): raise exceptions.RateLimit(data["Note"]) if ( "Information" in data - and "ways to unlock premium" in data["Information"] + and "unlock all premium endpoints" in data["Information"] ): msg = "You were denied access to a premium endpoint." raise exceptions.CredentialsError([self.API_KEY_NAME], self, msg) diff --git a/tests/live.sh b/tests/live.sh index 6b391e7..c9454d4 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -47,11 +47,11 @@ name="Alpha Vantage stocks" cmd="pricehist fetch alphavantage TSLA -s 2021-01-04 -e 2021-01-08" read -r -d '' expected < Date: Thu, 24 Nov 2022 15:29:39 +0100 Subject: [PATCH 102/149] Note deprecation of Coindesk Bitcoin Price Index source. --- src/pricehist/sources/coindesk.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pricehist/sources/coindesk.py b/src/pricehist/sources/coindesk.py index 0170a99..7947d1a 100644 --- a/src/pricehist/sources/coindesk.py +++ b/src/pricehist/sources/coindesk.py @@ -1,5 +1,6 @@ import dataclasses import json +import logging from decimal import Decimal import requests @@ -19,7 +20,9 @@ class CoinDesk(BaseSource): def description(self): return ( - "An average of Bitcoin prices across leading global exchanges. \n" + "WARNING: This source is deprecated. Data stops at 2022-07-10.\n" + "The documentation URL now redirects to the main page.\n" + "An average of Bitcoin prices across leading global exchanges.\n" "Powered by CoinDesk, https://www.coindesk.com/price/bitcoin" ) @@ -64,6 +67,8 @@ class CoinDesk(BaseSource): return results def fetch(self, series): + logging.warning("This source is deprecated. Data stops at 2022-07-10.") + if series.base != "BTC" or series.quote in ["BTC", "XBT"]: # BTC is the only valid base. # BTC as the quote will return BTC/USD, which we don't want. From 71ed246956c7edfc8f0f357dc16a8371052e3607 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 24 Nov 2022 15:33:35 +0100 Subject: [PATCH 103/149] Version 1.4.3. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a0c1f84..18ce60f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.2" +version = "1.4.3" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index daa50c7..aa56ed4 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.2" +__version__ = "1.4.3" From b522a0961cdbf6c258ad664688923f39944c5653 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 24 Nov 2022 15:38:13 +0100 Subject: [PATCH 104/149] Fix live tests. --- tests/live.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/live.sh b/tests/live.sh index c9454d4..20c2651 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -107,6 +107,7 @@ run_test "$name" "$cmd" "$expected" name="CoinDesk Bitcoin Price Index" cmd="pricehist fetch coindesk BTC/EUR -s 2021-01-04 -e 2021-01-08" read -r -d '' expected < Date: Thu, 24 Nov 2022 15:43:23 +0100 Subject: [PATCH 105/149] Add coverage regex to gitlab CI config. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29ec768..e2fc260 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,3 +30,4 @@ coverage: script: - poetry run coverage run --source=pricehist -m pytest - poetry run coverage report + coverage: '/^TOTAL.+?(\d+\%)$/' From 2398b8340f2647cfd75ddbac5df2e9283223cef1 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Wed, 25 Jan 2023 11:50:28 +0100 Subject: [PATCH 106/149] Update IOS 4217 data. --- src/pricehist/resources/list-one.xml | 8 ++++---- src/pricehist/resources/list-three.xml | 11 +++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/pricehist/resources/list-one.xml b/src/pricehist/resources/list-one.xml index 031048f..1f72a89 100644 --- a/src/pricehist/resources/list-one.xml +++ b/src/pricehist/resources/list-one.xml @@ -1,5 +1,5 @@ - + AFGHANISTAN @@ -413,9 +413,9 @@ CROATIA - Kuna - HRK - 191 + Euro + EUR + 978 2 diff --git a/src/pricehist/resources/list-three.xml b/src/pricehist/resources/list-three.xml index 3959003..584b33b 100644 --- a/src/pricehist/resources/list-three.xml +++ b/src/pricehist/resources/list-three.xml @@ -1,5 +1,5 @@ - - + + AFGHANISTAN @@ -253,6 +253,13 @@ 191 2015-06 + + CROATIA + Kuna + HRK + 191 + 2023-01 + CYPRUS Cyprus Pound From b99e71202ab72a25953376cc2eabf3881c4487bf Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Wed, 25 Jan 2023 11:52:25 +0100 Subject: [PATCH 107/149] Version 1.4.4. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 18ce60f..814f06f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.3" +version = "1.4.4" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index aa56ed4..c0f285b 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.3" +__version__ = "1.4.4" From 7f4ed2f8b55dd8a0cb9dd548b42118dd43d2f603 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 29 May 2023 14:36:07 +0200 Subject: [PATCH 108/149] Skip test of known failing Alphavantage endpoint for a couple of weeks. --- tests/live.sh | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/live.sh b/tests/live.sh index 20c2651..9224a0d 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -11,6 +11,7 @@ cmd_prefix="poetry run" passed=0 failed=0 +skipped=0 run_test(){ name=$1 @@ -33,12 +34,27 @@ run_test(){ echo } +skip_test(){ + name=$1 + cmd=$2 + echo "TEST: $name" + echo " Action: $cmd" + echo " Result: SKIPPED!" + skipped=$((skipped+1)) + echo +} + report(){ total=$((passed+failed)) - if [[ "$failed" -eq "0" ]]; then - echo "SUMMARY: $passed tests passed, none failed" + if [[ "$skipped" -eq "0" ]]; then + skipped_str="none" else - echo "SUMMARY: $failed/$total tests failed" + skipped_str="$skipped" + fi + if [[ "$failed" -eq "0" ]]; then + echo "SUMMARY: $passed tests passed, none failed, $skipped_str skipped" + else + echo "SUMMARY: $failed/$total tests failed, $skipped_str skipped" exit 1 fi } @@ -78,7 +94,11 @@ date,base,quote,amount,source,type 2021-01-07,BTC,USD,39432.28000000,alphavantage,close 2021-01-08,BTC,USD,40582.81000000,alphavantage,close END -run_test "$name" "$cmd" "$expected" +if [[ "$(date --iso-8601)" < "2023-06-15" ]]; then + skip_test "$name" "$cmd" "$expected" +else + run_test "$name" "$cmd" "$expected" +fi name="Bank of Canada" cmd="pricehist fetch bankofcanada CAD/USD -s 2021-01-04 -e 2021-01-08" From 34c503f6cbf7a74be3bf42183844a034d06f02f1 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 10 Jun 2023 13:19:35 +0200 Subject: [PATCH 109/149] Use non-deprecated importlib_resources API. --- src/pricehist/isocurrencies.py | 18 +++++++++++++----- src/pricehist/outputs/gnucashsql.py | 21 +++++++++++++-------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/pricehist/isocurrencies.py b/src/pricehist/isocurrencies.py index bd453dc..624fe1d 100644 --- a/src/pricehist/isocurrencies.py +++ b/src/pricehist/isocurrencies.py @@ -24,7 +24,7 @@ Functions: """ from dataclasses import dataclass, field -from importlib.resources import read_binary +from importlib.resources import files from typing import List from lxml import etree @@ -43,20 +43,28 @@ class ISOCurrency: def current_data_date(): - one = etree.fromstring(read_binary("pricehist.resources", "list-one.xml")) + one = etree.fromstring( + files("pricehist.resources").joinpath("list-one.xml").read_bytes() + ) return one.cssselect("ISO_4217")[0].attrib["Pblshd"] def historical_data_date(): - three = etree.fromstring(read_binary("pricehist.resources", "list-three.xml")) + three = etree.fromstring( + files("pricehist.resources").joinpath("list-three.xml").read_bytes() + ) return three.cssselect("ISO_4217")[0].attrib["Pblshd"] def by_code(): result = {} - one = etree.fromstring(read_binary("pricehist.resources", "list-one.xml")) - three = etree.fromstring(read_binary("pricehist.resources", "list-three.xml")) + one = etree.fromstring( + files("pricehist.resources").joinpath("list-one.xml").read_bytes() + ) + three = etree.fromstring( + files("pricehist.resources").joinpath("list-three.xml").read_bytes() + ) for entry in three.cssselect("HstrcCcyNtry") + one.cssselect("CcyNtry"): if currency := _parse(entry): diff --git a/src/pricehist/outputs/gnucashsql.py b/src/pricehist/outputs/gnucashsql.py index 723fcd4..44841c1 100644 --- a/src/pricehist/outputs/gnucashsql.py +++ b/src/pricehist/outputs/gnucashsql.py @@ -42,7 +42,7 @@ import hashlib import logging from datetime import datetime from decimal import Decimal -from importlib.resources import read_text +from importlib.resources import files from pricehist import __version__ from pricehist.format import Format @@ -119,13 +119,18 @@ class GnuCashSQL(BaseOutput): "well." ) - sql = read_text("pricehist.resources", "gnucash.sql").format( - version=__version__, - timestamp=datetime.utcnow().isoformat() + "Z", - base=self._sql_str(base), - quote=self._sql_str(quote), - values_comment=values_comment, - values=values, + sql = ( + files("pricehist.resources") + .joinpath("gnucash.sql") + .read_text() + .format( + version=__version__, + timestamp=datetime.utcnow().isoformat() + "Z", + base=self._sql_str(base), + quote=self._sql_str(quote), + values_comment=values_comment, + values=values, + ) ) return sql From b7b2862b77c0e6f30eca2ec646b12e81d872ae65 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 10 Jun 2023 13:21:40 +0200 Subject: [PATCH 110/149] Yahoo: keep padding the end timestamp but ignore any extra day returned. --- src/pricehist/sources/yahoo.py | 6 ++++-- tests/pricehist/sources/test_yahoo.py | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index f93f53f..6997301 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -140,7 +140,7 @@ class Yahoo(BaseSource): .timestamp() ) + ( 24 * 60 * 60 - ) # round up to include the last day + ) # some symbols require padding on the end timestamp history_url = f"{base_url}/download/{series.base}" history_params = { @@ -191,4 +191,6 @@ class Yahoo(BaseSource): if history_lines[0] != "date,open,high,low,close,adjclose,volume": raise exceptions.ResponseParsingError("Unexpected CSV format") - return (quote, history) + requested_history = [row for row in history if row["date"] <= series.end] + + return (quote, requested_history) diff --git a/tests/pricehist/sources/test_yahoo.py b/tests/pricehist/sources/test_yahoo.py index e1954dc..ba1849b 100644 --- a/tests/pricehist/sources/test_yahoo.py +++ b/tests/pricehist/sources/test_yahoo.py @@ -126,6 +126,12 @@ def test_fetch_requests_and_receives_correct_times(src, type, spark_ok, recent_o assert series.prices[-1] == Price("2021-01-08", Decimal("880.020020")) +def test_fetch_ignores_any_extra_row(src, type, spark_ok, recent_ok): + series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-07")) + assert series.prices[0] == Price("2021-01-04", Decimal("729.770020")) + assert series.prices[-1] == Price("2021-01-07", Decimal("816.039978")) + + def test_fetch_requests_logged(src, type, spark_ok, recent_ok, caplog): with caplog.at_level(logging.DEBUG): src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) From 2b8460ff4b80ba0e0c4d7a8089e9efeaf871aa07 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 10 Jun 2023 13:35:33 +0200 Subject: [PATCH 111/149] Version 1.4.5. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 814f06f..35a776c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.4" +version = "1.4.5" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index c0f285b..56dadec 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.4" +__version__ = "1.4.5" From bd3489ea71da5cdc6fa8ae2820dc7023cdad16d0 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 26 Aug 2023 10:38:31 +0200 Subject: [PATCH 112/149] Handle coinmarketcap return null for some prices. --- src/pricehist/sources/coinmarketcap.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pricehist/sources/coinmarketcap.py b/src/pricehist/sources/coinmarketcap.py index 313c493..7da1a25 100644 --- a/src/pricehist/sources/coinmarketcap.py +++ b/src/pricehist/sources/coinmarketcap.py @@ -57,7 +57,8 @@ class CoinMarketCap(BaseSource): for item in data.get("quotes", []): d = item["time_open"][0:10] amount = self._amount(next(iter(item["quote"].values())), series.type) - prices.append(Price(d, amount)) + if amount is not None: + prices.append(Price(d, amount)) output_base, output_quote = self._output_pair(series.base, series.quote, data) @@ -155,12 +156,14 @@ class CoinMarketCap(BaseSource): return parsed["data"] def _amount(self, data, type): - if type in ["mid"]: + if type in ["mid"] and data["high"] is not None and data["low"] is not None: high = Decimal(str(data["high"])) low = Decimal(str(data["low"])) return sum([high, low]) / 2 - else: + elif type in data and data[type] is not None: return Decimal(str(data[type])) + else: + return None def _output_pair(self, base, quote, data): data_base = data["symbol"] From 786ddd3c8c53dbfdd4f0dde889ffcdd7aaa279f2 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 26 Aug 2023 10:48:49 +0200 Subject: [PATCH 113/149] Update which Alphavantage test is skipped. --- tests/live.sh | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/live.sh b/tests/live.sh index 9224a0d..2fd2b09 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -69,8 +69,11 @@ date,base,quote,amount,source,type 2021-01-07,TSLA,USD,816.04,alphavantage,close 2021-01-08,TSLA,USD,880.02,alphavantage,close END -run_test "$name" "$cmd" "$expected" - +if [[ "$(date --iso-8601)" < "2023-10-01" ]]; then + skip_test "$name" "$cmd" "$expected" +else + run_test "$name" "$cmd" "$expected" +fi name="Alpha Vantage physical currency" cmd="pricehist fetch alphavantage AUD/EUR -s 2021-01-04 -e 2021-01-08" @@ -80,7 +83,7 @@ date,base,quote,amount,source,type 2021-01-05,AUD,EUR,0.63086,alphavantage,close 2021-01-06,AUD,EUR,0.63306,alphavantage,close 2021-01-07,AUD,EUR,0.63284,alphavantage,close -2021-01-08,AUD,EUR,0.63530,alphavantage,close +2021-01-08,AUD,EUR,0.63360,alphavantage,close END run_test "$name" "$cmd" "$expected" @@ -94,11 +97,7 @@ date,base,quote,amount,source,type 2021-01-07,BTC,USD,39432.28000000,alphavantage,close 2021-01-08,BTC,USD,40582.81000000,alphavantage,close END -if [[ "$(date --iso-8601)" < "2023-06-15" ]]; then - skip_test "$name" "$cmd" "$expected" -else - run_test "$name" "$cmd" "$expected" -fi +run_test "$name" "$cmd" "$expected" name="Bank of Canada" cmd="pricehist fetch bankofcanada CAD/USD -s 2021-01-04 -e 2021-01-08" @@ -141,11 +140,11 @@ name="CoinMarketCap" cmd="pricehist fetch coinmarketcap BTC/EUR -s 2021-01-04 -e 2021-01-08" read -r -d '' expected < Date: Sat, 26 Aug 2023 10:50:41 +0200 Subject: [PATCH 114/149] Revert "Update Alphavantage source for changes in which endpoint is premium." This reverts commit d6036c9d148b79b3fac39c0ba5c713dd88497c8b. --- src/pricehist/sources/alphavantage.py | 12 +++++++----- tests/live.sh | 10 +++++----- tests/pricehist/sources/test_alphavantage.py | 16 ++++++++++++---- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index 4824282..707ba50 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -51,7 +51,8 @@ class AlphaVantage(BaseSource): "will list all digital and physical currency symbols.\n" "The PAIR for stocks is the stock symbol only. The quote currency " f"will be determined automatically. {self._stock_symbols_message()}\n" - "The price type 'adjclose' is only available for stocks.\n" + "The price type 'adjclose' is only available for stocks, and " + "requires an access key for which premium endpoints are unlocked.\n" "Beware that digital currencies quoted in non-USD currencies may " "be converted from USD data at one recent exchange rate rather " "than using historical rates.\n" @@ -186,9 +187,10 @@ class AlphaVantage(BaseSource): def _stock_data(self, series): output_quote = self._stock_currency(series.base) or "UNKNOWN" - # As of 2022-11-24 TIME_SERIES_DAILY_ADJUSTED is no longer premium, but - # now TIME_SERIES_DAILY is. So, always use TIME_SERIES_DAILY_ADJUSTED. - function = "TIME_SERIES_DAILY_ADJUSTED" + if series.type == "adjclose": + function = "TIME_SERIES_DAILY_ADJUSTED" + else: + function = "TIME_SERIES_DAILY" params = { "function": function, @@ -339,7 +341,7 @@ class AlphaVantage(BaseSource): raise exceptions.RateLimit(data["Note"]) if ( "Information" in data - and "unlock all premium endpoints" in data["Information"] + and "ways to unlock premium" in data["Information"] ): msg = "You were denied access to a premium endpoint." raise exceptions.CredentialsError([self.API_KEY_NAME], self, msg) diff --git a/tests/live.sh b/tests/live.sh index 2fd2b09..78da3c0 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -63,11 +63,11 @@ name="Alpha Vantage stocks" cmd="pricehist fetch alphavantage TSLA -s 2021-01-04 -e 2021-01-08" read -r -d '' expected < Date: Sat, 26 Aug 2023 10:57:06 +0200 Subject: [PATCH 115/149] Don't skip any AlphaVantage tests anymore. All pass. --- tests/live.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/live.sh b/tests/live.sh index 78da3c0..48e1e94 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -69,11 +69,7 @@ date,base,quote,amount,source,type 2021-01-07,TSLA,USD,816.0400,alphavantage,close 2021-01-08,TSLA,USD,880.0200,alphavantage,close END -if [[ "$(date --iso-8601)" < "2023-10-01" ]]; then - skip_test "$name" "$cmd" "$expected" -else - run_test "$name" "$cmd" "$expected" -fi +run_test "$name" "$cmd" "$expected" name="Alpha Vantage physical currency" cmd="pricehist fetch alphavantage AUD/EUR -s 2021-01-04 -e 2021-01-08" From 06c2876152135f87dd46850bf4d0108e19e9dccc Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 26 Aug 2023 10:57:40 +0200 Subject: [PATCH 116/149] Make AlphaVantage premium endpoint rejection message check more robust. --- src/pricehist/sources/alphavantage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index 707ba50..270f70f 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -341,7 +341,8 @@ class AlphaVantage(BaseSource): raise exceptions.RateLimit(data["Note"]) if ( "Information" in data - and "ways to unlock premium" in data["Information"] + and "unlock" in data["Information"] + and "premium" in data["Information"] ): msg = "You were denied access to a premium endpoint." raise exceptions.CredentialsError([self.API_KEY_NAME], self, msg) From 46dfd876eaa542b51ac82da283fd1bb1fccdd9ad Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 26 Aug 2023 11:00:09 +0200 Subject: [PATCH 117/149] Version 1.4.6. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 35a776c..d42be19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.5" +version = "1.4.6" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 56dadec..bde0031 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.5" +__version__ = "1.4.6" From 04936c5cd62fd2858747c15a4612c13e05549791 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 11 Jul 2024 11:40:12 +0200 Subject: [PATCH 118/149] Update lxml dependency. --- poetry.lock | 1210 ++++++++++++++++++++++++++---------------------- pyproject.toml | 2 +- 2 files changed, 654 insertions(+), 558 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9df92df..5aff69d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,32 +1,54 @@ +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + [[package]] name = "atomicwrites" version = "1.4.1" description = "Atomic file writes." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] [[package]] name = "attrs" -version = "22.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" -category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] [package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "black" -version = "22.10.0" +version = "22.12.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, + {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, + {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, + {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, + {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, + {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, + {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, + {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, + {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, + {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, + {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, + {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, +] [package.dependencies] click = ">=8.0.0" @@ -44,30 +66,124 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "certifi" -version = "2022.9.24" +version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, +] [[package]] name = "charset-normalizer" -version = "2.1.1" +version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false -python-versions = ">=3.6.0" - -[package.extras] -unicode-backport = ["unicodedata2"] +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] [[package]] name = "click" -version = "8.1.3" +version = "8.1.7" description = "Composable command line interface toolkit" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -76,417 +192,20 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "coverage" version = "5.5" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" - -[package.extras] -toml = ["toml"] - -[[package]] -name = "cssselect" -version = "1.2.0" -description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "curlify" -version = "2.2.1" -description = "Library to convert python requests object to curl command." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -requests = "*" - -[[package]] -name = "distlib" -version = "0.3.6" -description = "Distribution utilities" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "filelock" -version = "3.8.0" -description = "A platform independent file lock." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] -testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "flake8" -version = "3.9.2" -description = "the modular source code checker: pep8 pyflakes and co" -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[package.dependencies] -mccabe = ">=0.6.0,<0.7.0" -pycodestyle = ">=2.7.0,<2.8.0" -pyflakes = ">=2.3.0,<2.4.0" - -[[package]] -name = "idna" -version = "3.4" -description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "isort" -version = "5.10.1" -description = "A Python utility / library to sort Python imports." -category = "dev" -optional = false -python-versions = ">=3.6.1,<4.0" - -[package.extras] -colors = ["colorama (>=0.4.3,<0.5.0)"] -pipfile-deprecated-finder = ["pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - -[[package]] -name = "lxml" -version = "4.9.1" -description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" - -[package.extras] -cssselect = ["cssselect (>=0.7)"] -html5 = ["html5lib"] -htmlsoup = ["BeautifulSoup4"] -source = ["Cython (>=0.29.7)"] - -[[package]] -name = "mccabe" -version = "0.6.1" -description = "McCabe checker, plugin for flake8" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "packaging" -version = "21.3" -description = "Core utilities for Python packages" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" - -[[package]] -name = "pathspec" -version = "0.10.2" -description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "platformdirs" -version = "2.5.4" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] -test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] - -[[package]] -name = "pluggy" -version = "1.0.0" -description = "plugin and hook calling mechanisms for python" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[[package]] -name = "pycodestyle" -version = "2.7.0" -description = "Python style guide checker" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pyflakes" -version = "2.3.1" -description = "passive checker of Python programs" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "dev" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - -[[package]] -name = "pytest" -version = "6.2.5" -description = "pytest: simple powerful testing with Python" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=19.2.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -toml = "*" - -[package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] - -[[package]] -name = "pytest-mock" -version = "3.10.0" -description = "Thin-wrapper around the mock package for easier use with pytest" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -pytest = ">=5.0" - -[package.extras] -dev = ["pre-commit", "pytest-asyncio", "tox"] - -[[package]] -name = "requests" -version = "2.28.1" -description = "Python HTTP for Humans." -category = "main" -optional = false -python-versions = ">=3.7, <4" - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<3" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "responses" -version = "0.13.4" -description = "A utility library for mocking out the `requests` Python library." -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.dependencies] -requests = ">=2.0" -six = "*" -urllib3 = ">=1.25.10" - -[package.extras] -tests = ["coverage (>=3.7.1,<6.0.0)", "flake8", "mypy", "pytest (>=4.6)", "pytest (>=4.6,<5.0)", "pytest-cov", "pytest-localserver", "types-mock", "types-requests", "types-six"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "tox" -version = "3.27.1" -description = "tox is a generic virtualenv management and test command line tool" -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[package.dependencies] -colorama = {version = ">=0.4.1", markers = "platform_system == \"Windows\""} -filelock = ">=3.0.0" -packaging = ">=14" -pluggy = ">=0.12.0" -py = ">=1.4.17" -six = ">=1.14.0" -tomli = {version = ">=2.0.1", markers = "python_version >= \"3.7\" and python_version < \"3.11\""} -virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2,<20.0.3 || >20.0.3,<20.0.4 || >20.0.4,<20.0.5 || >20.0.5,<20.0.6 || >20.0.6,<20.0.7 || >20.0.7" - -[package.extras] -docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"] -testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)"] - -[[package]] -name = "typing-extensions" -version = "4.4.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "urllib3" -version = "1.26.13" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "virtualenv" -version = "20.16.7" -description = "Virtual Python Environment builder" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -distlib = ">=0.3.6,<1" -filelock = ">=3.4.1,<4" -platformdirs = ">=2.4,<3" - -[package.extras] -docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] -testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.8" -content-hash = "c8dc7901eea7c89dca425872abe275d5a1fc8d6f1a1d39ebf7aa7cd632d7dc44" - -[metadata.files] -atomicwrites = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] -black = [ - {file = "black-22.10.0-1fixedarch-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:5cc42ca67989e9c3cf859e84c2bf014f6633db63d1cbdf8fdb666dcd9e77e3fa"}, - {file = "black-22.10.0-1fixedarch-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:5d8f74030e67087b219b032aa33a919fae8806d49c867846bfacde57f43972ef"}, - {file = "black-22.10.0-1fixedarch-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:197df8509263b0b8614e1df1756b1dd41be6738eed2ba9e9769f3880c2b9d7b6"}, - {file = "black-22.10.0-1fixedarch-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:2644b5d63633702bc2c5f3754b1b475378fbbfb481f62319388235d0cd104c2d"}, - {file = "black-22.10.0-1fixedarch-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:e41a86c6c650bcecc6633ee3180d80a025db041a8e2398dcc059b3afa8382cd4"}, - {file = "black-22.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2039230db3c6c639bd84efe3292ec7b06e9214a2992cd9beb293d639c6402edb"}, - {file = "black-22.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14ff67aec0a47c424bc99b71005202045dc09270da44a27848d534600ac64fc7"}, - {file = "black-22.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:819dc789f4498ecc91438a7de64427c73b45035e2e3680c92e18795a839ebb66"}, - {file = "black-22.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b9b29da4f564ba8787c119f37d174f2b69cdfdf9015b7d8c5c16121ddc054ae"}, - {file = "black-22.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8b49776299fece66bffaafe357d929ca9451450f5466e997a7285ab0fe28e3b"}, - {file = "black-22.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:21199526696b8f09c3997e2b4db8d0b108d801a348414264d2eb8eb2532e540d"}, - {file = "black-22.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e464456d24e23d11fced2bc8c47ef66d471f845c7b7a42f3bd77bf3d1789650"}, - {file = "black-22.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9311e99228ae10023300ecac05be5a296f60d2fd10fff31cf5c1fa4ca4b1988d"}, - {file = "black-22.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fba8a281e570adafb79f7755ac8721b6cf1bbf691186a287e990c7929c7692ff"}, - {file = "black-22.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:915ace4ff03fdfff953962fa672d44be269deb2eaf88499a0f8805221bc68c87"}, - {file = "black-22.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:444ebfb4e441254e87bad00c661fe32df9969b2bf224373a448d8aca2132b395"}, - {file = "black-22.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:974308c58d057a651d182208a484ce80a26dac0caef2895836a92dd6ebd725e0"}, - {file = "black-22.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72ef3925f30e12a184889aac03d77d031056860ccae8a1e519f6cbb742736383"}, - {file = "black-22.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:432247333090c8c5366e69627ccb363bc58514ae3e63f7fc75c54b1ea80fa7de"}, - {file = "black-22.10.0-py3-none-any.whl", hash = "sha256:c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458"}, - {file = "black-22.10.0.tar.gz", hash = "sha256:f513588da599943e0cde4e32cc9879e825d58720d6557062d1098c5ad80080e1"}, -] -certifi = [ - {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, - {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -coverage = [ +files = [ {file = "coverage-5.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf"}, {file = "coverage-5.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b"}, {file = "coverage-5.5-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:c2723d347ab06e7ddad1a58b2a821218239249a9e4365eaff6649d31180c1669"}, @@ -540,190 +259,567 @@ coverage = [ {file = "coverage-5.5-pp37-none-any.whl", hash = "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4"}, {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"}, ] -cssselect = [ + +[package.extras] +toml = ["toml"] + +[[package]] +name = "cssselect" +version = "1.2.0" +description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0" +optional = false +python-versions = ">=3.7" +files = [ {file = "cssselect-1.2.0-py2.py3-none-any.whl", hash = "sha256:da1885f0c10b60c03ed5eccbb6b68d6eff248d91976fcde348f395d54c9fd35e"}, {file = "cssselect-1.2.0.tar.gz", hash = "sha256:666b19839cfaddb9ce9d36bfe4c969132c647b92fc9088c4e23f786b30f1b3dc"}, ] -curlify = [ + +[[package]] +name = "curlify" +version = "2.2.1" +description = "Library to convert python requests object to curl command." +optional = false +python-versions = "*" +files = [ {file = "curlify-2.2.1.tar.gz", hash = "sha256:0d3f02e7235faf952de8ef45ef469845196d30632d5838bcd5aee217726ddd6d"}, ] -distlib = [ - {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, - {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, + +[package.dependencies] +requests = "*" + +[[package]] +name = "distlib" +version = "0.3.8" +description = "Distribution utilities" +optional = false +python-versions = "*" +files = [ + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, ] -filelock = [ - {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, - {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, + +[[package]] +name = "filelock" +version = "3.15.4" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.15.4-py3-none-any.whl", hash = "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"}, + {file = "filelock-3.15.4.tar.gz", hash = "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb"}, ] -flake8 = [ + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-asyncio (>=0.21)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)", "virtualenv (>=20.26.2)"] +typing = ["typing-extensions (>=4.8)"] + +[[package]] +name = "flake8" +version = "3.9.2" +description = "the modular source code checker: pep8 pyflakes and co" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] -idna = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + +[package.dependencies] +mccabe = ">=0.6.0,<0.7.0" +pycodestyle = ">=2.7.0,<2.8.0" +pyflakes = ">=2.3.0,<2.4.0" + +[[package]] +name = "idna" +version = "3.7" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] -iniconfig = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -isort = [ - {file = "isort-5.10.1-py3-none-any.whl", hash = "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7"}, - {file = "isort-5.10.1.tar.gz", hash = "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"}, + +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, ] -lxml = [ - {file = "lxml-4.9.1-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed"}, - {file = "lxml-4.9.1-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc"}, - {file = "lxml-4.9.1-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc"}, - {file = "lxml-4.9.1-cp27-cp27m-win32.whl", hash = "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3"}, - {file = "lxml-4.9.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627"}, - {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84"}, - {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837"}, - {file = "lxml-4.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad"}, - {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5"}, - {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8"}, - {file = "lxml-4.9.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8"}, - {file = "lxml-4.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d"}, - {file = "lxml-4.9.1-cp310-cp310-win32.whl", hash = "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7"}, - {file = "lxml-4.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b"}, - {file = "lxml-4.9.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d"}, - {file = "lxml-4.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3"}, - {file = "lxml-4.9.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29"}, - {file = "lxml-4.9.1-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d"}, - {file = "lxml-4.9.1-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318"}, - {file = "lxml-4.9.1-cp35-cp35m-win32.whl", hash = "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7"}, - {file = "lxml-4.9.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4"}, - {file = "lxml-4.9.1-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb"}, - {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067"}, - {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536"}, - {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8"}, - {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b"}, - {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf"}, - {file = "lxml-4.9.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3"}, - {file = "lxml-4.9.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391"}, - {file = "lxml-4.9.1-cp36-cp36m-win32.whl", hash = "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e"}, - {file = "lxml-4.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7"}, - {file = "lxml-4.9.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2"}, - {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc"}, - {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c"}, - {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4"}, - {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3"}, - {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca"}, - {file = "lxml-4.9.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785"}, - {file = "lxml-4.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785"}, - {file = "lxml-4.9.1-cp37-cp37m-win32.whl", hash = "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a"}, - {file = "lxml-4.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e"}, - {file = "lxml-4.9.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b"}, - {file = "lxml-4.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97"}, - {file = "lxml-4.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21"}, - {file = "lxml-4.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2"}, - {file = "lxml-4.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130"}, - {file = "lxml-4.9.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715"}, - {file = "lxml-4.9.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036"}, - {file = "lxml-4.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387"}, - {file = "lxml-4.9.1-cp38-cp38-win32.whl", hash = "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94"}, - {file = "lxml-4.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345"}, - {file = "lxml-4.9.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67"}, - {file = "lxml-4.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb"}, - {file = "lxml-4.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448"}, - {file = "lxml-4.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7"}, - {file = "lxml-4.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91"}, - {file = "lxml-4.9.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000"}, - {file = "lxml-4.9.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25"}, - {file = "lxml-4.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd"}, - {file = "lxml-4.9.1-cp39-cp39-win32.whl", hash = "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb"}, - {file = "lxml-4.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d"}, - {file = "lxml-4.9.1-pp37-pypy37_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c"}, - {file = "lxml-4.9.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b"}, - {file = "lxml-4.9.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc"}, - {file = "lxml-4.9.1-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b"}, - {file = "lxml-4.9.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2"}, - {file = "lxml-4.9.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73"}, - {file = "lxml-4.9.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c"}, - {file = "lxml-4.9.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9"}, - {file = "lxml-4.9.1.tar.gz", hash = "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f"}, + +[package.extras] +colors = ["colorama (>=0.4.6)"] + +[[package]] +name = "lxml" +version = "5.2.2" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.6" +files = [ + {file = "lxml-5.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:364d03207f3e603922d0d3932ef363d55bbf48e3647395765f9bfcbdf6d23632"}, + {file = "lxml-5.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50127c186f191b8917ea2fb8b206fbebe87fd414a6084d15568c27d0a21d60db"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74e4f025ef3db1c6da4460dd27c118d8cd136d0391da4e387a15e48e5c975147"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:981a06a3076997adf7c743dcd0d7a0415582661e2517c7d961493572e909aa1d"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aef5474d913d3b05e613906ba4090433c515e13ea49c837aca18bde190853dff"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e275ea572389e41e8b039ac076a46cb87ee6b8542df3fff26f5baab43713bca"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5b65529bb2f21ac7861a0e94fdbf5dc0daab41497d18223b46ee8515e5ad297"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bcc98f911f10278d1daf14b87d65325851a1d29153caaf146877ec37031d5f36"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:b47633251727c8fe279f34025844b3b3a3e40cd1b198356d003aa146258d13a2"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:fbc9d316552f9ef7bba39f4edfad4a734d3d6f93341232a9dddadec4f15d425f"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:13e69be35391ce72712184f69000cda04fc89689429179bc4c0ae5f0b7a8c21b"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3b6a30a9ab040b3f545b697cb3adbf3696c05a3a68aad172e3fd7ca73ab3c835"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a233bb68625a85126ac9f1fc66d24337d6e8a0f9207b688eec2e7c880f012ec0"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:dfa7c241073d8f2b8e8dbc7803c434f57dbb83ae2a3d7892dd068d99e96efe2c"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1a7aca7964ac4bb07680d5c9d63b9d7028cace3e2d43175cb50bba8c5ad33316"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ae4073a60ab98529ab8a72ebf429f2a8cc612619a8c04e08bed27450d52103c0"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ffb2be176fed4457e445fe540617f0252a72a8bc56208fd65a690fdb1f57660b"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e290d79a4107d7d794634ce3e985b9ae4f920380a813717adf61804904dc4393"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96e85aa09274955bb6bd483eaf5b12abadade01010478154b0ec70284c1b1526"}, + {file = "lxml-5.2.2-cp310-cp310-win32.whl", hash = "sha256:f956196ef61369f1685d14dad80611488d8dc1ef00be57c0c5a03064005b0f30"}, + {file = "lxml-5.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:875a3f90d7eb5c5d77e529080d95140eacb3c6d13ad5b616ee8095447b1d22e7"}, + {file = "lxml-5.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:45f9494613160d0405682f9eee781c7e6d1bf45f819654eb249f8f46a2c22545"}, + {file = "lxml-5.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0b3f2df149efb242cee2ffdeb6674b7f30d23c9a7af26595099afaf46ef4e88"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d28cb356f119a437cc58a13f8135ab8a4c8ece18159eb9194b0d269ec4e28083"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:657a972f46bbefdbba2d4f14413c0d079f9ae243bd68193cb5061b9732fa54c1"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b74b9ea10063efb77a965a8d5f4182806fbf59ed068b3c3fd6f30d2ac7bee734"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07542787f86112d46d07d4f3c4e7c760282011b354d012dc4141cc12a68cef5f"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:303f540ad2dddd35b92415b74b900c749ec2010e703ab3bfd6660979d01fd4ed"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:2eb2227ce1ff998faf0cd7fe85bbf086aa41dfc5af3b1d80867ecfe75fb68df3"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:1d8a701774dfc42a2f0b8ccdfe7dbc140500d1049e0632a611985d943fcf12df"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:56793b7a1a091a7c286b5f4aa1fe4ae5d1446fe742d00cdf2ffb1077865db10d"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eb00b549b13bd6d884c863554566095bf6fa9c3cecb2e7b399c4bc7904cb33b5"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a2569a1f15ae6c8c64108a2cd2b4a858fc1e13d25846be0666fc144715e32ab"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:8cf85a6e40ff1f37fe0f25719aadf443686b1ac7652593dc53c7ef9b8492b115"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:d237ba6664b8e60fd90b8549a149a74fcc675272e0e95539a00522e4ca688b04"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0b3f5016e00ae7630a4b83d0868fca1e3d494c78a75b1c7252606a3a1c5fc2ad"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:23441e2b5339bc54dc949e9e675fa35efe858108404ef9aa92f0456929ef6fe8"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2fb0ba3e8566548d6c8e7dd82a8229ff47bd8fb8c2da237607ac8e5a1b8312e5"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:79d1fb9252e7e2cfe4de6e9a6610c7cbb99b9708e2c3e29057f487de5a9eaefa"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6dcc3d17eac1df7859ae01202e9bb11ffa8c98949dcbeb1069c8b9a75917e01b"}, + {file = "lxml-5.2.2-cp311-cp311-win32.whl", hash = "sha256:4c30a2f83677876465f44c018830f608fa3c6a8a466eb223535035fbc16f3438"}, + {file = "lxml-5.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:49095a38eb333aaf44c06052fd2ec3b8f23e19747ca7ec6f6c954ffea6dbf7be"}, + {file = "lxml-5.2.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:7429e7faa1a60cad26ae4227f4dd0459efde239e494c7312624ce228e04f6391"}, + {file = "lxml-5.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:50ccb5d355961c0f12f6cf24b7187dbabd5433f29e15147a67995474f27d1776"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc911208b18842a3a57266d8e51fc3cfaccee90a5351b92079beed912a7914c2"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33ce9e786753743159799fdf8e92a5da351158c4bfb6f2db0bf31e7892a1feb5"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ec87c44f619380878bd49ca109669c9f221d9ae6883a5bcb3616785fa8f94c97"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08ea0f606808354eb8f2dfaac095963cb25d9d28e27edcc375d7b30ab01abbf6"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75a9632f1d4f698b2e6e2e1ada40e71f369b15d69baddb8968dcc8e683839b18"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:74da9f97daec6928567b48c90ea2c82a106b2d500f397eeb8941e47d30b1ca85"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:0969e92af09c5687d769731e3f39ed62427cc72176cebb54b7a9d52cc4fa3b73"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:9164361769b6ca7769079f4d426a41df6164879f7f3568be9086e15baca61466"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d26a618ae1766279f2660aca0081b2220aca6bd1aa06b2cf73f07383faf48927"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab67ed772c584b7ef2379797bf14b82df9aa5f7438c5b9a09624dd834c1c1aaf"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:3d1e35572a56941b32c239774d7e9ad724074d37f90c7a7d499ab98761bd80cf"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:8268cbcd48c5375f46e000adb1390572c98879eb4f77910c6053d25cc3ac2c67"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e282aedd63c639c07c3857097fc0e236f984ceb4089a8b284da1c526491e3f3d"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfdc2bfe69e9adf0df4915949c22a25b39d175d599bf98e7ddf620a13678585"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4aefd911793b5d2d7a921233a54c90329bf3d4a6817dc465f12ffdfe4fc7b8fe"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:8b8df03a9e995b6211dafa63b32f9d405881518ff1ddd775db4e7b98fb545e1c"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f11ae142f3a322d44513de1018b50f474f8f736bc3cd91d969f464b5bfef8836"}, + {file = "lxml-5.2.2-cp312-cp312-win32.whl", hash = "sha256:16a8326e51fcdffc886294c1e70b11ddccec836516a343f9ed0f82aac043c24a"}, + {file = "lxml-5.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:bbc4b80af581e18568ff07f6395c02114d05f4865c2812a1f02f2eaecf0bfd48"}, + {file = "lxml-5.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e3d9d13603410b72787579769469af730c38f2f25505573a5888a94b62b920f8"}, + {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38b67afb0a06b8575948641c1d6d68e41b83a3abeae2ca9eed2ac59892b36706"}, + {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c689d0d5381f56de7bd6966a4541bff6e08bf8d3871bbd89a0c6ab18aa699573"}, + {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:cf2a978c795b54c539f47964ec05e35c05bd045db5ca1e8366988c7f2fe6b3ce"}, + {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:739e36ef7412b2bd940f75b278749106e6d025e40027c0b94a17ef7968d55d56"}, + {file = "lxml-5.2.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d8bbcd21769594dbba9c37d3c819e2d5847656ca99c747ddb31ac1701d0c0ed9"}, + {file = "lxml-5.2.2-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:2304d3c93f2258ccf2cf7a6ba8c761d76ef84948d87bf9664e14d203da2cd264"}, + {file = "lxml-5.2.2-cp36-cp36m-win32.whl", hash = "sha256:02437fb7308386867c8b7b0e5bc4cd4b04548b1c5d089ffb8e7b31009b961dc3"}, + {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, + {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, + {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, + {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, + {file = "lxml-5.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7ed07b3062b055d7a7f9d6557a251cc655eed0b3152b76de619516621c56f5d3"}, + {file = "lxml-5.2.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f60fdd125d85bf9c279ffb8e94c78c51b3b6a37711464e1f5f31078b45002421"}, + {file = "lxml-5.2.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a7e24cb69ee5f32e003f50e016d5fde438010c1022c96738b04fc2423e61706"}, + {file = "lxml-5.2.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23cfafd56887eaed93d07bc4547abd5e09d837a002b791e9767765492a75883f"}, + {file = "lxml-5.2.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:19b4e485cd07b7d83e3fe3b72132e7df70bfac22b14fe4bf7a23822c3a35bff5"}, + {file = "lxml-5.2.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:7ce7ad8abebe737ad6143d9d3bf94b88b93365ea30a5b81f6877ec9c0dee0a48"}, + {file = "lxml-5.2.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e49b052b768bb74f58c7dda4e0bdf7b79d43a9204ca584ffe1fb48a6f3c84c66"}, + {file = "lxml-5.2.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d14a0d029a4e176795cef99c056d58067c06195e0c7e2dbb293bf95c08f772a3"}, + {file = "lxml-5.2.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:be49ad33819d7dcc28a309b86d4ed98e1a65f3075c6acd3cd4fe32103235222b"}, + {file = "lxml-5.2.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:a6d17e0370d2516d5bb9062c7b4cb731cff921fc875644c3d751ad857ba9c5b1"}, + {file = "lxml-5.2.2-cp38-cp38-win32.whl", hash = "sha256:5b8c041b6265e08eac8a724b74b655404070b636a8dd6d7a13c3adc07882ef30"}, + {file = "lxml-5.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:f61efaf4bed1cc0860e567d2ecb2363974d414f7f1f124b1df368bbf183453a6"}, + {file = "lxml-5.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:fb91819461b1b56d06fa4bcf86617fac795f6a99d12239fb0c68dbeba41a0a30"}, + {file = "lxml-5.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d4ed0c7cbecde7194cd3228c044e86bf73e30a23505af852857c09c24e77ec5d"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54401c77a63cc7d6dc4b4e173bb484f28a5607f3df71484709fe037c92d4f0ed"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:625e3ef310e7fa3a761d48ca7ea1f9d8718a32b1542e727d584d82f4453d5eeb"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:519895c99c815a1a24a926d5b60627ce5ea48e9f639a5cd328bda0515ea0f10c"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c7079d5eb1c1315a858bbf180000757db8ad904a89476653232db835c3114001"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:343ab62e9ca78094f2306aefed67dcfad61c4683f87eee48ff2fd74902447726"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:cd9e78285da6c9ba2d5c769628f43ef66d96ac3085e59b10ad4f3707980710d3"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:546cf886f6242dff9ec206331209db9c8e1643ae642dea5fdbecae2453cb50fd"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:02f6a8eb6512fdc2fd4ca10a49c341c4e109aa6e9448cc4859af5b949622715a"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:339ee4a4704bc724757cd5dd9dc8cf4d00980f5d3e6e06d5847c1b594ace68ab"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0a028b61a2e357ace98b1615fc03f76eb517cc028993964fe08ad514b1e8892d"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f90e552ecbad426eab352e7b2933091f2be77115bb16f09f78404861c8322981"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d83e2d94b69bf31ead2fa45f0acdef0757fa0458a129734f59f67f3d2eb7ef32"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a02d3c48f9bb1e10c7788d92c0c7db6f2002d024ab6e74d6f45ae33e3d0288a3"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6d68ce8e7b2075390e8ac1e1d3a99e8b6372c694bbe612632606d1d546794207"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:453d037e09a5176d92ec0fd282e934ed26d806331a8b70ab431a81e2fbabf56d"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:3b019d4ee84b683342af793b56bb35034bd749e4cbdd3d33f7d1107790f8c472"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cb3942960f0beb9f46e2a71a3aca220d1ca32feb5a398656be934320804c0df9"}, + {file = "lxml-5.2.2-cp39-cp39-win32.whl", hash = "sha256:ac6540c9fff6e3813d29d0403ee7a81897f1d8ecc09a8ff84d2eea70ede1cdbf"}, + {file = "lxml-5.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:610b5c77428a50269f38a534057444c249976433f40f53e3b47e68349cca1425"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b537bd04d7ccd7c6350cdaaaad911f6312cbd61e6e6045542f781c7f8b2e99d2"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4820c02195d6dfb7b8508ff276752f6b2ff8b64ae5d13ebe02e7667e035000b9"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a09f6184f17a80897172863a655467da2b11151ec98ba8d7af89f17bf63dae"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:76acba4c66c47d27c8365e7c10b3d8016a7da83d3191d053a58382311a8bf4e1"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b128092c927eaf485928cec0c28f6b8bead277e28acf56800e972aa2c2abd7a2"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ae791f6bd43305aade8c0e22f816b34f3b72b6c820477aab4d18473a37e8090b"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a2f6a1bc2460e643785a2cde17293bd7a8f990884b822f7bca47bee0a82fc66b"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e8d351ff44c1638cb6e980623d517abd9f580d2e53bfcd18d8941c052a5a009"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bec4bd9133420c5c52d562469c754f27c5c9e36ee06abc169612c959bd7dbb07"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:55ce6b6d803890bd3cc89975fca9de1dff39729b43b73cb15ddd933b8bc20484"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ab6a358d1286498d80fe67bd3d69fcbc7d1359b45b41e74c4a26964ca99c3f8"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:06668e39e1f3c065349c51ac27ae430719d7806c026fec462e5693b08b95696b"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9cd5323344d8ebb9fb5e96da5de5ad4ebab993bbf51674259dbe9d7a18049525"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89feb82ca055af0fe797a2323ec9043b26bc371365847dbe83c7fd2e2f181c34"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e481bba1e11ba585fb06db666bfc23dbe181dbafc7b25776156120bf12e0d5a6"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d6c6ea6a11ca0ff9cd0390b885984ed31157c168565702959c25e2191674a14"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3d98de734abee23e61f6b8c2e08a88453ada7d6486dc7cdc82922a03968928db"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:69ab77a1373f1e7563e0fb5a29a8440367dec051da6c7405333699d07444f511"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:34e17913c431f5ae01d8658dbf792fdc457073dcdfbb31dc0cc6ab256e664a8d"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05f8757b03208c3f50097761be2dea0aba02e94f0dc7023ed73a7bb14ff11eb0"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a520b4f9974b0a0a6ed73c2154de57cdfd0c8800f4f15ab2b73238ffed0b36e"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5e097646944b66207023bc3c634827de858aebc226d5d4d6d16f0b77566ea182"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b5e4ef22ff25bfd4ede5f8fb30f7b24446345f3e79d9b7455aef2836437bc38a"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ff69a9a0b4b17d78170c73abe2ab12084bdf1691550c5629ad1fe7849433f324"}, + {file = "lxml-5.2.2.tar.gz", hash = "sha256:bb2dc4898180bea79863d5487e5f9c7c34297414bad54bcd0f0852aee9cfdb87"}, ] -mccabe = [ + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml-html-clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.10)"] + +[[package]] +name = "mccabe" +version = "0.6.1" +description = "McCabe checker, plugin for flake8" +optional = false +python-versions = "*" +files = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] -mypy-extensions = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] -packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, + +[[package]] +name = "packaging" +version = "24.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] -pathspec = [ - {file = "pathspec-0.10.2-py3-none-any.whl", hash = "sha256:88c2606f2c1e818b978540f73ecc908e13999c6c3a383daf3705652ae79807a5"}, - {file = "pathspec-0.10.2.tar.gz", hash = "sha256:8f6bf73e5758fd365ef5d58ce09ac7c27d2833a8d7da51712eac6e27e35141b0"}, + +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] -platformdirs = [ - {file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"}, - {file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"}, + +[[package]] +name = "platformdirs" +version = "4.2.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, ] -pluggy = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] -py = [ + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pycodestyle = [ + +[[package]] +name = "pycodestyle" +version = "2.7.0" +description = "Python style guide checker" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, ] -pyflakes = [ + +[[package]] +name = "pyflakes" +version = "2.3.1" +description = "passive checker of Python programs" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, ] -pyparsing = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] -pytest = [ + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] -pytest-mock = [ - {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, - {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, ] -requests = [ - {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, - {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] -responses = [ + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "responses" +version = "0.13.4" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ {file = "responses-0.13.4-py2.py3-none-any.whl", hash = "sha256:d8d0f655710c46fd3513b9202a7f0dcedd02ca0f8cf4976f27fa8ab5b81e656d"}, {file = "responses-0.13.4.tar.gz", hash = "sha256:9476775d856d3c24ae660bbebe29fb6d789d4ad16acd723efbfb6ee20990b899"}, ] -six = [ + +[package.dependencies] +requests = ">=2.0" +six = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=3.7.1,<6.0.0)", "flake8", "mypy", "pytest (>=4.6)", "pytest (>=4.6,<5.0)", "pytest-cov", "pytest-localserver", "types-mock", "types-requests", "types-six"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -toml = [ + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] -tomli = [ + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -tox = [ - {file = "tox-3.27.1-py2.py3-none-any.whl", hash = "sha256:f52ca66eae115fcfef0e77ef81fd107133d295c97c52df337adedb8dfac6ab84"}, - {file = "tox-3.27.1.tar.gz", hash = "sha256:b2a920e35a668cc06942ffd1cf3a4fb221a4d909ca72191fb6d84b0b18a7be04"}, + +[[package]] +name = "tox" +version = "3.28.0" +description = "tox is a generic virtualenv management and test command line tool" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ + {file = "tox-3.28.0-py2.py3-none-any.whl", hash = "sha256:57b5ab7e8bb3074edc3c0c0b4b192a4f3799d3723b2c5b76f1fa9f2d40316eea"}, + {file = "tox-3.28.0.tar.gz", hash = "sha256:d0d28f3fe6d6d7195c27f8b054c3e99d5451952b54abdae673b71609a581f640"}, ] -typing-extensions = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, + +[package.dependencies] +colorama = {version = ">=0.4.1", markers = "platform_system == \"Windows\""} +filelock = ">=3.0.0" +packaging = ">=14" +pluggy = ">=0.12.0" +py = ">=1.4.17" +six = ">=1.14.0" +tomli = {version = ">=2.0.1", markers = "python_version >= \"3.7\" and python_version < \"3.11\""} +virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2,<20.0.3 || >20.0.3,<20.0.4 || >20.0.4,<20.0.5 || >20.0.5,<20.0.6 || >20.0.6,<20.0.7 || >20.0.7" + +[package.extras] +docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"] +testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)"] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] -urllib3 = [ - {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, - {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, + +[[package]] +name = "urllib3" +version = "2.2.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, ] -virtualenv = [ - {file = "virtualenv-20.16.7-py3-none-any.whl", hash = "sha256:efd66b00386fdb7dbe4822d172303f40cd05e50e01740b19ea42425cbe653e29"}, - {file = "virtualenv-20.16.7.tar.gz", hash = "sha256:8691e3ff9387f743e00f6bb20f70121f5e4f596cae754531f2b3b3a1b1ac696e"}, + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "virtualenv" +version = "20.26.3" +description = "Virtual Python Environment builder" +optional = false +python-versions = ">=3.7" +files = [ + {file = "virtualenv-20.26.3-py3-none-any.whl", hash = "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589"}, + {file = "virtualenv-20.26.3.tar.gz", hash = "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a"}, ] + +[package.dependencies] +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<5" + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "98395da76c16ddf7b7c3c9ae128f47b574e7d6775ddbc7f9b61ceea31492ab92" diff --git a/pyproject.toml b/pyproject.toml index d42be19..0840342 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ include = [ [tool.poetry.dependencies] python = "^3.8" requests = "^2.25.1" -lxml = "^4.6.2" +lxml = "^5.1.0" cssselect = "^1.1.0" curlify = "^2.2.1" From 6519cf28454a5af6097acfdb75f95cf7a22c3ce7 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 11 Jul 2024 11:40:38 +0200 Subject: [PATCH 119/149] Update datetime formatting. --- src/pricehist/outputs/gnucashsql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pricehist/outputs/gnucashsql.py b/src/pricehist/outputs/gnucashsql.py index 44841c1..8ccbff7 100644 --- a/src/pricehist/outputs/gnucashsql.py +++ b/src/pricehist/outputs/gnucashsql.py @@ -40,7 +40,7 @@ Classes: import hashlib import logging -from datetime import datetime +from datetime import datetime, timezone from decimal import Decimal from importlib.resources import files @@ -125,7 +125,7 @@ class GnuCashSQL(BaseOutput): .read_text() .format( version=__version__, - timestamp=datetime.utcnow().isoformat() + "Z", + timestamp=datetime.now(timezone.utc).isoformat()[:-6] + "Z", base=self._sql_str(base), quote=self._sql_str(quote), values_comment=values_comment, From 96d3e44738d194e0c85a94b88a11b4f324c7eb99 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 11 Jul 2024 14:56:28 +0200 Subject: [PATCH 120/149] Update python to ^3.8.1 and flake8 to ^7.1.0. --- poetry.lock | 42 +++++++++++++++++++++--------------------- pyproject.toml | 4 ++-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5aff69d..9d3681b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -316,19 +316,19 @@ typing = ["typing-extensions (>=4.8)"] [[package]] name = "flake8" -version = "3.9.2" +version = "7.1.0" description = "the modular source code checker: pep8 pyflakes and co" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.8.1" files = [ - {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, - {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, + {file = "flake8-7.1.0-py2.py3-none-any.whl", hash = "sha256:2e416edcc62471a64cea09353f4e7bdba32aeb079b6e360554c659a122b1bc6a"}, + {file = "flake8-7.1.0.tar.gz", hash = "sha256:48a07b626b55236e0fb4784ee69a465fbf59d79eec1f5b4785c3d3bc57d17aa5"}, ] [package.dependencies] -mccabe = ">=0.6.0,<0.7.0" -pycodestyle = ">=2.7.0,<2.8.0" -pyflakes = ">=2.3.0,<2.4.0" +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.12.0,<2.13.0" +pyflakes = ">=3.2.0,<3.3.0" [[package]] name = "idna" @@ -526,13 +526,13 @@ source = ["Cython (>=3.0.10)"] [[package]] name = "mccabe" -version = "0.6.1" +version = "0.7.0" description = "McCabe checker, plugin for flake8" optional = false -python-versions = "*" +python-versions = ">=3.6" files = [ - {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, - {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] [[package]] @@ -612,24 +612,24 @@ files = [ [[package]] name = "pycodestyle" -version = "2.7.0" +version = "2.12.0" description = "Python style guide checker" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.8" files = [ - {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, - {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, + {file = "pycodestyle-2.12.0-py2.py3-none-any.whl", hash = "sha256:949a39f6b86c3e1515ba1787c2022131d165a8ad271b11370a8819aa070269e4"}, + {file = "pycodestyle-2.12.0.tar.gz", hash = "sha256:442f950141b4f43df752dd303511ffded3a04c2b6fb7f65980574f0c31e6e79c"}, ] [[package]] name = "pyflakes" -version = "2.3.1" +version = "3.2.0" description = "passive checker of Python programs" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.8" files = [ - {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, - {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, + {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, + {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, ] [[package]] @@ -821,5 +821,5 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.0" -python-versions = "^3.8" -content-hash = "98395da76c16ddf7b7c3c9ae128f47b574e7d6775ddbc7f9b61ceea31492ab92" +python-versions = "^3.8.1" +content-hash = "de3fe2ed9cb9ec204b3e1f94150f17b0a27b3c13a722164e5118512691cf248d" diff --git a/pyproject.toml b/pyproject.toml index 0840342..a4d2752 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ include = [ ] [tool.poetry.dependencies] -python = "^3.8" +python = "^3.8.1" requests = "^2.25.1" lxml = "^5.1.0" cssselect = "^1.1.0" @@ -23,7 +23,7 @@ curlify = "^2.2.1" [tool.poetry.dev-dependencies] pytest = "^6.2.2" black = "^22.10.0" -flake8 = "^3.9.1" +flake8 = "^7.1.0" isort = "^5.8.0" responses = "^0.13.3" coverage = "^5.5" From 733c849286df1379782429aed7826a3749a98712 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 11 Jul 2024 14:59:02 +0200 Subject: [PATCH 121/149] Follow flake8 advice. --- src/pricehist/sources/alphavantage.py | 10 +++++----- src/pricehist/sources/coinmarketcap.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index 270f70f..ea80621 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -175,9 +175,9 @@ class AlphaVantage(BaseSource): expected_keys = ["1. symbol", "2. name", "3. type", "4. region", "8. currency"] if ( - type(data) != dict + type(data) is not dict or "bestMatches" not in data - or type(data["bestMatches"]) != list + or type(data["bestMatches"]) is not list or not all(k in m for k in expected_keys for m in data["bestMatches"]) ): raise exceptions.ResponseParsingError("Unexpected content.") @@ -267,7 +267,7 @@ class AlphaVantage(BaseSource): self._raise_for_generic_errors(data) - if type(data) != dict or "Time Series FX (Daily)" not in data: + if type(data) is not dict or "Time Series FX (Daily)" not in data: raise exceptions.ResponseParsingError("Unexpected content.") normalized_data = { @@ -308,7 +308,7 @@ class AlphaVantage(BaseSource): self._raise_for_generic_errors(data) - if type(data) != dict or "Time Series (Digital Currency Daily)" not in data: + if type(data) is not dict or "Time Series (Digital Currency Daily)" not in data: raise exceptions.ResponseParsingError("Unexpected content.") normalized_data = { @@ -336,7 +336,7 @@ class AlphaVantage(BaseSource): return key def _raise_for_generic_errors(self, data): - if type(data) == dict: + if type(data) is dict: if "Note" in data and "call frequency" in data["Note"]: raise exceptions.RateLimit(data["Note"]) if ( diff --git a/src/pricehist/sources/coinmarketcap.py b/src/pricehist/sources/coinmarketcap.py index 7da1a25..8cdf03a 100644 --- a/src/pricehist/sources/coinmarketcap.py +++ b/src/pricehist/sources/coinmarketcap.py @@ -144,7 +144,7 @@ class CoinMarketCap(BaseSource): except Exception as e: raise exceptions.ResponseParsingError(str(e)) from e - if type(parsed) != dict or "data" not in parsed: + if type(parsed) is not dict or "data" not in parsed: raise exceptions.ResponseParsingError("Unexpected content.") elif len(parsed["data"]) == 0: @@ -208,7 +208,7 @@ class CoinMarketCap(BaseSource): except Exception as e: raise exceptions.ResponseParsingError(str(e)) from e - if type(parsed) != dict or "data" not in parsed: + if type(parsed) is not dict or "data" not in parsed: raise exceptions.ResponseParsingError("Unexpected content.") elif len(parsed["data"]) == 0: From 0b377a8d65f357c040b24dabf1cad8a6f2782b11 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 11 Jul 2024 15:29:51 +0200 Subject: [PATCH 122/149] Fix description of data taht doesn't overlap the requested range. --- src/pricehist/fetch.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pricehist/fetch.py b/src/pricehist/fetch.py index db051de..a7faba7 100644 --- a/src/pricehist/fetch.py +++ b/src/pricehist/fetch.py @@ -80,5 +80,8 @@ def _cov_description( f"and ends {end_uncovered} day{s(end_uncovered)} earlier " f"than requested" ) - else: + elif start_uncovered == 0 and end_uncovered == 0: return "as requested" + else: + return "which doesn't match the request" + From f4aee183603b92440177e83ad1dba2c196c460a4 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 11 Jul 2024 15:30:45 +0200 Subject: [PATCH 123/149] Update parsing of Alphavantage digital currency response data. --- src/pricehist/sources/alphavantage.py | 8 +- .../test_alphavantage/btc-aud-partial.json | 105 ++++++------------ 2 files changed, 39 insertions(+), 74 deletions(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index ea80621..698add7 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -313,10 +313,10 @@ class AlphaVantage(BaseSource): normalized_data = { day: { - "open": entries[f"1a. open ({series.quote})"], - "high": entries[f"2a. high ({series.quote})"], - "low": entries[f"3a. low ({series.quote})"], - "close": entries[f"4a. close ({series.quote})"], + "open": entries[f"1. open"], + "high": entries[f"2. high"], + "low": entries[f"3. low"], + "close": entries[f"4. close"], } for day, entries in reversed( data["Time Series (Digital Currency Daily)"].items() diff --git a/tests/pricehist/sources/test_alphavantage/btc-aud-partial.json b/tests/pricehist/sources/test_alphavantage/btc-aud-partial.json index cd6412d..737658f 100644 --- a/tests/pricehist/sources/test_alphavantage/btc-aud-partial.json +++ b/tests/pricehist/sources/test_alphavantage/btc-aud-partial.json @@ -10,88 +10,53 @@ }, "Time Series (Digital Currency Daily)": { "2021-01-09": { - "1a. open (AUD)": "55074.06950240", - "1b. open (USD)": "40586.96000000", - "2a. high (AUD)": "56150.17720000", - "2b. high (USD)": "41380.00000000", - "3a. low (AUD)": "52540.71680000", - "3b. low (USD)": "38720.00000000", - "4a. close (AUD)": "54397.30924680", - "4b. close (USD)": "40088.22000000", - "5. volume": "75785.97967500", - "6. market cap (USD)": "75785.97967500" + "1. open": "55074.06950240", + "2. high": "56150.17720000", + "3. low": "52540.71680000", + "4. close": "54397.30924680", + "5. volume": "75785.97967500" }, "2021-01-08": { - "1a. open (AUD)": "53507.50941120", - "1b. open (USD)": "39432.48000000", - "2a. high (AUD)": "56923.63300000", - "2b. high (USD)": "41950.00000000", - "3a. low (AUD)": "49528.31000000", - "3b. low (USD)": "36500.00000000", - "4a. close (AUD)": "55068.43820140", - "4b. close (USD)": "40582.81000000", - "5. volume": "139789.95749900", - "6. market cap (USD)": "139789.95749900" + "1. open": "53507.50941120", + "2. high": "56923.63300000", + "3. low": "49528.31000000", + "4. close": "55068.43820140", + "5. volume": "139789.95749900" }, "2021-01-07": { - "1a. open (AUD)": "49893.81535840", - "1b. open (USD)": "36769.36000000", - "2a. high (AUD)": "54772.88310000", - "2b. high (USD)": "40365.00000000", - "3a. low (AUD)": "49256.92200000", - "3b. low (USD)": "36300.00000000", - "4a. close (AUD)": "53507.23802320", - "4b. close (USD)": "39432.28000000", - "5. volume": "132825.70043700", - "6. market cap (USD)": "132825.70043700" + "1. open": "49893.81535840", + "2. high": "54772.88310000", + "3. low": "49256.92200000", + "4. close": "53507.23802320", + "5. volume": "132825.70043700" }, "2021-01-06": { - "1a. open (AUD)": "46067.47523820", - "1b. open (USD)": "33949.53000000", - "2a. high (AUD)": "50124.29161740", - "2b. high (USD)": "36939.21000000", - "3a. low (AUD)": "45169.81872000", - "3b. low (USD)": "33288.00000000", - "4a. close (AUD)": "49893.81535840", - "4b. close (USD)": "36769.36000000", - "5. volume": "127139.20131000", - "6. market cap (USD)": "127139.20131000" + "1. open": "46067.47523820", + "2. high": "50124.29161740", + "3. low": "45169.81872000", + "4. close": "49893.81535840", + "5. volume": "127139.20131000" }, "2021-01-05": { - "1a. open (AUD)": "43408.17136500", - "1b. open (USD)": "31989.75000000", - "2a. high (AUD)": "46624.45840000", - "2b. high (USD)": "34360.00000000", - "3a. low (AUD)": "40572.50600000", - "3b. low (USD)": "29900.00000000", - "4a. close (AUD)": "46067.47523820", - "4b. close (USD)": "33949.53000000", - "5. volume": "116049.99703800", - "6. market cap (USD)": "116049.99703800" + "1. open": "43408.17136500", + "2. high": "46624.45840000", + "3. low": "40572.50600000", + "4. close": "46067.47523820", + "5. volume": "116049.99703800" }, "2021-01-04": { - "1a. open (AUD)": "44779.08784700", - "1b. open (USD)": "33000.05000000", - "2a. high (AUD)": "45593.18400000", - "2b. high (USD)": "33600.00000000", - "3a. low (AUD)": "38170.72220000", - "3b. low (USD)": "28130.00000000", - "4a. close (AUD)": "43406.76014740", - "4b. close (USD)": "31988.71000000", - "5. volume": "140899.88569000", - "6. market cap (USD)": "140899.88569000" + "1. open": "44779.08784700", + "2. high": "45593.18400000", + "3. low": "38170.72220000", + "4. close": "43406.76014740", + "5. volume": "140899.88569000" }, "2021-01-03": { - "1a. open (AUD)": "43661.51206300", - "1b. open (USD)": "32176.45000000", - "2a. high (AUD)": "47191.80858340", - "2b. high (USD)": "34778.11000000", - "3a. low (AUD)": "43371.85965060", - "3b. low (USD)": "31962.99000000", - "4a. close (AUD)": "44779.08784700", - "4b. close (USD)": "33000.05000000", - "5. volume": "120957.56675000", - "6. market cap (USD)": "120957.56675000" + "1. open": "43661.51206300", + "2. high": "47191.80858340", + "3. low": "43371.85965060", + "4. close": "44779.08784700", + "5. volume": "120957.56675000" } } } From 1f01c54c4dfdacc31c912565feba500d196b3a96 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 11 Jul 2024 15:31:37 +0200 Subject: [PATCH 124/149] Update alphavantage physical and digital currency live test cases. --- tests/live.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/live.sh b/tests/live.sh index 48e1e94..93bcb4e 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -72,26 +72,26 @@ END run_test "$name" "$cmd" "$expected" name="Alpha Vantage physical currency" -cmd="pricehist fetch alphavantage AUD/EUR -s 2021-01-04 -e 2021-01-08" +cmd="pricehist fetch alphavantage AUD/EUR -s 2021-01-10 -e 2021-01-14" read -r -d '' expected < Date: Thu, 11 Jul 2024 15:32:10 +0200 Subject: [PATCH 125/149] Skip live tests for sources with known issues that need more work. --- tests/live.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/live.sh b/tests/live.sh index 93bcb4e..a25b93d 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -130,7 +130,7 @@ date,base,quote,amount,source,type 2021-01-07,BTC,EUR,32183.1594,coindesk,close 2021-01-08,BTC,EUR,33238.5724,coindesk,close END -run_test "$name" "$cmd" "$expected" +skip_test "$name" "$cmd" "$expected" name="CoinMarketCap" cmd="pricehist fetch coinmarketcap BTC/EUR -s 2021-01-04 -e 2021-01-08" @@ -142,7 +142,7 @@ date,base,quote,amount,source,type 2021-01-07,BTC,EUR,31200.6391028267445,coinmarketcap,mid 2021-01-08,BTC,EUR,32154.244768031175,coinmarketcap,mid END -run_test "$name" "$cmd" "$expected" +skip_test "$name" "$cmd" "$expected" name="European Central Bank" cmd="pricehist fetch ecb EUR/JPY -s 2021-01-04 -e 2021-01-08" From a12f3d3899a2916f3ef8e080fd12560543fda0e3 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 11 Jul 2024 15:34:32 +0200 Subject: [PATCH 126/149] Version 1.4.7. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a4d2752..b6b53d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.6" +version = "1.4.7" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index bde0031..ac329c9 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.6" +__version__ = "1.4.7" From 47544a11b6a492610f300f0df733d09c7bab0f91 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 11 Jul 2024 15:52:35 +0200 Subject: [PATCH 127/149] Minor formatting. --- src/pricehist/fetch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pricehist/fetch.py b/src/pricehist/fetch.py index a7faba7..aba61f2 100644 --- a/src/pricehist/fetch.py +++ b/src/pricehist/fetch.py @@ -84,4 +84,3 @@ def _cov_description( return "as requested" else: return "which doesn't match the request" - From b8c4554298f12056344ef347b5329801493712a1 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 11 Jul 2024 16:25:30 +0200 Subject: [PATCH 128/149] Fix flake8 warning. --- src/pricehist/sources/alphavantage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index 698add7..e60f0f6 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -313,10 +313,10 @@ class AlphaVantage(BaseSource): normalized_data = { day: { - "open": entries[f"1. open"], - "high": entries[f"2. high"], - "low": entries[f"3. low"], - "close": entries[f"4. close"], + "open": entries["1. open"], + "high": entries["2. high"], + "low": entries["3. low"], + "close": entries["4. close"], } for day, entries in reversed( data["Time Series (Digital Currency Daily)"].items() From 8921653154b43e71e9647b6fcbccaf13aff9c3db Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 23 Jul 2024 22:08:22 +0200 Subject: [PATCH 129/149] Fix coinmarketcap: first pass. --- src/pricehist/sources/coinmarketcap.py | 144 ++++++++++++++++++++++--- 1 file changed, 127 insertions(+), 17 deletions(-) diff --git a/src/pricehist/sources/coinmarketcap.py b/src/pricehist/sources/coinmarketcap.py index 8cdf03a..f7e89ab 100644 --- a/src/pricehist/sources/coinmarketcap.py +++ b/src/pricehist/sources/coinmarketcap.py @@ -2,6 +2,7 @@ import dataclasses import json from datetime import datetime, timezone from decimal import Decimal +from functools import lru_cache import requests @@ -55,8 +56,8 @@ class CoinMarketCap(BaseSource): prices = [] for item in data.get("quotes", []): - d = item["time_open"][0:10] - amount = self._amount(next(iter(item["quote"].values())), series.type) + d = item["timeOpen"][0:10] + amount = self._amount(item["quote"], series.type) if amount is not None: prices.append(Price(d, amount)) @@ -67,21 +68,21 @@ class CoinMarketCap(BaseSource): ) def _data(self, series): - url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical" + url = "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical" params = {} if series.base.startswith("ID="): params["id"] = series.base[3:] else: - params["symbol"] = series.base + params["id"] = self._id_from_symbol(series.base, series) if series.quote.startswith("ID="): - params["convert_id"] = series.quote[3:] + params["convertId"] = series.quote[3:] else: - params["convert"] = series.quote + params["convertId"] = self._id_from_symbol(series.quote, series) - params["time_start"] = int( + params["timeStart"] = int( int( datetime.strptime(series.start, "%Y-%m-%d") .replace(tzinfo=timezone.utc) @@ -90,12 +91,14 @@ class CoinMarketCap(BaseSource): - 24 * 60 * 60 # Start one period earlier since the start is exclusive. ) - params["time_end"] = int( + params["timeEnd"] = int( datetime.strptime(series.end, "%Y-%m-%d") .replace(tzinfo=timezone.utc) .timestamp() ) # Don't round up since it's inclusive of the period covering the end time. + params["interval"] = "daily" + try: response = self.log_curl(requests.get(url, params=params)) except Exception as e: @@ -114,11 +117,6 @@ class CoinMarketCap(BaseSource): series.base, series.quote, self, "Bad quote ID." ) - elif code == 400 and 'Invalid value for \\"convert\\"' in text: - raise exceptions.InvalidPair( - series.base, series.quote, self, "Bad quote symbol." - ) - elif code == 400 and "must be older than" in text: if series.start <= series.end: raise exceptions.BadResponse("The start date must be in the past.") @@ -182,15 +180,127 @@ class CoinMarketCap(BaseSource): return (output_base, output_quote) + def _id_from_symbol(self, symbol, series): + for i in self._symbol_data(): + if i["symbol"] == symbol: + return i["id"] + raise exceptions.InvalidPair( + series.base, series.quote, self, f"Invalid symbol '{symbol}'." + ) + + @lru_cache(maxsize=1) def _symbol_data(self): - base_url = "https://web-api.coinmarketcap.com/v1/" - fiat_url = f"{base_url}fiat/map?include_metals=true" + + base_url = "https://api.coinmarketcap.com/data-api/v1/" crypto_url = f"{base_url}cryptocurrency/map?sort=cmc_rank" - fiat = self._get_json_data(fiat_url) crypto = self._get_json_data(crypto_url) - return crypto + fiat + # fmt: off + fiat = [ + {"id": 2781, "symbol": "USD", "name": "United States Dollar"}, + {"id": 3526, "symbol": "ALL", "name": "Albanian Lek"}, + {"id": 3537, "symbol": "DZD", "name": "Algerian Dinar"}, + {"id": 2821, "symbol": "ARS", "name": "Argentine Peso"}, + {"id": 3527, "symbol": "AMD", "name": "Armenian Dram"}, + {"id": 2782, "symbol": "AUD", "name": "Australian Dollar"}, + {"id": 3528, "symbol": "AZN", "name": "Azerbaijani Manat"}, + {"id": 3531, "symbol": "BHD", "name": "Bahraini Dinar"}, + {"id": 3530, "symbol": "BDT", "name": "Bangladeshi Taka"}, + {"id": 3533, "symbol": "BYN", "name": "Belarusian Ruble"}, + {"id": 3532, "symbol": "BMD", "name": "Bermudan Dollar"}, + {"id": 2832, "symbol": "BOB", "name": "Bolivian Boliviano"}, + {"id": 3529, "symbol": "BAM", "name": "Bosnia-Herzegovina Convertible Mark"}, # noqa: E501 + {"id": 2783, "symbol": "BRL", "name": "Brazilian Real"}, + {"id": 2814, "symbol": "BGN", "name": "Bulgarian Lev"}, + {"id": 3549, "symbol": "KHR", "name": "Cambodian Riel"}, + {"id": 2784, "symbol": "CAD", "name": "Canadian Dollar"}, + {"id": 2786, "symbol": "CLP", "name": "Chilean Peso"}, + {"id": 2787, "symbol": "CNY", "name": "Chinese Yuan"}, + {"id": 2820, "symbol": "COP", "name": "Colombian Peso"}, + {"id": 3534, "symbol": "CRC", "name": "Costa Rican Colón"}, + {"id": 2815, "symbol": "HRK", "name": "Croatian Kuna"}, + {"id": 3535, "symbol": "CUP", "name": "Cuban Peso"}, + {"id": 2788, "symbol": "CZK", "name": "Czech Koruna"}, + {"id": 2789, "symbol": "DKK", "name": "Danish Krone"}, + {"id": 3536, "symbol": "DOP", "name": "Dominican Peso"}, + {"id": 3538, "symbol": "EGP", "name": "Egyptian Pound"}, + {"id": 2790, "symbol": "EUR", "name": "Euro"}, + {"id": 3539, "symbol": "GEL", "name": "Georgian Lari"}, + {"id": 3540, "symbol": "GHS", "name": "Ghanaian Cedi"}, + {"id": 3541, "symbol": "GTQ", "name": "Guatemalan Quetzal"}, + {"id": 3542, "symbol": "HNL", "name": "Honduran Lempira"}, + {"id": 2792, "symbol": "HKD", "name": "Hong Kong Dollar"}, + {"id": 2793, "symbol": "HUF", "name": "Hungarian Forint"}, + {"id": 2818, "symbol": "ISK", "name": "Icelandic Króna"}, + {"id": 2796, "symbol": "INR", "name": "Indian Rupee"}, + {"id": 2794, "symbol": "IDR", "name": "Indonesian Rupiah"}, + {"id": 3544, "symbol": "IRR", "name": "Iranian Rial"}, + {"id": 3543, "symbol": "IQD", "name": "Iraqi Dinar"}, + {"id": 2795, "symbol": "ILS", "name": "Israeli New Shekel"}, + {"id": 3545, "symbol": "JMD", "name": "Jamaican Dollar"}, + {"id": 2797, "symbol": "JPY", "name": "Japanese Yen"}, + {"id": 3546, "symbol": "JOD", "name": "Jordanian Dinar"}, + {"id": 3551, "symbol": "KZT", "name": "Kazakhstani Tenge"}, + {"id": 3547, "symbol": "KES", "name": "Kenyan Shilling"}, + {"id": 3550, "symbol": "KWD", "name": "Kuwaiti Dinar"}, + {"id": 3548, "symbol": "KGS", "name": "Kyrgystani Som"}, + {"id": 3552, "symbol": "LBP", "name": "Lebanese Pound"}, + {"id": 3556, "symbol": "MKD", "name": "Macedonian Denar"}, + {"id": 2800, "symbol": "MYR", "name": "Malaysian Ringgit"}, + {"id": 2816, "symbol": "MUR", "name": "Mauritian Rupee"}, + {"id": 2799, "symbol": "MXN", "name": "Mexican Peso"}, + {"id": 3555, "symbol": "MDL", "name": "Moldovan Leu"}, + {"id": 3558, "symbol": "MNT", "name": "Mongolian Tugrik"}, + {"id": 3554, "symbol": "MAD", "name": "Moroccan Dirham"}, + {"id": 3557, "symbol": "MMK", "name": "Myanma Kyat"}, + {"id": 3559, "symbol": "NAD", "name": "Namibian Dollar"}, + {"id": 3561, "symbol": "NPR", "name": "Nepalese Rupee"}, + {"id": 2811, "symbol": "TWD", "name": "New Taiwan Dollar"}, + {"id": 2802, "symbol": "NZD", "name": "New Zealand Dollar"}, + {"id": 3560, "symbol": "NIO", "name": "Nicaraguan Córdoba"}, + {"id": 2819, "symbol": "NGN", "name": "Nigerian Naira"}, + {"id": 2801, "symbol": "NOK", "name": "Norwegian Krone"}, + {"id": 3562, "symbol": "OMR", "name": "Omani Rial"}, + {"id": 2804, "symbol": "PKR", "name": "Pakistani Rupee"}, + {"id": 3563, "symbol": "PAB", "name": "Panamanian Balboa"}, + {"id": 2822, "symbol": "PEN", "name": "Peruvian Sol"}, + {"id": 2803, "symbol": "PHP", "name": "Philippine Peso"}, + {"id": 2805, "symbol": "PLN", "name": "Polish Złoty"}, + {"id": 2791, "symbol": "GBP", "name": "Pound Sterling"}, + {"id": 3564, "symbol": "QAR", "name": "Qatari Rial"}, + {"id": 2817, "symbol": "RON", "name": "Romanian Leu"}, + {"id": 2806, "symbol": "RUB", "name": "Russian Ruble"}, + {"id": 3566, "symbol": "SAR", "name": "Saudi Riyal"}, + {"id": 3565, "symbol": "RSD", "name": "Serbian Dinar"}, + {"id": 2808, "symbol": "SGD", "name": "Singapore Dollar"}, + {"id": 2812, "symbol": "ZAR", "name": "South African Rand"}, + {"id": 2798, "symbol": "KRW", "name": "South Korean Won"}, + {"id": 3567, "symbol": "SSP", "name": "South Sudanese Pound"}, + {"id": 3573, "symbol": "VES", "name": "Sovereign Bolivar"}, + {"id": 3553, "symbol": "LKR", "name": "Sri Lankan Rupee"}, + {"id": 2807, "symbol": "SEK", "name": "Swedish Krona"}, + {"id": 2785, "symbol": "CHF", "name": "Swiss Franc"}, + {"id": 2809, "symbol": "THB", "name": "Thai Baht"}, + {"id": 3569, "symbol": "TTD", "name": "Trinidad and Tobago Dollar"}, + {"id": 3568, "symbol": "TND", "name": "Tunisian Dinar"}, + {"id": 2810, "symbol": "TRY", "name": "Turkish Lira"}, + {"id": 3570, "symbol": "UGX", "name": "Ugandan Shilling"}, + {"id": 2824, "symbol": "UAH", "name": "Ukrainian Hryvnia"}, + {"id": 2813, "symbol": "AED", "name": "United Arab Emirates Dirham"}, + {"id": 3571, "symbol": "UYU", "name": "Uruguayan Peso"}, + {"id": 3572, "symbol": "UZS", "name": "Uzbekistan Som"}, + {"id": 2823, "symbol": "VND", "name": "Vietnamese Dong"}, + ] + metals = [ + {"id": 3575, "symbol": "XAU", "name": "Gold Troy Ounce"}, + {"id": 3574, "symbol": "XAG", "name": "Silver Troy Ounce"}, + {"id": 3577, "symbol": "XPT", "name": "Platinum Ounce"}, + {"id": 3576, "symbol": "XPD", "name": "Palladium Ounce"}, + ] + # fmt: on + + return fiat + metals + crypto def _get_json_data(self, url, params={}): try: From 5a0de59aba80d4858b06b776bdb4ac214e4d0868 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 2 Aug 2024 09:47:07 +0200 Subject: [PATCH 130/149] coinmarketcap: fix quote output. --- src/pricehist/beanprice/exchangeratehost.py | 4 + src/pricehist/sources/coinmarketcap.py | 7 +- src/pricehist/sources/exchangeratehost.py | 122 ++++++++++++++++++++ 3 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 src/pricehist/beanprice/exchangeratehost.py create mode 100644 src/pricehist/sources/exchangeratehost.py diff --git a/src/pricehist/beanprice/exchangeratehost.py b/src/pricehist/beanprice/exchangeratehost.py new file mode 100644 index 0000000..ad6525a --- /dev/null +++ b/src/pricehist/beanprice/exchangeratehost.py @@ -0,0 +1,4 @@ +from pricehist import beanprice +from pricehist.sources.exchangeratehost import ExchangeRateHost + +Source = beanprice.source(ExchangeRateHost()) diff --git a/src/pricehist/sources/coinmarketcap.py b/src/pricehist/sources/coinmarketcap.py index f7e89ab..275df76 100644 --- a/src/pricehist/sources/coinmarketcap.py +++ b/src/pricehist/sources/coinmarketcap.py @@ -166,17 +166,18 @@ class CoinMarketCap(BaseSource): def _output_pair(self, base, quote, data): data_base = data["symbol"] + symbols = {i["id"]: (i["symbol"] or i["code"]) for i in self._symbol_data()} + data_quote = None if len(data["quotes"]) > 0: - data_quote = next(iter(data["quotes"][0]["quote"].keys())) + data_quote = symbols[int(data["quotes"][0]["quote"]["name"])] lookup_quote = None if quote.startswith("ID="): - symbols = {i["id"]: (i["symbol"] or i["code"]) for i in self._symbol_data()} lookup_quote = symbols[int(quote[3:])] output_base = data_base - output_quote = lookup_quote or data_quote or quote + output_quote = data_quote or lookup_quote or quote return (output_base, output_quote) diff --git a/src/pricehist/sources/exchangeratehost.py b/src/pricehist/sources/exchangeratehost.py new file mode 100644 index 0000000..76c412a --- /dev/null +++ b/src/pricehist/sources/exchangeratehost.py @@ -0,0 +1,122 @@ +import dataclasses +import json +from decimal import Decimal + +import requests + +from pricehist import exceptions +from pricehist.price import Price + +from .basesource import BaseSource + + +class ExchangeRateHost(BaseSource): + def id(self): + return "exchangeratehost" + + def name(self): + return "exchangerate.host Exchange rates API" + + def description(self): + return ( + "Exchange rates API is a simple and lightweight free service for " + "current and historical foreign exchange rates & crypto exchange " + "rates." + ) + + def source_url(self): + return "https://exchangerate.host/" + + def start(self): + return "1999-01-01" + + def types(self): + return ["close"] + + def notes(self): + return "" + + def symbols(self): + url = "https://api.coindesk.com/v1/bpi/supported-currencies.json" + + try: + response = self.log_curl(requests.get(url)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + data = json.loads(response.content) + relevant = [i for i in data if i["currency"] not in ["BTC", "XBT"]] + results = [ + (f"BTC/{i['currency']}", f"Bitcoin against {i['country']}") + for i in sorted(relevant, key=lambda i: i["currency"]) + ] + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + if not results: + raise exceptions.ResponseParsingError("Expected data not found") + else: + return results + + def fetch(self, series): + if series.base != "BTC" or series.quote in ["BTC", "XBT"]: + # BTC is the only valid base. + # BTC as the quote will return BTC/USD, which we don't want. + # XBT as the quote will fail with HTTP status 500. + raise exceptions.InvalidPair(series.base, series.quote, self) + + data = self._data(series) + + prices = [] + for (d, v) in data.get("bpi", {}).items(): + prices.append(Price(d, Decimal(str(v)))) + + return dataclasses.replace(series, prices=prices) + + def _data(self, series): + url = "https://api.coindesk.com/v1/bpi/historical/close.json" + params = { + "currency": series.quote, + "start": series.start, + "end": series.end, + } + + try: + response = self.log_curl(requests.get(url, params=params)) + except Exception as e: + raise exceptions.RequestError(str(e)) from e + + code = response.status_code + text = response.text + if code == 404 and "currency was not found" in text: + raise exceptions.InvalidPair(series.base, series.quote, self) + elif code == 404 and "only covers data from" in text: + raise exceptions.BadResponse(text) + elif code == 404 and "end date is before" in text and series.end < series.start: + raise exceptions.BadResponse("End date is before start date.") + elif code == 404 and "end date is before" in text: + raise exceptions.BadResponse("The start date must be in the past.") + elif code == 500 and "No results returned from database" in text: + raise exceptions.BadResponse( + "No results returned from database. This can happen when data " + "for a valid quote currency (e.g. CUP) doesn't go all the way " + "back to the start date, and potentially for other reasons." + ) + else: + try: + response.raise_for_status() + except Exception as e: + raise exceptions.BadResponse(str(e)) from e + + try: + result = json.loads(response.content) + except Exception as e: + raise exceptions.ResponseParsingError(str(e)) from e + + return result From 5fdf16edb7dcfb315bc2f68b4a8d51564f80f816 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 3 Aug 2024 12:33:29 +0200 Subject: [PATCH 131/149] Update pytest. --- poetry.lock | 74 ++++++++++++++++---------------------------------- pyproject.toml | 2 +- 2 files changed, 24 insertions(+), 52 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9d3681b..4016d9c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,34 +1,5 @@ # This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. -[[package]] -name = "atomicwrites" -version = "1.4.1" -description = "Atomic file writes." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] - -[[package]] -name = "attrs" -version = "23.2.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.7" -files = [ - {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, - {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, -] - -[package.extras] -cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[tests]", "pre-commit"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] -tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] -tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] - [[package]] name = "black" version = "22.12.0" @@ -298,6 +269,20 @@ files = [ {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, ] +[[package]] +name = "exceptiongroup" +version = "1.2.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, +] + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "filelock" version = "3.15.4" @@ -634,27 +619,25 @@ files = [ [[package]] name = "pytest" -version = "6.2.5" +version = "8.3.2" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, - {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, + {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, + {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, ] [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -toml = "*" +pluggy = ">=1.5,<2" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-mock" @@ -724,17 +707,6 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] - [[package]] name = "tomli" version = "2.0.1" @@ -822,4 +794,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "de3fe2ed9cb9ec204b3e1f94150f17b0a27b3c13a722164e5118512691cf248d" +content-hash = "0d56bfdf88b0280475309ade51b6bb230ab96cc6111a7dbe8291c7aba12b5c20" diff --git a/pyproject.toml b/pyproject.toml index b6b53d6..116373b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ cssselect = "^1.1.0" curlify = "^2.2.1" [tool.poetry.dev-dependencies] -pytest = "^6.2.2" +pytest = "^8.3.2" black = "^22.10.0" flake8 = "^7.1.0" isort = "^5.8.0" From 9dd6121d4da71c6bbb2785e25118b5c635e07b5f Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 3 Aug 2024 12:33:48 +0200 Subject: [PATCH 132/149] Update coinmarketcap error handling and tests. --- src/pricehist/sources/coinmarketcap.py | 26 +- tests/pricehist/sources/test_coinmarketcap.py | 285 ++++-------------- .../test_coinmarketcap/fiat-partial.json | 30 -- .../long-btc-aud-partial.json | 255 ---------------- .../test_coinmarketcap/recent-btc-aud.json | 136 --------- .../test_coinmarketcap/recent-btc-id2782.json | 136 --------- .../test_coinmarketcap/recent-id1-aud.json | 136 --------- .../test_coinmarketcap/recent-id1-id2782.json | 191 ++++++------ 8 files changed, 170 insertions(+), 1025 deletions(-) delete mode 100644 tests/pricehist/sources/test_coinmarketcap/fiat-partial.json delete mode 100644 tests/pricehist/sources/test_coinmarketcap/long-btc-aud-partial.json delete mode 100644 tests/pricehist/sources/test_coinmarketcap/recent-btc-aud.json delete mode 100644 tests/pricehist/sources/test_coinmarketcap/recent-btc-id2782.json delete mode 100644 tests/pricehist/sources/test_coinmarketcap/recent-id1-aud.json diff --git a/src/pricehist/sources/coinmarketcap.py b/src/pricehist/sources/coinmarketcap.py index 275df76..f7e0fb8 100644 --- a/src/pricehist/sources/coinmarketcap.py +++ b/src/pricehist/sources/coinmarketcap.py @@ -117,21 +117,6 @@ class CoinMarketCap(BaseSource): series.base, series.quote, self, "Bad quote ID." ) - elif code == 400 and "must be older than" in text: - if series.start <= series.end: - raise exceptions.BadResponse("The start date must be in the past.") - else: - raise exceptions.BadResponse( - "The start date must preceed or match the end date." - ) - - elif ( - code == 400 - and "must be a valid ISO 8601 timestamp or unix time" in text - and series.start < "2001-09-11" - ): - raise exceptions.BadResponse("The start date can't preceed 2001-09-11.") - try: response.raise_for_status() except Exception as e: @@ -142,6 +127,17 @@ class CoinMarketCap(BaseSource): except Exception as e: raise exceptions.ResponseParsingError(str(e)) from e + if ( + "status" in parsed + and "error_code" in parsed["status"] + and parsed["status"]["error_code"] == "500" + and "The system is busy" in parsed["status"]["error_message"] + ): + raise exceptions.BadResponse( + "The server indicated a general error. " + "There may be problem with your request." + ) + if type(parsed) is not dict or "data" not in parsed: raise exceptions.ResponseParsingError("Unexpected content.") diff --git a/tests/pricehist/sources/test_coinmarketcap.py b/tests/pricehist/sources/test_coinmarketcap.py index a7fec0c..b7bd721 100644 --- a/tests/pricehist/sources/test_coinmarketcap.py +++ b/tests/pricehist/sources/test_coinmarketcap.py @@ -36,9 +36,10 @@ def requests_mock(): yield mock -crypto_url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/map?sort=cmc_rank" -fiat_url = "https://web-api.coinmarketcap.com/v1/fiat/map?include_metals=true" -fetch_url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical" +crypto_url = ( + "https://api.coinmarketcap.com/data-api/v1/cryptocurrency/map?sort=cmc_rank" +) +fetch_url = "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical" @pytest.fixture @@ -48,13 +49,6 @@ def crypto_ok(requests_mock): yield requests_mock -@pytest.fixture -def fiat_ok(requests_mock): - json = (Path(os.path.splitext(__file__)[0]) / "fiat-partial.json").read_text() - requests_mock.add(responses.GET, fiat_url, body=json, status=200) - yield requests_mock - - @pytest.fixture def recent_id_id_ok(requests_mock): json = (Path(os.path.splitext(__file__)[0]) / "recent-id1-id2782.json").read_text() @@ -62,36 +56,6 @@ def recent_id_id_ok(requests_mock): yield requests_mock -@pytest.fixture -def recent_id_sym_ok(requests_mock): - json = (Path(os.path.splitext(__file__)[0]) / "recent-id1-aud.json").read_text() - requests_mock.add(responses.GET, fetch_url, body=json, status=200) - yield requests_mock - - -@pytest.fixture -def recent_sym_id_ok(requests_mock): - json = (Path(os.path.splitext(__file__)[0]) / "recent-btc-id2782.json").read_text() - requests_mock.add(responses.GET, fetch_url, body=json, status=200) - yield requests_mock - - -@pytest.fixture -def recent_sym_sym_ok(requests_mock): - json = (Path(os.path.splitext(__file__)[0]) / "recent-btc-aud.json").read_text() - requests_mock.add(responses.GET, fetch_url, body=json, status=200) - yield requests_mock - - -@pytest.fixture -def long_sym_sym_ok(requests_mock): - json = ( - Path(os.path.splitext(__file__)[0]) / "long-btc-aud-partial.json" - ).read_text() - requests_mock.add(responses.GET, fetch_url, body=json, status=200) - yield requests_mock - - def test_normalizesymbol(src): assert src.normalizesymbol("btc") == "BTC" assert src.normalizesymbol("id=1") == "ID=1" @@ -120,63 +84,31 @@ def test_metadata(src): assert isinstance(src.notes(), str) -def test_symbols(src, crypto_ok, fiat_ok): +def test_symbols(src, crypto_ok): syms = src.symbols() assert ("id=1", "BTC Bitcoin") in syms assert ("id=2782", "AUD Australian Dollar") in syms assert len(syms) > 2 -def test_symbols_requests_logged(src, crypto_ok, fiat_ok, caplog): +def test_symbols_request_logged(src, crypto_ok, caplog): with caplog.at_level(logging.DEBUG): src.symbols() logged_requests = 0 for r in caplog.records: if r.levelname == "DEBUG" and "curl " in r.message: logged_requests += 1 - assert logged_requests == 2 + assert logged_requests == 1 -def test_symbols_fiat_not_found(src, requests_mock): - requests_mock.add(responses.GET, fiat_url, body="{}", status=200) - with pytest.raises(exceptions.ResponseParsingError) as e: - src.symbols() - assert "Unexpected content" in str(e.value) - - -def test_symbols_fiat_network_issue(src, requests_mock): - requests_mock.add( - responses.GET, - fiat_url, - body=requests.exceptions.ConnectionError("Network issue"), - ) - with pytest.raises(exceptions.RequestError) as e: - src.symbols() - assert "Network issue" in str(e.value) - - -def test_symbols_fiat_bad_status(src, requests_mock): - requests_mock.add(responses.GET, fiat_url, status=500) - with pytest.raises(exceptions.BadResponse) as e: - src.symbols() - assert "Server Error" in str(e.value) - - -def test_symbols_fiat_parsing_error(src, requests_mock): - requests_mock.add(responses.GET, fiat_url, body="NOT JSON") - with pytest.raises(exceptions.ResponseParsingError) as e: - src.symbols() - assert "while parsing data" in str(e.value) - - -def test_symbols_crypto_not_found(src, requests_mock, fiat_ok): +def test_symbols_crypto_not_found(src, requests_mock): requests_mock.add(responses.GET, crypto_url, body="{}", status=200) with pytest.raises(exceptions.ResponseParsingError) as e: src.symbols() assert "Unexpected content" in str(e.value) -def test_symbols_crypto_network_issue(src, requests_mock, fiat_ok): +def test_symbols_crypto_network_issue(src, requests_mock): requests_mock.add( responses.GET, crypto_url, @@ -187,14 +119,14 @@ def test_symbols_crypto_network_issue(src, requests_mock, fiat_ok): assert "Network issue" in str(e.value) -def test_symbols_crypto_bad_status(src, requests_mock, fiat_ok): +def test_symbols_crypto_bad_status(src, requests_mock): requests_mock.add(responses.GET, crypto_url, status=500) with pytest.raises(exceptions.BadResponse) as e: src.symbols() assert "Server Error" in str(e.value) -def test_symbols_crypto_parsing_error(src, requests_mock, fiat_ok): +def test_symbols_crypto_parsing_error(src, requests_mock): requests_mock.add(responses.GET, crypto_url, body="NOT JSON") with pytest.raises(exceptions.ResponseParsingError) as e: src.symbols() @@ -202,59 +134,59 @@ def test_symbols_crypto_parsing_error(src, requests_mock, fiat_ok): def test_symbols_no_data(src, type, requests_mock): - requests_mock.add(responses.GET, fiat_url, body='{"data": []}') + requests_mock.add(responses.GET, crypto_url, body='{"data": []}') with pytest.raises(exceptions.ResponseParsingError) as e: src.symbols() assert "Empty data section" in str(e.value) -def test_fetch_known_pair_id_id(src, type, recent_id_id_ok, crypto_ok, fiat_ok): +def test_fetch_known_pair_id_id(src, type, recent_id_id_ok, crypto_ok): series = src.fetch(Series("ID=1", "ID=2782", type, "2021-01-01", "2021-01-07")) req = recent_id_id_ok.calls[0].request assert req.params["id"] == "1" - assert req.params["convert_id"] == "2782" + assert req.params["convertId"] == "2782" assert (series.base, series.quote) == ("BTC", "AUD") assert len(series.prices) == 7 -def test_fetch_known_pair_id_sym(src, type, recent_id_sym_ok): +def test_fetch_known_pair_id_sym(src, type, recent_id_id_ok, crypto_ok): series = src.fetch(Series("ID=1", "AUD", type, "2021-01-01", "2021-01-07")) - req = recent_id_sym_ok.calls[0].request + req = recent_id_id_ok.calls[1].request assert req.params["id"] == "1" - assert req.params["convert"] == "AUD" + assert req.params["convertId"] == "2782" assert (series.base, series.quote) == ("BTC", "AUD") assert len(series.prices) == 7 -def test_fetch_known_pair_sym_id(src, type, recent_sym_id_ok, crypto_ok, fiat_ok): +def test_fetch_known_pair_sym_id(src, type, recent_id_id_ok, crypto_ok): series = src.fetch(Series("BTC", "ID=2782", type, "2021-01-01", "2021-01-07")) - req = recent_sym_id_ok.calls[0].request - assert req.params["symbol"] == "BTC" - assert req.params["convert_id"] == "2782" + req = recent_id_id_ok.calls[1].request + assert req.params["id"] == "1" + assert req.params["convertId"] == "2782" assert (series.base, series.quote) == ("BTC", "AUD") assert len(series.prices) == 7 -def test_fetch_known_pair_sym_sym(src, type, recent_sym_sym_ok): +def test_fetch_known_pair_sym_sym(src, type, recent_id_id_ok, crypto_ok): series = src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) - req = recent_sym_sym_ok.calls[0].request - assert req.params["symbol"] == "BTC" - assert req.params["convert"] == "AUD" + req = recent_id_id_ok.calls[1].request + assert req.params["id"] == "1" + assert req.params["convertId"] == "2782" assert len(series.prices) == 7 def test_fetch_requests_and_receives_correct_times( - src, type, recent_id_id_ok, crypto_ok, fiat_ok + src, type, recent_id_id_ok, crypto_ok ): series = src.fetch(Series("ID=1", "ID=2782", type, "2021-01-01", "2021-01-07")) req = recent_id_id_ok.calls[0].request - assert req.params["time_start"] == str(timestamp("2020-12-31")) # back one period - assert req.params["time_end"] == str(timestamp("2021-01-07")) - assert series.prices[0] == Price("2021-01-01", Decimal("37914.350602379853")) - assert series.prices[-1] == Price("2021-01-07", Decimal("49370.064689585612")) + assert req.params["timeStart"] == str(timestamp("2020-12-31")) # back one period + assert req.params["timeEnd"] == str(timestamp("2021-01-07")) + assert series.prices[0] == Price("2021-01-01", Decimal("37914.35060237985")) + assert series.prices[-1] == Price("2021-01-07", Decimal("49369.66288590665")) -def test_fetch_requests_logged(src, type, recent_sym_sym_ok, caplog): +def test_fetch_requests_logged(src, type, crypto_ok, recent_id_id_ok, caplog): with caplog.at_level(logging.DEBUG): src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) assert any( @@ -262,20 +194,20 @@ def test_fetch_requests_logged(src, type, recent_sym_sym_ok, caplog): ) -def test_fetch_types_all_available(src, recent_sym_sym_ok): +def test_fetch_types_all_available(src, crypto_ok, recent_id_id_ok): mid = src.fetch(Series("BTC", "AUD", "mid", "2021-01-01", "2021-01-07")) opn = src.fetch(Series("BTC", "AUD", "open", "2021-01-01", "2021-01-07")) hgh = src.fetch(Series("BTC", "AUD", "high", "2021-01-01", "2021-01-07")) low = src.fetch(Series("BTC", "AUD", "low", "2021-01-01", "2021-01-07")) cls = src.fetch(Series("BTC", "AUD", "close", "2021-01-01", "2021-01-07")) - assert mid.prices[0].amount == Decimal("37914.350602379853") - assert opn.prices[0].amount == Decimal("37658.83948707033") + assert mid.prices[0].amount == Decimal("37914.35060237985") + assert opn.prices[0].amount == Decimal("37658.1146368474") assert hgh.prices[0].amount == Decimal("38417.9137031205") - assert low.prices[0].amount == Decimal("37410.787501639206") - assert cls.prices[0].amount == Decimal("38181.99133300758") + assert low.prices[0].amount == Decimal("37410.7875016392") + assert cls.prices[0].amount == Decimal("38181.9913330076") -def test_fetch_type_mid_is_mean_of_low_and_high(src, recent_sym_sym_ok): +def test_fetch_type_mid_is_mean_of_low_and_high(src, crypto_ok, recent_id_id_ok): mid = src.fetch(Series("BTC", "AUD", "mid", "2021-01-01", "2021-01-07")).prices low = src.fetch(Series("BTC", "AUD", "low", "2021-01-01", "2021-01-07")).prices hgh = src.fetch(Series("BTC", "AUD", "high", "2021-01-01", "2021-01-07")).prices @@ -287,80 +219,24 @@ def test_fetch_type_mid_is_mean_of_low_and_high(src, recent_sym_sym_ok): ) -def test_fetch_long_hist_from_start(src, type, long_sym_sym_ok): - series = src.fetch(Series("BTC", "AUD", type, src.start(), "2021-01-07")) - assert series.prices[0] == Price("2013-04-28", Decimal("130.45956234123247")) - assert series.prices[-1] == Price("2021-01-07", Decimal("49370.064689585612")) - assert len(series.prices) > 13 - - -def test_fetch_from_before_start(src, type, requests_mock): - requests_mock.add( - responses.GET, - fetch_url, - status=400, - body="""{ "status": { "error_code": 400, "error_message": - "\\"time_start\\" must be a valid ISO 8601 timestamp or unix time value", - } }""", - ) - with pytest.raises(exceptions.BadResponse) as e: - src.fetch(Series("BTC", "AUD", type, "2001-09-10", "2001-10-01")) - assert "start date can't preceed" in str(e.value) - - -def test_fetch_to_future(src, type, recent_sym_sym_ok): - series = src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2100-01-01")) - assert len(series.prices) > 0 - - -def test_fetch_in_future(src, type, requests_mock): - requests_mock.add( - responses.GET, - fetch_url, - status=400, - body="""{ - "status": { - "error_code": 400, - "error_message": "\\"time_start\\" must be older than \\"time_end\\"." - } - }""", - ) - with pytest.raises(exceptions.BadResponse) as e: - src.fetch(Series("BTC", "AUD", type, "2030-01-01", "2030-01-07")) - assert "start date must be in the past" in str(e.value) - - -def test_fetch_reversed_dates(src, type, requests_mock): - requests_mock.add( - responses.GET, - fetch_url, - status=400, - body="""{ - "status": { - "error_code": 400, - "error_message": "\\"time_start\\" must be older than \\"time_end\\"." - } - }""", - ) - with pytest.raises(exceptions.BadResponse) as e: - src.fetch(Series("BTC", "AUD", type, "2021-01-07", "2021-01-01")) - assert "start date must preceed or match the end" in str(e.value) - - -def test_fetch_empty(src, type, requests_mock): +def test_fetch_empty(src, type, crypto_ok, requests_mock): requests_mock.add( responses.GET, fetch_url, body="""{ - "status": { - "error_code": 0, - "error_message": null - }, "data": { "id": 1, "name": "Bitcoin", "symbol": "BTC", + "timeEnd": "1228348799", "quotes": [] + }, + "status": { + "timestamp": "2024-08-03T09:31:52.719Z", + "error_code": "0", + "error_message": "SUCCESS", + "elapsed": "14", + "credit_count": 0 } }""", ) @@ -368,63 +244,36 @@ def test_fetch_empty(src, type, requests_mock): assert len(series.prices) == 0 -def test_fetch_bad_base_sym(src, type, requests_mock): - requests_mock.add(responses.GET, fetch_url, body='{"data":{}}') - with pytest.raises(exceptions.ResponseParsingError) as e: +def test_fetch_bad_base_sym(src, type, crypto_ok): + with pytest.raises(exceptions.InvalidPair) as e: src.fetch(Series("NOTABASE", "USD", type, "2021-01-01", "2021-01-07")) - assert "quote currency symbol can't be found" in str(e.value) - assert "other reasons" in str(e.value) + assert "Invalid symbol 'NOTABASE'" in str(e.value) -def test_fetch_bad_quote_sym(src, type, requests_mock): - requests_mock.add( - responses.GET, - fetch_url, - status=400, - body="""{ - "status": { - "error_code": 400, - "error_message": "Invalid value for \\"convert\\": \\"NOTAQUOTE\\"" - } - }""", - ) +def test_fetch_bad_quote_sym(src, type, crypto_ok): with pytest.raises(exceptions.InvalidPair) as e: src.fetch(Series("BTC", "NOTAQUOTE", type, "2021-01-01", "2021-01-07")) - assert "Bad quote symbol" in str(e.value) + assert "Invalid symbol 'NOTAQUOTE'" in str(e.value) -def test_fetch_bad_base_id(src, type, requests_mock): +def test_fetch_bad_response(src, type, crypto_ok, requests_mock): requests_mock.add( responses.GET, fetch_url, - status=400, + status=200, body="""{ - "status": { - "error_code": 400, - "error_message": "No items found." - } + "status": { + "timestamp": "2024-08-03T09:42:43.699Z", + "error_code": "500", + "error_message": "The system is busy, please try again later!", + "elapsed": "0", + "credit_count": 0 + } }""", ) - with pytest.raises(exceptions.InvalidPair) as e: - src.fetch(Series("ID=20000", "USD", type, "2021-01-01", "2021-01-07")) - assert "Bad base ID" in str(e.value) - - -def test_fetch_bad_quote_id(src, type, requests_mock): - requests_mock.add( - responses.GET, - fetch_url, - status=400, - body="""{ - "status": { - "error_code": 400, - "error_message": "Invalid value for \\"convert_id\\": \\"20000\\"" - } - }""", - ) - with pytest.raises(exceptions.InvalidPair) as e: - src.fetch(Series("BTC", "ID=20000", type, "2021-01-01", "2021-01-07")) - assert "Bad quote ID" in str(e.value) + with pytest.raises(exceptions.BadResponse) as e: + src.fetch(Series("ID=987654321", "USD", type, "2021-01-01", "2021-01-07")) + assert "general error" in str(e.value) def test_fetch_no_quote(src, type): @@ -432,7 +281,7 @@ def test_fetch_no_quote(src, type): src.fetch(Series("BTC", "", type, "2021-01-01", "2021-01-07")) -def test_fetch_network_issue(src, type, requests_mock): +def test_fetch_network_issue(src, type, crypto_ok, requests_mock): body = requests.exceptions.ConnectionError("Network issue") requests_mock.add(responses.GET, fetch_url, body=body) with pytest.raises(exceptions.RequestError) as e: @@ -440,21 +289,21 @@ def test_fetch_network_issue(src, type, requests_mock): assert "Network issue" in str(e.value) -def test_fetch_bad_status(src, type, requests_mock): +def test_fetch_bad_status(src, type, crypto_ok, requests_mock): requests_mock.add(responses.GET, fetch_url, status=500, body="Some other reason") with pytest.raises(exceptions.BadResponse) as e: src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) assert "Internal Server Error" in str(e.value) -def test_fetch_parsing_error(src, type, requests_mock): +def test_fetch_parsing_error(src, type, crypto_ok, requests_mock): requests_mock.add(responses.GET, fetch_url, body="NOT JSON") with pytest.raises(exceptions.ResponseParsingError) as e: src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) assert "while parsing data" in str(e.value) -def test_fetch_unexpected_json(src, type, requests_mock): +def test_fetch_unexpected_json(src, type, crypto_ok, requests_mock): requests_mock.add(responses.GET, fetch_url, body='{"notdata": []}') with pytest.raises(exceptions.ResponseParsingError) as e: src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) diff --git a/tests/pricehist/sources/test_coinmarketcap/fiat-partial.json b/tests/pricehist/sources/test_coinmarketcap/fiat-partial.json deleted file mode 100644 index 781824b..0000000 --- a/tests/pricehist/sources/test_coinmarketcap/fiat-partial.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "status": { - "timestamp": "2021-07-16T10:08:13.272Z", - "error_code": 0, - "error_message": null, - "elapsed": 1, - "credit_count": 0, - "notice": null - }, - "data": [ - { - "id": 2781, - "name": "United States Dollar", - "sign": "$", - "symbol": "USD" - }, - { - "id": 2782, - "name": "Australian Dollar", - "sign": "$", - "symbol": "AUD" - }, - { - "id": 3575, - "name": "Gold Troy Ounce", - "symbol": "", - "code": "XAU" - } - ] -} diff --git a/tests/pricehist/sources/test_coinmarketcap/long-btc-aud-partial.json b/tests/pricehist/sources/test_coinmarketcap/long-btc-aud-partial.json deleted file mode 100644 index 0b11696..0000000 --- a/tests/pricehist/sources/test_coinmarketcap/long-btc-aud-partial.json +++ /dev/null @@ -1,255 +0,0 @@ -{ - "status": { - "timestamp": "2021-07-17T16:16:11.926Z", - "error_code": 0, - "error_message": null, - "elapsed": 2262, - "credit_count": 0, - "notice": null - }, - "data": { - "id": 1, - "name": "Bitcoin", - "symbol": "BTC", - "quotes": [ - { - "time_open": "2013-04-28T00:00:00.000Z", - "time_close": "2013-04-28T23:59:59.999Z", - "time_high": "2013-04-28T18:50:02.000Z", - "time_low": "2013-04-28T20:15:02.000Z", - "quote": { - "AUD": { - "open": null, - "high": 132.39216797540558, - "low": 128.52695670705936, - "close": 130.52908647526473, - "volume": 0, - "market_cap": 1447740447.626921, - "timestamp": "2013-04-28T23:59:00.000Z" - } - } - }, - { - "time_open": "2013-04-29T00:00:00.000Z", - "time_close": "2013-04-29T23:59:59.999Z", - "time_high": "2013-04-29T13:15:01.000Z", - "time_low": "2013-04-29T05:20:01.000Z", - "quote": { - "AUD": { - "open": 130.75666236543535, - "high": 142.67970067891736, - "low": 129.9456943366951, - "close": 139.77370978254794, - "volume": 0, - "market_cap": 1550883729.329852, - "timestamp": "2013-04-29T23:59:00.000Z" - } - } - }, - { - "time_open": "2013-04-30T00:00:00.000Z", - "time_close": "2013-04-30T23:59:59.999Z", - "time_high": "2013-04-30T08:25:02.000Z", - "time_low": "2013-04-30T18:55:01.000Z", - "quote": { - "AUD": { - "open": 139.2515230635335, - "high": 141.93391873626476, - "low": 129.37940647790543, - "close": 134.06635802469137, - "volume": 0, - "market_cap": 1488052782.6003087, - "timestamp": "2013-04-30T23:59:00.000Z" - } - } - }, - { - "time_open": "2013-05-01T00:00:00.000Z", - "time_close": "2013-05-01T23:59:59.999Z", - "time_high": "2013-05-01T00:15:01.000Z", - "time_low": "2013-05-01T19:55:01.000Z", - "quote": { - "AUD": { - "open": 134.06635802469137, - "high": 134.88573849160971, - "low": 104.93911468163968, - "close": 113.79243056489595, - "volume": 0, - "market_cap": 1263451603.6864119, - "timestamp": "2013-05-01T23:59:00.000Z" - } - } - }, - { - "time_open": "2013-05-02T00:00:00.000Z", - "time_close": "2013-05-02T23:59:59.999Z", - "time_high": "2013-05-02T14:25:01.000Z", - "time_low": "2013-05-02T14:30:02.000Z", - "quote": { - "AUD": { - "open": 113.19910247390133, - "high": 122.60835462135991, - "low": 90.08385249759387, - "close": 102.63388848353591, - "volume": 0, - "market_cap": 1139905858.2089553, - "timestamp": "2013-05-02T23:59:00.000Z" - } - } - }, - { - "time_open": "2013-05-03T00:00:00.000Z", - "time_close": "2013-05-03T23:59:59.999Z", - "time_high": "2013-05-03T05:30:02.000Z", - "time_low": "2013-05-03T03:05:01.000Z", - "quote": { - "AUD": { - "open": 103.64842454394694, - "high": 105.43929629649027, - "low": 77.03544845551335, - "close": 94.77409346519293, - "volume": 0, - "market_cap": 1052933070.3412836, - "timestamp": "2013-05-03T23:59:00.000Z" - } - } - }, - { - "time_open": "2013-05-04T00:00:00.000Z", - "time_close": "2013-05-04T23:59:59.999Z", - "time_high": "2013-05-04T07:15:01.000Z", - "time_low": "2013-05-04T06:50:01.000Z", - "quote": { - "AUD": { - "open": 95.11343656595025, - "high": 111.49893348846227, - "low": 89.68392476245879, - "close": 109.07504363001745, - "volume": 0, - "market_cap": 1212251854.2757416, - "timestamp": "2013-05-04T23:59:00.000Z" - } - } - }, - { - "time_open": "2021-01-01T00:00:00.000Z", - "time_close": "2021-01-01T23:59:59.999Z", - "time_high": "2021-01-01T12:38:43.000Z", - "time_low": "2021-01-01T00:16:43.000Z", - "quote": { - "AUD": { - "open": 37658.83948707033, - "high": 38417.9137031205, - "low": 37410.787501639206, - "close": 38181.99133300758, - "volume": 52943282221.028366, - "market_cap": 709720173049.5383, - "timestamp": "2021-01-01T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-02T00:00:00.000Z", - "time_close": "2021-01-02T23:59:59.999Z", - "time_high": "2021-01-02T19:49:42.000Z", - "time_low": "2021-01-02T00:31:44.000Z", - "quote": { - "AUD": { - "open": 38184.98611600682, - "high": 43096.681197423015, - "low": 37814.17187096531, - "close": 41760.62923079505, - "volume": 88214867181.97835, - "market_cap": 776278147177.8037, - "timestamp": "2021-01-02T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-03T00:00:00.000Z", - "time_close": "2021-01-03T23:59:59.999Z", - "time_high": "2021-01-03T07:47:38.000Z", - "time_low": "2021-01-03T00:20:45.000Z", - "quote": { - "AUD": { - "open": 41763.41015117659, - "high": 44985.93247585023, - "low": 41663.204350601605, - "close": 42511.10646879765, - "volume": 102011582370.28117, - "market_cap": 790270288834.0249, - "timestamp": "2021-01-03T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-04T00:00:00.000Z", - "time_close": "2021-01-04T23:59:59.999Z", - "time_high": "2021-01-04T04:07:42.000Z", - "time_low": "2021-01-04T10:19:42.000Z", - "quote": { - "AUD": { - "open": 42548.61349648768, - "high": 43360.96165147421, - "low": 37133.98436952697, - "close": 41686.38761359174, - "volume": 105824510346.65779, - "market_cap": 774984045201.7122, - "timestamp": "2021-01-04T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-05T00:00:00.000Z", - "time_close": "2021-01-05T23:59:59.999Z", - "time_high": "2021-01-05T22:44:35.000Z", - "time_low": "2021-01-05T06:16:41.000Z", - "quote": { - "AUD": { - "open": 41693.07321807638, - "high": 44403.79487147647, - "low": 39221.81167941294, - "close": 43790.067253370056, - "volume": 87016490203.50436, - "market_cap": 814135603090.2502, - "timestamp": "2021-01-05T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-06T00:00:00.000Z", - "time_close": "2021-01-06T23:59:59.999Z", - "time_high": "2021-01-06T23:57:36.000Z", - "time_low": "2021-01-06T00:25:38.000Z", - "quote": { - "AUD": { - "open": 43817.35864984641, - "high": 47186.65232598287, - "low": 43152.60281764236, - "close": 47115.85365360005, - "volume": 96330948324.8061, - "market_cap": 876019742889.9551, - "timestamp": "2021-01-06T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-07T00:00:00.000Z", - "time_close": "2021-01-07T23:59:59.999Z", - "time_high": "2021-01-07T18:17:42.000Z", - "time_low": "2021-01-07T08:25:51.000Z", - "quote": { - "AUD": { - "open": 47128.02139328098, - "high": 51833.478207775144, - "low": 46906.65117139608, - "close": 50686.90986207153, - "volume": 109124136558.20264, - "market_cap": 942469208700.134, - "timestamp": "2021-01-07T23:59:06.000Z" - } - } - } - ] - } -} diff --git a/tests/pricehist/sources/test_coinmarketcap/recent-btc-aud.json b/tests/pricehist/sources/test_coinmarketcap/recent-btc-aud.json deleted file mode 100644 index 342b824..0000000 --- a/tests/pricehist/sources/test_coinmarketcap/recent-btc-aud.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "status": { - "timestamp": "2021-07-16T10:42:32.013Z", - "error_code": 0, - "error_message": null, - "elapsed": 20, - "credit_count": 0, - "notice": null - }, - "data": { - "id": 1, - "name": "Bitcoin", - "symbol": "BTC", - "quotes": [ - { - "time_open": "2021-01-01T00:00:00.000Z", - "time_close": "2021-01-01T23:59:59.999Z", - "time_high": "2021-01-01T12:38:43.000Z", - "time_low": "2021-01-01T00:16:43.000Z", - "quote": { - "AUD": { - "open": 37658.83948707033, - "high": 38417.9137031205, - "low": 37410.787501639206, - "close": 38181.99133300758, - "volume": 52943282221.028366, - "market_cap": 709720173049.5383, - "timestamp": "2021-01-01T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-02T00:00:00.000Z", - "time_close": "2021-01-02T23:59:59.999Z", - "time_high": "2021-01-02T19:49:42.000Z", - "time_low": "2021-01-02T00:31:44.000Z", - "quote": { - "AUD": { - "open": 38184.98611600682, - "high": 43096.681197423015, - "low": 37814.17187096531, - "close": 41760.62923079505, - "volume": 88214867181.97835, - "market_cap": 776278147177.8037, - "timestamp": "2021-01-02T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-03T00:00:00.000Z", - "time_close": "2021-01-03T23:59:59.999Z", - "time_high": "2021-01-03T07:47:38.000Z", - "time_low": "2021-01-03T00:20:45.000Z", - "quote": { - "AUD": { - "open": 41763.41015117659, - "high": 44985.93247585023, - "low": 41663.204350601605, - "close": 42511.10646879765, - "volume": 102011582370.28117, - "market_cap": 790270288834.0249, - "timestamp": "2021-01-03T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-04T00:00:00.000Z", - "time_close": "2021-01-04T23:59:59.999Z", - "time_high": "2021-01-04T04:07:42.000Z", - "time_low": "2021-01-04T10:19:42.000Z", - "quote": { - "AUD": { - "open": 42548.61349648768, - "high": 43360.96165147421, - "low": 37133.98436952697, - "close": 41686.38761359174, - "volume": 105824510346.65779, - "market_cap": 774984045201.7122, - "timestamp": "2021-01-04T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-05T00:00:00.000Z", - "time_close": "2021-01-05T23:59:59.999Z", - "time_high": "2021-01-05T22:44:35.000Z", - "time_low": "2021-01-05T06:16:41.000Z", - "quote": { - "AUD": { - "open": 41693.07321807638, - "high": 44403.79487147647, - "low": 39221.81167941294, - "close": 43790.067253370056, - "volume": 87016490203.50436, - "market_cap": 814135603090.2502, - "timestamp": "2021-01-05T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-06T00:00:00.000Z", - "time_close": "2021-01-06T23:59:59.999Z", - "time_high": "2021-01-06T23:57:36.000Z", - "time_low": "2021-01-06T00:25:38.000Z", - "quote": { - "AUD": { - "open": 43817.35864984641, - "high": 47186.65232598287, - "low": 43152.60281764236, - "close": 47115.85365360005, - "volume": 96330948324.8061, - "market_cap": 876019742889.9551, - "timestamp": "2021-01-06T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-07T00:00:00.000Z", - "time_close": "2021-01-07T23:59:59.999Z", - "time_high": "2021-01-07T18:17:42.000Z", - "time_low": "2021-01-07T08:25:51.000Z", - "quote": { - "AUD": { - "open": 47128.02139328098, - "high": 51833.478207775144, - "low": 46906.65117139608, - "close": 50686.90986207153, - "volume": 109124136558.20264, - "market_cap": 942469208700.134, - "timestamp": "2021-01-07T23:59:06.000Z" - } - } - } - ] - } -} diff --git a/tests/pricehist/sources/test_coinmarketcap/recent-btc-id2782.json b/tests/pricehist/sources/test_coinmarketcap/recent-btc-id2782.json deleted file mode 100644 index 8614b07..0000000 --- a/tests/pricehist/sources/test_coinmarketcap/recent-btc-id2782.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "status": { - "timestamp": "2021-07-16T10:42:27.169Z", - "error_code": 0, - "error_message": null, - "elapsed": 19, - "credit_count": 0, - "notice": null - }, - "data": { - "id": 1, - "name": "Bitcoin", - "symbol": "BTC", - "quotes": [ - { - "time_open": "2021-01-01T00:00:00.000Z", - "time_close": "2021-01-01T23:59:59.999Z", - "time_high": "2021-01-01T12:38:43.000Z", - "time_low": "2021-01-01T00:16:43.000Z", - "quote": { - "2782": { - "open": 37658.83948707033, - "high": 38417.9137031205, - "low": 37410.787501639206, - "close": 38181.99133300758, - "volume": 52943282221.028366, - "market_cap": 709720173049.5383, - "timestamp": "2021-01-01T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-02T00:00:00.000Z", - "time_close": "2021-01-02T23:59:59.999Z", - "time_high": "2021-01-02T19:49:42.000Z", - "time_low": "2021-01-02T00:31:44.000Z", - "quote": { - "2782": { - "open": 38184.98611600682, - "high": 43096.681197423015, - "low": 37814.17187096531, - "close": 41760.62923079505, - "volume": 88214867181.97835, - "market_cap": 776278147177.8037, - "timestamp": "2021-01-02T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-03T00:00:00.000Z", - "time_close": "2021-01-03T23:59:59.999Z", - "time_high": "2021-01-03T07:47:38.000Z", - "time_low": "2021-01-03T00:20:45.000Z", - "quote": { - "2782": { - "open": 41763.41015117659, - "high": 44985.93247585023, - "low": 41663.204350601605, - "close": 42511.10646879765, - "volume": 102011582370.28117, - "market_cap": 790270288834.0249, - "timestamp": "2021-01-03T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-04T00:00:00.000Z", - "time_close": "2021-01-04T23:59:59.999Z", - "time_high": "2021-01-04T04:07:42.000Z", - "time_low": "2021-01-04T10:19:42.000Z", - "quote": { - "2782": { - "open": 42548.61349648768, - "high": 43360.96165147421, - "low": 37133.98436952697, - "close": 41686.38761359174, - "volume": 105824510346.65779, - "market_cap": 774984045201.7122, - "timestamp": "2021-01-04T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-05T00:00:00.000Z", - "time_close": "2021-01-05T23:59:59.999Z", - "time_high": "2021-01-05T22:44:35.000Z", - "time_low": "2021-01-05T06:16:41.000Z", - "quote": { - "2782": { - "open": 41693.07321807638, - "high": 44403.79487147647, - "low": 39221.81167941294, - "close": 43790.067253370056, - "volume": 87016490203.50436, - "market_cap": 814135603090.2502, - "timestamp": "2021-01-05T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-06T00:00:00.000Z", - "time_close": "2021-01-06T23:59:59.999Z", - "time_high": "2021-01-06T23:57:36.000Z", - "time_low": "2021-01-06T00:25:38.000Z", - "quote": { - "2782": { - "open": 43817.35864984641, - "high": 47186.65232598287, - "low": 43152.60281764236, - "close": 47115.85365360005, - "volume": 96330948324.8061, - "market_cap": 876019742889.9551, - "timestamp": "2021-01-06T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-07T00:00:00.000Z", - "time_close": "2021-01-07T23:59:59.999Z", - "time_high": "2021-01-07T18:17:42.000Z", - "time_low": "2021-01-07T08:25:51.000Z", - "quote": { - "2782": { - "open": 47128.02139328098, - "high": 51833.478207775144, - "low": 46906.65117139608, - "close": 50686.90986207153, - "volume": 109124136558.20264, - "market_cap": 942469208700.134, - "timestamp": "2021-01-07T23:59:06.000Z" - } - } - } - ] - } -} diff --git a/tests/pricehist/sources/test_coinmarketcap/recent-id1-aud.json b/tests/pricehist/sources/test_coinmarketcap/recent-id1-aud.json deleted file mode 100644 index 8b70b6a..0000000 --- a/tests/pricehist/sources/test_coinmarketcap/recent-id1-aud.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "status": { - "timestamp": "2021-07-16T10:42:24.612Z", - "error_code": 0, - "error_message": null, - "elapsed": 57, - "credit_count": 0, - "notice": null - }, - "data": { - "id": 1, - "name": "Bitcoin", - "symbol": "BTC", - "quotes": [ - { - "time_open": "2021-01-01T00:00:00.000Z", - "time_close": "2021-01-01T23:59:59.999Z", - "time_high": "2021-01-01T12:38:43.000Z", - "time_low": "2021-01-01T00:16:43.000Z", - "quote": { - "AUD": { - "open": 37658.83948707033, - "high": 38417.9137031205, - "low": 37410.787501639206, - "close": 38181.99133300758, - "volume": 52943282221.028366, - "market_cap": 709720173049.5383, - "timestamp": "2021-01-01T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-02T00:00:00.000Z", - "time_close": "2021-01-02T23:59:59.999Z", - "time_high": "2021-01-02T19:49:42.000Z", - "time_low": "2021-01-02T00:31:44.000Z", - "quote": { - "AUD": { - "open": 38184.98611600682, - "high": 43096.681197423015, - "low": 37814.17187096531, - "close": 41760.62923079505, - "volume": 88214867181.97835, - "market_cap": 776278147177.8037, - "timestamp": "2021-01-02T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-03T00:00:00.000Z", - "time_close": "2021-01-03T23:59:59.999Z", - "time_high": "2021-01-03T07:47:38.000Z", - "time_low": "2021-01-03T00:20:45.000Z", - "quote": { - "AUD": { - "open": 41763.41015117659, - "high": 44985.93247585023, - "low": 41663.204350601605, - "close": 42511.10646879765, - "volume": 102011582370.28117, - "market_cap": 790270288834.0249, - "timestamp": "2021-01-03T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-04T00:00:00.000Z", - "time_close": "2021-01-04T23:59:59.999Z", - "time_high": "2021-01-04T04:07:42.000Z", - "time_low": "2021-01-04T10:19:42.000Z", - "quote": { - "AUD": { - "open": 42548.61349648768, - "high": 43360.96165147421, - "low": 37133.98436952697, - "close": 41686.38761359174, - "volume": 105824510346.65779, - "market_cap": 774984045201.7122, - "timestamp": "2021-01-04T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-05T00:00:00.000Z", - "time_close": "2021-01-05T23:59:59.999Z", - "time_high": "2021-01-05T22:44:35.000Z", - "time_low": "2021-01-05T06:16:41.000Z", - "quote": { - "AUD": { - "open": 41693.07321807638, - "high": 44403.79487147647, - "low": 39221.81167941294, - "close": 43790.067253370056, - "volume": 87016490203.50436, - "market_cap": 814135603090.2502, - "timestamp": "2021-01-05T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-06T00:00:00.000Z", - "time_close": "2021-01-06T23:59:59.999Z", - "time_high": "2021-01-06T23:57:36.000Z", - "time_low": "2021-01-06T00:25:38.000Z", - "quote": { - "AUD": { - "open": 43817.35864984641, - "high": 47186.65232598287, - "low": 43152.60281764236, - "close": 47115.85365360005, - "volume": 96330948324.8061, - "market_cap": 876019742889.9551, - "timestamp": "2021-01-06T23:59:06.000Z" - } - } - }, - { - "time_open": "2021-01-07T00:00:00.000Z", - "time_close": "2021-01-07T23:59:59.999Z", - "time_high": "2021-01-07T18:17:42.000Z", - "time_low": "2021-01-07T08:25:51.000Z", - "quote": { - "AUD": { - "open": 47128.02139328098, - "high": 51833.478207775144, - "low": 46906.65117139608, - "close": 50686.90986207153, - "volume": 109124136558.20264, - "market_cap": 942469208700.134, - "timestamp": "2021-01-07T23:59:06.000Z" - } - } - } - ] - } -} diff --git a/tests/pricehist/sources/test_coinmarketcap/recent-id1-id2782.json b/tests/pricehist/sources/test_coinmarketcap/recent-id1-id2782.json index d172453..e05e7bf 100644 --- a/tests/pricehist/sources/test_coinmarketcap/recent-id1-id2782.json +++ b/tests/pricehist/sources/test_coinmarketcap/recent-id1-id2782.json @@ -1,136 +1,129 @@ { - "status": { - "timestamp": "2021-07-16T10:42:21.065Z", - "error_code": 0, - "error_message": null, - "elapsed": 17, - "credit_count": 0, - "notice": null - }, "data": { "id": 1, "name": "Bitcoin", "symbol": "BTC", + "timeEnd": "1575503999", "quotes": [ { - "time_open": "2021-01-01T00:00:00.000Z", - "time_close": "2021-01-01T23:59:59.999Z", - "time_high": "2021-01-01T12:38:43.000Z", - "time_low": "2021-01-01T00:16:43.000Z", + "timeOpen": "2021-01-01T00:00:00.000Z", + "timeClose": "2021-01-01T23:59:59.999Z", + "timeHigh": "2021-01-01T12:38:43.000Z", + "timeLow": "2021-01-01T00:16:43.000Z", "quote": { - "2782": { - "open": 37658.83948707033, - "high": 38417.9137031205, - "low": 37410.787501639206, - "close": 38181.99133300758, - "volume": 52943282221.028366, - "market_cap": 709720173049.5383, - "timestamp": "2021-01-01T23:59:06.000Z" - } + "name": "2782", + "open": 37658.1146368474, + "high": 38417.9137031205, + "low": 37410.7875016392, + "close": 38181.9913330076, + "volume": 52901492931.8344367080, + "marketCap": 709159975413.2388897949, + "timestamp": "2021-01-01T23:59:59.999Z" } }, { - "time_open": "2021-01-02T00:00:00.000Z", - "time_close": "2021-01-02T23:59:59.999Z", - "time_high": "2021-01-02T19:49:42.000Z", - "time_low": "2021-01-02T00:31:44.000Z", + "timeOpen": "2021-01-02T00:00:00.000Z", + "timeClose": "2021-01-02T23:59:59.999Z", + "timeHigh": "2021-01-02T19:49:42.000Z", + "timeLow": "2021-01-02T00:31:44.000Z", "quote": { - "2782": { - "open": 38184.98611600682, - "high": 43096.681197423015, - "low": 37814.17187096531, - "close": 41760.62923079505, - "volume": 88214867181.97835, - "market_cap": 776278147177.8037, - "timestamp": "2021-01-02T23:59:06.000Z" - } + "name": "2782", + "open": 38184.9861160068, + "high": 43096.6811974230, + "low": 37814.1718709653, + "close": 41760.6292307951, + "volume": 88214867181.9830439141, + "marketCap": 776278147177.8037261338, + "timestamp": "2021-01-02T23:59:59.999Z" } }, { - "time_open": "2021-01-03T00:00:00.000Z", - "time_close": "2021-01-03T23:59:59.999Z", - "time_high": "2021-01-03T07:47:38.000Z", - "time_low": "2021-01-03T00:20:45.000Z", + "timeOpen": "2021-01-03T00:00:00.000Z", + "timeClose": "2021-01-03T23:59:59.999Z", + "timeHigh": "2021-01-03T07:47:38.000Z", + "timeLow": "2021-01-03T00:20:45.000Z", "quote": { - "2782": { - "open": 41763.41015117659, - "high": 44985.93247585023, - "low": 41663.204350601605, - "close": 42511.10646879765, - "volume": 102011582370.28117, - "market_cap": 790270288834.0249, - "timestamp": "2021-01-03T23:59:06.000Z" - } + "name": "2782", + "open": 41763.4101511766, + "high": 44985.9324758502, + "low": 41663.2043506016, + "close": 42534.0538859236, + "volume": 102253005977.1115650988, + "marketCap": 792140565709.1701340036, + "timestamp": "2021-01-03T23:59:59.999Z" } }, { - "time_open": "2021-01-04T00:00:00.000Z", - "time_close": "2021-01-04T23:59:59.999Z", - "time_high": "2021-01-04T04:07:42.000Z", - "time_low": "2021-01-04T10:19:42.000Z", + "timeOpen": "2021-01-04T00:00:00.000Z", + "timeClose": "2021-01-04T23:59:59.999Z", + "timeHigh": "2021-01-04T04:07:42.000Z", + "timeLow": "2021-01-04T10:19:42.000Z", "quote": { - "2782": { - "open": 42548.61349648768, - "high": 43360.96165147421, - "low": 37133.98436952697, - "close": 41686.38761359174, - "volume": 105824510346.65779, - "market_cap": 774984045201.7122, - "timestamp": "2021-01-04T23:59:06.000Z" - } + "name": "2782", + "open": 42548.6134964877, + "high": 43347.7527651400, + "low": 37111.8678479690, + "close": 41707.4890765162, + "volume": 105251252720.3013091567, + "marketCap": 770785910830.3801120744, + "timestamp": "2021-01-04T23:59:59.999Z" } }, { - "time_open": "2021-01-05T00:00:00.000Z", - "time_close": "2021-01-05T23:59:59.999Z", - "time_high": "2021-01-05T22:44:35.000Z", - "time_low": "2021-01-05T06:16:41.000Z", + "timeOpen": "2021-01-05T00:00:00.000Z", + "timeClose": "2021-01-05T23:59:59.999Z", + "timeHigh": "2021-01-05T22:44:35.000Z", + "timeLow": "2021-01-05T06:16:41.000Z", "quote": { - "2782": { - "open": 41693.07321807638, - "high": 44403.79487147647, - "low": 39221.81167941294, - "close": 43790.067253370056, - "volume": 87016490203.50436, - "market_cap": 814135603090.2502, - "timestamp": "2021-01-05T23:59:06.000Z" - } + "name": "2782", + "open": 41693.0732180764, + "high": 44406.6531914952, + "low": 39220.9654861842, + "close": 43777.4560620835, + "volume": 88071174132.6445648582, + "marketCap": 824003338903.4613958343, + "timestamp": "2021-01-05T23:59:59.999Z" } }, { - "time_open": "2021-01-06T00:00:00.000Z", - "time_close": "2021-01-06T23:59:59.999Z", - "time_high": "2021-01-06T23:57:36.000Z", - "time_low": "2021-01-06T00:25:38.000Z", + "timeOpen": "2021-01-06T00:00:00.000Z", + "timeClose": "2021-01-06T23:59:59.999Z", + "timeHigh": "2021-01-06T23:57:36.000Z", + "timeLow": "2021-01-06T00:25:38.000Z", "quote": { - "2782": { - "open": 43817.35864984641, - "high": 47186.65232598287, - "low": 43152.60281764236, - "close": 47115.85365360005, - "volume": 96330948324.8061, - "market_cap": 876019742889.9551, - "timestamp": "2021-01-06T23:59:06.000Z" - } + "name": "2782", + "open": 43798.3790529373, + "high": 47185.7303335186, + "low": 43152.6028176424, + "close": 47114.9330444897, + "volume": 96948095813.7503737302, + "marketCap": 881631993096.0701475336, + "timestamp": "2021-01-06T23:59:59.999Z" } }, { - "time_open": "2021-01-07T00:00:00.000Z", - "time_close": "2021-01-07T23:59:59.999Z", - "time_high": "2021-01-07T18:17:42.000Z", - "time_low": "2021-01-07T08:25:51.000Z", + "timeOpen": "2021-01-07T00:00:00.000Z", + "timeClose": "2021-01-07T23:59:59.999Z", + "timeHigh": "2021-01-07T18:17:42.000Z", + "timeLow": "2021-01-07T08:25:51.000Z", "quote": { - "2782": { - "open": 47128.02139328098, - "high": 51833.478207775144, - "low": 46906.65117139608, - "close": 50686.90986207153, - "volume": 109124136558.20264, - "market_cap": 942469208700.134, - "timestamp": "2021-01-07T23:59:06.000Z" - } + "name": "2782", + "open": 47128.0213932810, + "high": 51832.6746004172, + "low": 46906.6511713961, + "close": 50660.9643451606, + "volume": 108451040396.2660095877, + "marketCap": 936655898949.2177196744, + "timestamp": "2021-01-07T23:59:59.999Z" } } ] + }, + "status": { + "timestamp": "2024-08-02T18:23:21.586Z", + "error_code": "0", + "error_message": "SUCCESS", + "elapsed": "212", + "credit_count": 0 } } From 9eb6de4c440e7989ea86db9c8f2ce812abf54f5f Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 3 Aug 2024 12:40:28 +0200 Subject: [PATCH 133/149] live tests: reactivate coinmarketcap, update alphavantage physical for new data. --- tests/live.sh | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/live.sh b/tests/live.sh index a25b93d..dd5f626 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -72,14 +72,13 @@ END run_test "$name" "$cmd" "$expected" name="Alpha Vantage physical currency" -cmd="pricehist fetch alphavantage AUD/EUR -s 2021-01-10 -e 2021-01-14" +cmd="pricehist fetch alphavantage AUD/EUR -s 2021-01-11 -e 2021-01-14" read -r -d '' expected < Date: Sat, 3 Aug 2024 12:54:11 +0200 Subject: [PATCH 134/149] Update coinmarketcap source notes. --- src/pricehist/sources/coinmarketcap.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/pricehist/sources/coinmarketcap.py b/src/pricehist/sources/coinmarketcap.py index f7e0fb8..58eeaaf 100644 --- a/src/pricehist/sources/coinmarketcap.py +++ b/src/pricehist/sources/coinmarketcap.py @@ -33,13 +33,16 @@ class CoinMarketCap(BaseSource): def notes(self): return ( - "This source makes unoffical use of endpoints that power CoinMarketCap's " - "public web interface. The price data comes from a public equivalent of " - "the OHLCV Historical endpoint found in CoinMarketCap's official API.\n" - "CoinMarketCap currency symbols are not necessarily unique, so it " - "is recommended that you use IDs, which can be listed via the " - "--symbols option. For example, 'ETH/BTC' is 'id=1027/id=1'. The " - "corresponding symbols will be used in output." + "This source makes unoffical use of endpoints that power " + "CoinMarketCap's public web interface.\n" + "CoinMarketCap currency symbols are not necessarily unique. " + "Each symbol you give will be coverted an ID by checking fiat and " + "metals first, then crypto by CoinMarketCap rank. " + "The symbol data is hard-coded for fiat and metals, but fetched " + "live for crypto.\n" + "You can directly use IDs, which can be listed via the --symbols " + "option. For example, 'ETH/BTC' is 'id=1027/id=1'. " + "The corresponding symbols will be used in output, when available." ) def symbols(self): From 1e1003994ce84206082caac3a204a5e0bbb618f7 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 3 Aug 2024 13:04:25 +0200 Subject: [PATCH 135/149] Version 1.4.8. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 116373b..9cf089a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.7" +version = "1.4.8" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index ac329c9..4963389 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.7" +__version__ = "1.4.8" From e8dec0bf64310ebf9e33af19c455ee3c72631c55 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 3 Aug 2024 17:15:17 +0200 Subject: [PATCH 136/149] Update Alpha Vantage rate limit handling. --- src/pricehist/sources/alphavantage.py | 4 ++-- tests/pricehist/sources/test_alphavantage.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index e60f0f6..c42d490 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -337,8 +337,8 @@ class AlphaVantage(BaseSource): def _raise_for_generic_errors(self, data): if type(data) is dict: - if "Note" in data and "call frequency" in data["Note"]: - raise exceptions.RateLimit(data["Note"]) + if "Information" in data and "daily rate limits" in data["Information"]: + raise exceptions.RateLimit(data["Information"]) if ( "Information" in data and "unlock" in data["Information"] diff --git a/tests/pricehist/sources/test_alphavantage.py b/tests/pricehist/sources/test_alphavantage.py index 8ff70cc..40febb0 100644 --- a/tests/pricehist/sources/test_alphavantage.py +++ b/tests/pricehist/sources/test_alphavantage.py @@ -59,11 +59,11 @@ digital_url = re.compile( ) rate_limit_json = ( - '{ "Note": "' - "Thank you for using Alpha Vantage! Our standard API call frequency is 5 " - "calls per minute and 500 calls per day. Please visit " - "https://www.alphavantage.co/premium/ if you would like to target a higher " - "API call frequency." + '{ "Information": "' + "Thank you for using Alpha Vantage! Our standard API rate limit is 25 " + "requests per day. Please subscribe to any of the premium plans at " + "https://www.alphavantage.co/premium/ to instantly remove all daily rate " + "limits." '" }' ) From b7d0d739ab9841ef05e0793cac9041b0843640ff Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 3 Aug 2024 17:19:36 +0200 Subject: [PATCH 137/149] Version 1.4.9. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9cf089a..d1a7ce1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.8" +version = "1.4.9" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 4963389..1ad354e 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.8" +__version__ = "1.4.9" From 51e297b75252b44e4c8ba3f2bffbeb734c820b8f Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 3 Aug 2024 17:23:53 +0200 Subject: [PATCH 138/149] Update alphavantage source notes regarding API rate limit. --- src/pricehist/sources/alphavantage.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py index c42d490..40d5b98 100644 --- a/src/pricehist/sources/alphavantage.py +++ b/src/pricehist/sources/alphavantage.py @@ -56,10 +56,8 @@ class AlphaVantage(BaseSource): "Beware that digital currencies quoted in non-USD currencies may " "be converted from USD data at one recent exchange rate rather " "than using historical rates.\n" - "Alpha Vantage's standard API call frequency limits is 5 calls per " - "minute and 500 per day, so you may need to pause between successive " - "commands. Note that retrieving prices for one stock consumes two " - "API calls." + "Alpha Vantage's standard API rate limit is 25 requests per day. " + "Note that retrieving prices for one stock consumes two API calls." ) def _stock_symbols_message(self): From 59574e91566d80f970a90009064cf967662d0043 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 14 Sep 2024 22:22:35 +0200 Subject: [PATCH 139/149] Fix yahoo source. --- src/pricehist/sources/yahoo.py | 103 +++----- tests/live.sh | 35 ++- tests/pricehist/sources/test_yahoo.py | 181 ++++--------- .../test_yahoo/ibm-date-with-nulls.csv | 4 - .../sources/test_yahoo/ibm-long-partial.csv | 11 - .../sources/test_yahoo/ibm-long-partial.json | 249 ++++++++++++++++++ .../sources/test_yahoo/tsla-recent.csv | 6 - .../sources/test_yahoo/tsla-recent.json | 126 +++++++++ .../sources/test_yahoo/tsla-spark.json | 77 ------ 9 files changed, 475 insertions(+), 317 deletions(-) delete mode 100644 tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv delete mode 100644 tests/pricehist/sources/test_yahoo/ibm-long-partial.csv create mode 100644 tests/pricehist/sources/test_yahoo/ibm-long-partial.json delete mode 100644 tests/pricehist/sources/test_yahoo/tsla-recent.csv create mode 100644 tests/pricehist/sources/test_yahoo/tsla-recent.json delete mode 100644 tests/pricehist/sources/test_yahoo/tsla-spark.json diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 6997301..ce5cf7d 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -1,4 +1,3 @@ -import csv import dataclasses import json import logging @@ -71,63 +70,36 @@ class Yahoo(BaseSource): series.base, series.quote, self, "Don't specify the quote currency." ) - quote, history = self._data(series) + data = self._data(series) + quote = data["chart"]["result"][0]["meta"]["currency"] + + timestamps = data["chart"]["result"][0]["timestamp"] + adjclose_data = data["chart"]["result"][0]["indicators"]["adjclose"][0] + rest_data = data["chart"]["result"][0]["indicators"]["quote"][0] + amounts = {**adjclose_data, **rest_data} prices = [ - Price(row["date"], amount) - for row in history - if (amount := self._amount(row, series.type)) + Price(ts, amount) + for i in range(len(timestamps)) + if (ts := datetime.fromtimestamp(timestamps[i]).strftime("%Y-%m-%d")) + <= series.end + if (amount := self._amount(amounts, series.type, i)) is not None ] return dataclasses.replace(series, quote=quote, prices=prices) - def _amount(self, row, type): - if type == "mid" and row["high"] != "null" and row["low"] != "null": - return sum([Decimal(row["high"]), Decimal(row["low"])]) / 2 - elif row[type] != "null": - return Decimal(row[type]) + def _amount(self, amounts, type, i): + if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": + return sum([Decimal(amounts["high"][i]), Decimal(amounts["low"][i])]) / 2 + elif amounts[type] != "null": + return Decimal(amounts[type][i]) else: return None - def _data(self, series) -> (dict, csv.DictReader): - base_url = "https://query1.finance.yahoo.com/v7/finance" + def _data(self, series) -> dict: + base_url = "https://query1.finance.yahoo.com/v8/finance/chart" headers = {"User-Agent": f"pricehist/{__version__}"} - - spark_url = f"{base_url}/spark" - spark_params = { - "symbols": series.base, - "range": "1d", - "interval": "1d", - "indicators": "close", - "includeTimestamps": "false", - "includePrePost": "false", - } - try: - spark_response = self.log_curl( - requests.get(spark_url, params=spark_params, headers=headers) - ) - except Exception as e: - raise exceptions.RequestError(str(e)) from e - - code = spark_response.status_code - text = spark_response.text - if code == 404 and "No data found for spark symbols" in text: - raise exceptions.InvalidPair( - series.base, series.quote, self, "Symbol not found." - ) - - try: - spark_response.raise_for_status() - except Exception as e: - raise exceptions.BadResponse(str(e)) from e - - try: - spark = json.loads(spark_response.content) - quote = spark["spark"]["result"][0]["response"][0]["meta"]["currency"] - except Exception as e: - raise exceptions.ResponseParsingError( - "The spark data couldn't be parsed. " - ) from e + url = f"{base_url}/{series.base}" start_ts = int( datetime.strptime(series.start, "%Y-%m-%d") @@ -142,24 +114,26 @@ class Yahoo(BaseSource): 24 * 60 * 60 ) # some symbols require padding on the end timestamp - history_url = f"{base_url}/download/{series.base}" - history_params = { + params = { + "symbol": series.base, "period1": start_ts, "period2": end_ts, "interval": "1d", - "events": "history", + "events": "capitalGain%7Cdiv%7Csplit", "includeAdjustedClose": "true", + "formatted": "true", + "userYfid": "true", + "lang": "en-US", + "region": "US", } try: - history_response = self.log_curl( - requests.get(history_url, params=history_params, headers=headers) - ) + response = self.log_curl(requests.get(url, params=params, headers=headers)) except Exception as e: raise exceptions.RequestError(str(e)) from e - code = history_response.status_code - text = history_response.text + code = response.status_code + text = response.text if code == 404 and "No data found, symbol may be delisted" in text: raise exceptions.InvalidPair( @@ -177,20 +151,15 @@ class Yahoo(BaseSource): ) try: - history_response.raise_for_status() + response.raise_for_status() except Exception as e: raise exceptions.BadResponse(str(e)) from e try: - history_lines = history_response.content.decode("utf-8").splitlines() - history_lines[0] = history_lines[0].lower().replace(" ", "") - history = csv.DictReader(history_lines, delimiter=",") + data = json.loads(response.content) except Exception as e: - raise exceptions.ResponseParsingError(str(e)) from e + raise exceptions.ResponseParsingError( + "The data couldn't be parsed. " + ) from e - if history_lines[0] != "date,open,high,low,close,adjclose,volume": - raise exceptions.ResponseParsingError("Unexpected CSV format") - - requested_history = [row for row in history if row["date"] <= series.end] - - return (quote, requested_history) + return data diff --git a/tests/live.sh b/tests/live.sh index dd5f626..fc24c08 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -116,21 +116,20 @@ date,base,quote,amount,source,type 2021-01-07,BTC,EUR,31208.49,coinbasepro,mid 2021-01-08,BTC,EUR,32019,coinbasepro,mid END -run_test "$name" "$cmd" "$expected" - -name="CoinDesk Bitcoin Price Index" -cmd="pricehist fetch coindesk BTC/EUR -s 2021-01-04 -e 2021-01-08" -read -r -d '' expected < 9 -def test_fetch_skips_dates_with_nulls(src, type, spark_ok, date_with_nulls_ok): - series = src.fetch(Series("IBM", "", type, "2021-01-05", "2021-01-07")) - assert series.prices[0] == Price("2021-01-05", Decimal("123.101204")) - assert series.prices[1] == Price("2021-01-07", Decimal("125.882545")) - assert len(series.prices) == 2 - - -def test_fetch_to_future(src, type, spark_ok, recent_ok): +def test_fetch_to_future(src, type, recent_ok): series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2100-01-08")) assert len(series.prices) > 0 -def test_fetch_no_data_in_past(src, type, spark_ok, requests_mock): +def test_fetch_no_data_in_past(src, type, requests_mock): requests_mock.add( responses.GET, - history_url("TSLA"), + url("TSLA"), status=400, body=( "400 Bad Request: Data doesn't exist for " @@ -203,10 +177,10 @@ def test_fetch_no_data_in_past(src, type, spark_ok, requests_mock): assert "No data for the given interval" in str(e.value) -def test_fetch_no_data_in_future(src, type, spark_ok, requests_mock): +def test_fetch_no_data_in_future(src, type, requests_mock): requests_mock.add( responses.GET, - history_url("TSLA"), + url("TSLA"), status=400, body=( "400 Bad Request: Data doesn't exist for " @@ -218,10 +192,10 @@ def test_fetch_no_data_in_future(src, type, spark_ok, requests_mock): assert "No data for the given interval" in str(e.value) -def test_fetch_no_data_on_weekend(src, type, spark_ok, requests_mock): +def test_fetch_no_data_on_weekend(src, type, requests_mock): requests_mock.add( responses.GET, - history_url("TSLA"), + url("TSLA"), status=404, body="404 Not Found: Timestamp data missing.", ) @@ -233,30 +207,7 @@ def test_fetch_no_data_on_weekend(src, type, spark_ok, requests_mock): def test_fetch_bad_sym(src, type, requests_mock): requests_mock.add( responses.GET, - spark_url, - status=404, - body="""{ - "spark": { - "result": null, - "error": { - "code": "Not Found", - "description": "No data found for spark symbols" - } - } - }""", - ) - with pytest.raises(exceptions.InvalidPair) as e: - src.fetch(Series("NOTABASE", "", type, "2021-01-04", "2021-01-08")) - assert "Symbol not found" in str(e.value) - - -def test_fetch_bad_sym_history(src, type, spark_ok, requests_mock): - # In practice the spark history requests should succeed or fail together. - # This extra test ensures that a failure of the the history part is handled - # correctly even if the spark part succeeds. - requests_mock.add( - responses.GET, - history_url("NOTABASE"), + url("NOTABASE"), status=404, body="404 Not Found: No data found, symbol may be delisted", ) @@ -271,61 +222,23 @@ def test_fetch_giving_quote(src, type): assert "quote currency" in str(e.value) -def test_fetch_spark_network_issue(src, type, requests_mock): +def test_fetch_network_issue(src, type, requests_mock): body = requests.exceptions.ConnectionError("Network issue") - requests_mock.add(responses.GET, spark_url, body=body) + requests_mock.add(responses.GET, url("TSLA"), body=body) with pytest.raises(exceptions.RequestError) as e: src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) assert "Network issue" in str(e.value) -def test_fetch_spark_bad_status(src, type, requests_mock): - requests_mock.add(responses.GET, spark_url, status=500, body="Some other reason") +def test_fetch_bad_status(src, type, requests_mock): + requests_mock.add(responses.GET, url("TSLA"), status=500, body="Some other reason") with pytest.raises(exceptions.BadResponse) as e: src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) assert "Internal Server Error" in str(e.value) -def test_fetch_spark_parsing_error(src, type, requests_mock): - requests_mock.add(responses.GET, spark_url, body="NOT JSON") - with pytest.raises(exceptions.ResponseParsingError) as e: - src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) - assert "spark data couldn't be parsed" in str(e.value) - - -def test_fetch_spark_unexpected_json(src, type, requests_mock): - requests_mock.add(responses.GET, spark_url, body='{"notdata": []}') - with pytest.raises(exceptions.ResponseParsingError) as e: - src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) - assert "spark data couldn't be parsed" in str(e.value) - - -def test_fetch_history_network_issue(src, type, spark_ok, requests_mock): - body = requests.exceptions.ConnectionError("Network issue") - requests_mock.add(responses.GET, history_url("TSLA"), body=body) - with pytest.raises(exceptions.RequestError) as e: - src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) - assert "Network issue" in str(e.value) - - -def test_fetch_history_bad_status(src, type, spark_ok, requests_mock): - requests_mock.add( - responses.GET, history_url("TSLA"), status=500, body="Some other reason" - ) - with pytest.raises(exceptions.BadResponse) as e: - src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) - assert "Internal Server Error" in str(e.value) - - -def test_fetch_history_parsing_error(src, type, spark_ok, requests_mock): - requests_mock.add(responses.GET, history_url("TSLA"), body="") +def test_fetch_parsing_error(src, type, requests_mock): + requests_mock.add(responses.GET, url("TSLA"), body="") with pytest.raises(exceptions.ResponseParsingError) as e: src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) assert "error occurred while parsing data from the source" in str(e.value) - - -def test_fetch_history_unexpected_csv_format(src, type, spark_ok, requests_mock): - requests_mock.add(responses.GET, history_url("TSLA"), body="BAD HEADER\nBAD DATA") - with pytest.raises(exceptions.ResponseParsingError) as e: - src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) - assert "Unexpected CSV format" in str(e.value) diff --git a/tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv b/tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv deleted file mode 100644 index 601b395..0000000 --- a/tests/pricehist/sources/test_yahoo/ibm-date-with-nulls.csv +++ /dev/null @@ -1,4 +0,0 @@ -Date,Open,High,Low,Close,Adj Close,Volume -2021-01-05,125.010002,126.680000,124.610001,126.139999,123.101204,6114600 -2021-01-06,null,null,null,null,null,null -2021-01-07,130.039993,130.460007,128.259995,128.990005,125.882545,4507400 diff --git a/tests/pricehist/sources/test_yahoo/ibm-long-partial.csv b/tests/pricehist/sources/test_yahoo/ibm-long-partial.csv deleted file mode 100644 index 98149ad..0000000 --- a/tests/pricehist/sources/test_yahoo/ibm-long-partial.csv +++ /dev/null @@ -1,11 +0,0 @@ -Date,Open,High,Low,Close,Adj Close,Volume -1962-01-02,7.713333,7.713333,7.626667,7.626667,1.837710,390000 -1962-01-03,7.626667,7.693333,7.626667,7.693333,1.853774,292500 -1962-01-04,7.693333,7.693333,7.613333,7.616667,1.835299,262500 -1962-01-05,7.606667,7.606667,7.453333,7.466667,1.799155,367500 -1962-01-08,7.460000,7.460000,7.266667,7.326667,1.765422,547500 -2021-01-04,125.849998,125.919998,123.040001,123.940002,120.954201,5179200 -2021-01-05,125.010002,126.680000,124.610001,126.139999,123.101204,6114600 -2021-01-06,126.900002,131.880005,126.720001,129.289993,126.175316,7956700 -2021-01-07,130.039993,130.460007,128.259995,128.990005,125.882545,4507400 -2021-01-08,128.570007,129.320007,126.980003,128.529999,125.433624,4676200 diff --git a/tests/pricehist/sources/test_yahoo/ibm-long-partial.json b/tests/pricehist/sources/test_yahoo/ibm-long-partial.json new file mode 100644 index 0000000..df98efa --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/ibm-long-partial.json @@ -0,0 +1,249 @@ +{ + "chart": { + "result": [ + { + "meta": { + "currency": "USD", + "symbol": "IBM", + "exchangeName": "NYQ", + "fullExchangeName": "NYSE", + "instrumentType": "EQUITY", + "firstTradeDate": -252322200, + "regularMarketTime": 1726257602, + "hasPrePostMarketData": true, + "gmtoffset": -14400, + "timezone": "EDT", + "exchangeTimezoneName": "America/New_York", + "regularMarketPrice": 214.79, + "fiftyTwoWeekHigh": 216.08, + "fiftyTwoWeekLow": 212.13, + "regularMarketDayHigh": 216.08, + "regularMarketDayLow": 212.13, + "regularMarketVolume": 4553547, + "longName": "International Business Machines Corporation", + "shortName": "International Business Machines", + "chartPreviousClose": 7.291, + "priceHint": 2, + "currentTradingPeriod": { + "pre": { + "timezone": "EDT", + "end": 1726234200, + "start": 1726214400, + "gmtoffset": -14400 + }, + "regular": { + "timezone": "EDT", + "end": 1726257600, + "start": 1726234200, + "gmtoffset": -14400 + }, + "post": { + "timezone": "EDT", + "end": 1726272000, + "start": 1726257600, + "gmtoffset": -14400 + } + }, + "dataGranularity": "1d", + "range": "", + "validRanges": [ + "1d", + "5d", + "1mo", + "3mo", + "6mo", + "1y", + "2y", + "5y", + "10y", + "ytd", + "max" + ] + }, + "timestamp": [ + -252322200, + -252235800, + -252149400, + -252063000, + -251803800, + 1609770600, + 1609857000, + 1609943400, + 1610029800, + 1610116200 + ], + "events": { + "dividends": { + "-249298200": { + "amount": 0.000956, + "date": -249298200 + }, + "-241439400": { + "amount": 0.000956, + "date": -241439400 + }, + "-233577000": { + "amount": 0.000956, + "date": -233577000 + }, + "-225797400": { + "amount": 0.000956, + "date": -225797400 + }, + "-217848600": { + "amount": 0.001275, + "date": -217848600 + }, + "1573137000": { + "amount": 1.548757, + "date": 1573137000 + }, + "1581085800": { + "amount": 1.548757, + "date": 1581085800 + }, + "1588858200": { + "amount": 1.558317, + "date": 1588858200 + }, + "1596807000": { + "amount": 1.558317, + "date": 1596807000 + }, + "1604932200": { + "amount": 1.558317, + "date": 1604932200 + } + }, + "splits": { + "-177417000": { + "date": -177417000, + "numerator": 5.0, + "denominator": 4.0, + "splitRatio": "5:4" + }, + "-114345000": { + "date": -114345000, + "numerator": 3.0, + "denominator": 2.0, + "splitRatio": "3:2" + }, + "-53343000": { + "date": -53343000, + "numerator": 2.0, + "denominator": 1.0, + "splitRatio": "2:1" + }, + "107530200": { + "date": 107530200, + "numerator": 5.0, + "denominator": 4.0, + "splitRatio": "5:4" + }, + "297091800": { + "date": 297091800, + "numerator": 4.0, + "denominator": 1.0, + "splitRatio": "4:1" + }, + "864826200": { + "date": 864826200, + "numerator": 2.0, + "denominator": 1.0, + "splitRatio": "2:1" + }, + "927811800": { + "date": 927811800, + "numerator": 2.0, + "denominator": 1.0, + "splitRatio": "2:1" + } + } + }, + "indicators": { + "quote": [ + { + "close": [ + 7.2912678718566895, + 7.3550028800964355, + 7.281707763671875, + 7.138305187225342, + 7.00446081161499, + 118.48948669433594, + 120.59273529052734, + 123.60420989990234, + 123.31739807128906, + 122.87763214111328 + ], + "low": [ + 7.2912678718566895, + 7.2912678718566895, + 7.2785210609436035, + 7.125557899475098, + 6.9471001625061035, + 117.62906646728516, + 119.13002014160156, + 121.14722442626953, + 122.61949920654297, + 121.39579010009766 + ], + "open": [ + 7.374124050140381, + 7.2912678718566895, + 7.3550028800964355, + 7.272148132324219, + 7.131930828094482, + 120.31549072265625, + 119.5124282836914, + 121.3193130493164, + 124.32122039794922, + 122.9158706665039 + ], + "high": [ + 7.374124050140381, + 7.3550028800964355, + 7.3550028800964355, + 7.272148132324219, + 7.131930828094482, + 120.38240814208984, + 121.1089859008789, + 126.08030700683594, + 124.7227554321289, + 123.63288879394531 + ], + "volume": [ + 407940, + 305955, + 274575, + 384405, + 572685, + 5417443, + 6395872, + 8322708, + 4714740, + 4891305 + ] + } + ], + "adjclose": [ + { + "adjclose": [ + 1.5133211612701416, + 1.5265485048294067, + 1.5113375186920166, + 1.4815733432769775, + 1.4537923336029053, + 99.60364532470703, + 101.37164306640625, + 103.90313720703125, + 103.66202545166016, + 103.29237365722656 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/tests/pricehist/sources/test_yahoo/tsla-recent.csv b/tests/pricehist/sources/test_yahoo/tsla-recent.csv deleted file mode 100644 index 48b5692..0000000 --- a/tests/pricehist/sources/test_yahoo/tsla-recent.csv +++ /dev/null @@ -1,6 +0,0 @@ -Date,Open,High,Low,Close,Adj Close,Volume -2021-01-04,719.460022,744.489990,717.190002,729.770020,729.770020,48638200 -2021-01-05,723.659973,740.840027,719.200012,735.109985,735.109985,32245200 -2021-01-06,758.489990,774.000000,749.099976,755.979980,755.979980,44700000 -2021-01-07,777.630005,816.989990,775.200012,816.039978,816.039978,51498900 -2021-01-08,856.000000,884.489990,838.390015,880.020020,880.020020,75055500 \ No newline at end of file diff --git a/tests/pricehist/sources/test_yahoo/tsla-recent.json b/tests/pricehist/sources/test_yahoo/tsla-recent.json new file mode 100644 index 0000000..3f35daa --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/tsla-recent.json @@ -0,0 +1,126 @@ +{ + "chart": { + "result": [ + { + "meta": { + "currency": "USD", + "symbol": "TSLA", + "exchangeName": "NMS", + "fullExchangeName": "NasdaqGS", + "instrumentType": "EQUITY", + "firstTradeDate": 1277818200, + "regularMarketTime": 1726257600, + "hasPrePostMarketData": true, + "gmtoffset": -14400, + "timezone": "EDT", + "exchangeTimezoneName": "America/New_York", + "regularMarketPrice": 230.29, + "fiftyTwoWeekHigh": 232.664, + "fiftyTwoWeekLow": 226.32, + "regularMarketDayHigh": 232.664, + "regularMarketDayLow": 226.32, + "regularMarketVolume": 59096538, + "longName": "Tesla, Inc.", + "shortName": "Tesla, Inc.", + "chartPreviousClose": 235.223, + "priceHint": 2, + "currentTradingPeriod": { + "pre": { + "timezone": "EDT", + "start": 1726214400, + "end": 1726234200, + "gmtoffset": -14400 + }, + "regular": { + "timezone": "EDT", + "start": 1726234200, + "end": 1726257600, + "gmtoffset": -14400 + }, + "post": { + "timezone": "EDT", + "start": 1726257600, + "end": 1726272000, + "gmtoffset": -14400 + } + }, + "dataGranularity": "1d", + "range": "", + "validRanges": [ + "1d", + "5d", + "1mo", + "3mo", + "6mo", + "1y", + "2y", + "5y", + "10y", + "ytd", + "max" + ] + }, + "timestamp": [ + 1609770600, + 1609857000, + 1609943400, + 1610029800, + 1610116200 + ], + "indicators": { + "quote": [ + { + "open": [ + 239.82000732421875, + 241.22000122070312, + 252.8300018310547, + 259.2099914550781, + 285.3333435058594 + ], + "close": [ + 243.2566680908203, + 245.0366668701172, + 251.9933319091797, + 272.0133361816406, + 293.3399963378906 + ], + "high": [ + 248.163330078125, + 246.94667053222656, + 258.0, + 272.3299865722656, + 294.8299865722656 + ], + "low": [ + 239.06333923339844, + 239.73333740234375, + 249.6999969482422, + 258.3999938964844, + 279.46331787109375 + ], + "volume": [ + 145914600, + 96735600, + 134100000, + 154496700, + 225166500 + ] + } + ], + "adjclose": [ + { + "adjclose": [ + 243.2566680908203, + 245.0366668701172, + 251.9933319091797, + 272.0133361816406, + 293.3399963378906 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/tests/pricehist/sources/test_yahoo/tsla-spark.json b/tests/pricehist/sources/test_yahoo/tsla-spark.json deleted file mode 100644 index 53e7585..0000000 --- a/tests/pricehist/sources/test_yahoo/tsla-spark.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "spark": { - "result": [ - { - "symbol": "TSLA", - "response": [ - { - "meta": { - "currency": "USD", - "symbol": "TSLA", - "exchangeName": "NMS", - "instrumentType": "EQUITY", - "firstTradeDate": 1277818200, - "regularMarketTime": 1626465603, - "gmtoffset": -14400, - "timezone": "EDT", - "exchangeTimezoneName": "America/New_York", - "regularMarketPrice": 644.22, - "chartPreviousClose": 650.6, - "priceHint": 2, - "currentTradingPeriod": { - "pre": { - "timezone": "EDT", - "start": 1626422400, - "end": 1626442200, - "gmtoffset": -14400 - }, - "regular": { - "timezone": "EDT", - "start": 1626442200, - "end": 1626465600, - "gmtoffset": -14400 - }, - "post": { - "timezone": "EDT", - "start": 1626465600, - "end": 1626480000, - "gmtoffset": -14400 - } - }, - "dataGranularity": "1d", - "range": "1d", - "validRanges": [ - "1d", - "5d", - "1mo", - "3mo", - "6mo", - "1y", - "2y", - "5y", - "10y", - "ytd", - "max" - ] - }, - "timestamp": [ - 1626442200, - 1626465603 - ], - "indicators": { - "quote": [ - { - "close": [ - 644.22, - 644.22 - ] - } - ] - } - } - ] - } - ], - "error": null - } -} From 5e75759b0fa8c14bc7ab88e5fd7d93e0cf3b08a2 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 14 Sep 2024 22:24:46 +0200 Subject: [PATCH 140/149] Version 1.4.10. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d1a7ce1..d5fc989 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.9" +version = "1.4.10" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 1ad354e..738cf25 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.9" +__version__ = "1.4.10" From b6f4c175303e6470b24ded7a4112b81abc40ed7c Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sat, 14 Sep 2024 22:49:45 +0200 Subject: [PATCH 141/149] Skip coindesk live test. --- tests/live.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/live.sh b/tests/live.sh index fc24c08..cb2dbeb 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -128,7 +128,7 @@ date,base,quote,amount,source,type 2021-01-07,BTC,USD,39713.5079,coindeskbpi,close 2021-01-08,BTC,USD,40519.4486,coindeskbpi,close END -run_test "$name" "$cmd" "$expected" +skip_test "$name" "$cmd" "$expected" name="CoinMarketCap" cmd="pricehist fetch coinmarketcap BTC/EUR -s 2021-01-04 -e 2021-01-08" From ee8ca0573d73765f298a83961eb2eb1583173f48 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 12:46:43 +0200 Subject: [PATCH 142/149] yahoo: add back null handling, improve timestamp handling. Thanks @arkn98! --- src/pricehist/sources/yahoo.py | 15 +++++++++++---- tests/pricehist/sources/test_yahoo.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index ce5cf7d..65857bb 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -72,6 +72,7 @@ class Yahoo(BaseSource): data = self._data(series) quote = data["chart"]["result"][0]["meta"]["currency"] + offset = data["chart"]["result"][0]["meta"]["gmtoffset"] timestamps = data["chart"]["result"][0]["timestamp"] adjclose_data = data["chart"]["result"][0]["indicators"]["adjclose"][0] @@ -79,19 +80,25 @@ class Yahoo(BaseSource): amounts = {**adjclose_data, **rest_data} prices = [ - Price(ts, amount) + Price(date, amount) for i in range(len(timestamps)) - if (ts := datetime.fromtimestamp(timestamps[i]).strftime("%Y-%m-%d")) - <= series.end + if (date := self._date_from_ts(timestamps[i], offset)) <= series.end if (amount := self._amount(amounts, series.type, i)) is not None ] return dataclasses.replace(series, quote=quote, prices=prices) + def _date_from_ts(self, ts, offset) -> str: + return ( + datetime.fromtimestamp(ts - offset) + .replace(tzinfo=timezone.utc) + .strftime("%Y-%m-%d") + ) + def _amount(self, amounts, type, i): if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": return sum([Decimal(amounts["high"][i]), Decimal(amounts["low"][i])]) / 2 - elif amounts[type] != "null": + elif amounts[type] != "null" and amounts[type][i] is not None: return Decimal(amounts[type][i]) else: return None diff --git a/tests/pricehist/sources/test_yahoo.py b/tests/pricehist/sources/test_yahoo.py index 7e8d055..d490d86 100644 --- a/tests/pricehist/sources/test_yahoo.py +++ b/tests/pricehist/sources/test_yahoo.py @@ -54,6 +54,13 @@ def long_ok(requests_mock): yield requests_mock +@pytest.fixture +def with_null_ok(requests_mock): + json = (Path(os.path.splitext(__file__)[0]) / "inrx-with-null.json").read_text() + requests_mock.add(responses.GET, url("INR=X"), body=json, status=200) + yield requests_mock + + def test_normalizesymbol(src): assert src.normalizesymbol("tsla") == "TSLA" @@ -157,6 +164,13 @@ def test_fetch_from_before_start(src, type, long_ok): assert len(series.prices) > 9 +def test_fetch_skips_dates_with_nulls(src, type, with_null_ok): + series = src.fetch(Series("INR=X", "", type, "2017-07-10", "2017-07-12")) + assert series.prices[0] == Price("2017-07-10", Decimal("64.61170196533203125")) + assert series.prices[1] == Price("2017-07-12", Decimal("64.52559661865234375")) + assert len(series.prices) == 2 + + def test_fetch_to_future(src, type, recent_ok): series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2100-01-08")) assert len(series.prices) > 0 From 77b2776e55d0c9a6e4265bff94665f1838cba7c0 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 12:59:39 +0200 Subject: [PATCH 143/149] yahoo: More graceful handling of responses with meta but no timestamps. --- src/pricehist/sources/yahoo.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 65857bb..169bf71 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -146,11 +146,10 @@ class Yahoo(BaseSource): raise exceptions.InvalidPair( series.base, series.quote, self, "Symbol not found." ) - if code == 400 and "Data doesn't exist" in text: + elif code == 400 and "Data doesn't exist" in text: raise exceptions.BadResponse( "No data for the given interval. Try requesting a larger interval." ) - elif code == 404 and "Timestamp data missing" in text: raise exceptions.BadResponse( "Data missing. The given interval may be for a gap in the data " @@ -169,4 +168,10 @@ class Yahoo(BaseSource): "The data couldn't be parsed. " ) from e + if "timestamp" not in data["chart"]["result"][0]: + raise exceptions.BadResponse( + "No data for the given interval. " + "There may be a problem with the symbol or the interval." + ) + return data From 1164724ffb8cb2b614af06d2a1d4634cf0a2c886 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 13:01:11 +0200 Subject: [PATCH 144/149] Version 1.4.11. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d5fc989..98bb405 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.10" +version = "1.4.11" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index 738cf25..e42b3cf 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.10" +__version__ = "1.4.11" From c78154df3a265da9c26a149d78bff1989fe046fd Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 13:07:50 +0200 Subject: [PATCH 145/149] Add missing file. --- .../sources/test_yahoo/inrx-with-null.json | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tests/pricehist/sources/test_yahoo/inrx-with-null.json diff --git a/tests/pricehist/sources/test_yahoo/inrx-with-null.json b/tests/pricehist/sources/test_yahoo/inrx-with-null.json new file mode 100644 index 0000000..5ae762e --- /dev/null +++ b/tests/pricehist/sources/test_yahoo/inrx-with-null.json @@ -0,0 +1,119 @@ +{ + "chart": { + "result": [ + { + "meta": { + "currency": "INR", + "symbol": "INR=X", + "exchangeName": "CCY", + "fullExchangeName": "CCY", + "instrumentType": "CURRENCY", + "firstTradeDate": 1070236800, + "regularMarketTime": 1726284616, + "hasPrePostMarketData": false, + "gmtoffset": 3600, + "timezone": "BST", + "exchangeTimezoneName": "Europe/London", + "regularMarketPrice": 83.89, + "fiftyTwoWeekHigh": 83.89, + "fiftyTwoWeekLow": 83.89, + "regularMarketDayHigh": 83.89, + "regularMarketDayLow": 83.89, + "regularMarketVolume": 0, + "longName": "USD/INR", + "shortName": "USD/INR", + "chartPreviousClose": 64.6117, + "priceHint": 4, + "currentTradingPeriod": { + "pre": { + "timezone": "BST", + "start": 1726182000, + "end": 1726182000, + "gmtoffset": 3600 + }, + "regular": { + "timezone": "BST", + "start": 1726182000, + "end": 1726268340, + "gmtoffset": 3600 + }, + "post": { + "timezone": "BST", + "start": 1726268340, + "end": 1726268340, + "gmtoffset": 3600 + } + }, + "dataGranularity": "1d", + "range": "", + "validRanges": [ + "1d", + "5d", + "1mo", + "3mo", + "6mo", + "1y", + "2y", + "5y", + "10y", + "ytd", + "max" + ] + }, + "timestamp": [ + 1499641200, + 1499727600, + 1499814000, + 1499900400 + ], + "indicators": { + "quote": [ + { + "open": [ + 64.6155014038086, + null, + 64.55549621582031, + 64.46800231933594 + ], + "volume": [ + 0, + null, + 0, + 0 + ], + "low": [ + 64.41000366210938, + null, + 64.3499984741211, + 64.33999633789062 + ], + "close": [ + 64.61170196533203, + null, + 64.52559661865234, + 64.36499786376953 + ], + "high": [ + 64.6155014038086, + null, + 64.56999969482422, + 64.48419952392578 + ] + } + ], + "adjclose": [ + { + "adjclose": [ + 64.61170196533203, + null, + 64.52559661865234, + 64.36499786376953 + ] + } + ] + } + } + ], + "error": null + } +} From dffe6f8e89678751478382b8549b520f43296e9e Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 13:15:18 +0200 Subject: [PATCH 146/149] Timezone handling tweak. --- src/pricehist/sources/yahoo.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 169bf71..81dd0f7 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -89,11 +89,7 @@ class Yahoo(BaseSource): return dataclasses.replace(series, quote=quote, prices=prices) def _date_from_ts(self, ts, offset) -> str: - return ( - datetime.fromtimestamp(ts - offset) - .replace(tzinfo=timezone.utc) - .strftime("%Y-%m-%d") - ) + return datetime.fromtimestamp(ts - offset).strftime("%Y-%m-%d") def _amount(self, amounts, type, i): if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": From 53f39a26ef9e28565826456e65d119b6f4209d6a Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 11:35:56 +0000 Subject: [PATCH 147/149] More time correction. --- src/pricehist/sources/yahoo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pricehist/sources/yahoo.py b/src/pricehist/sources/yahoo.py index 81dd0f7..25d92fe 100644 --- a/src/pricehist/sources/yahoo.py +++ b/src/pricehist/sources/yahoo.py @@ -82,14 +82,14 @@ class Yahoo(BaseSource): prices = [ Price(date, amount) for i in range(len(timestamps)) - if (date := self._date_from_ts(timestamps[i], offset)) <= series.end + if (date := self._ts_to_date(timestamps[i] + offset)) <= series.end if (amount := self._amount(amounts, series.type, i)) is not None ] return dataclasses.replace(series, quote=quote, prices=prices) - def _date_from_ts(self, ts, offset) -> str: - return datetime.fromtimestamp(ts - offset).strftime("%Y-%m-%d") + def _ts_to_date(self, ts) -> str: + return datetime.fromtimestamp(ts, tz=timezone.utc).date().isoformat() def _amount(self, amounts, type, i): if type == "mid" and amounts["high"] != "null" and amounts["low"] != "null": From ab507b189cdb30f4df7ead380bc0151740f249b3 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Sun, 15 Sep 2024 12:16:36 +0000 Subject: [PATCH 148/149] Update live test. --- tests/live.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/live.sh b/tests/live.sh index cb2dbeb..157cdad 100755 --- a/tests/live.sh +++ b/tests/live.sh @@ -75,10 +75,10 @@ name="Alpha Vantage physical currency" cmd="pricehist fetch alphavantage AUD/EUR -s 2021-01-11 -e 2021-01-14" read -r -d '' expected < Date: Sun, 15 Sep 2024 12:17:10 +0000 Subject: [PATCH 149/149] Version 1.4.12. --- pyproject.toml | 2 +- src/pricehist/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98bb405..5d555aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pricehist" -version = "1.4.11" +version = "1.4.12" description = "Fetch and format historical price data" authors = ["Chris Berkhout "] license = "MIT" diff --git a/src/pricehist/__init__.py b/src/pricehist/__init__.py index e42b3cf..2736991 100644 --- a/src/pricehist/__init__.py +++ b/src/pricehist/__init__.py @@ -1 +1 @@ -__version__ = "1.4.11" +__version__ = "1.4.12"