CoinDesk error handling and tests.

src/pricehist/sources/coindesk.py

@@ -1,12 +1,11 @@
 import dataclasses
 import json
-import logging
-import sys
 from decimal import Decimal
 
 import requests
 
 from pricehist.price import Price
+from pricehist import exceptions
 
 from .basesource import BaseSource
 
@@ -38,26 +37,45 @@ class CoinDesk(BaseSource):
 
     def symbols(self):
         url = "https://api.coindesk.com/v1/bpi/supported-currencies.json"
-        response = self.log_curl(requests.get(url))
-        data = json.loads(response.content)
-        relevant = [i for i in data if i["currency"] not in ["XBT", "BTC"]]
-        return [
-            (f"BTC/{i['currency']}", f"Bitcoin against {i['country']}")
-            for i in sorted(relevant, key=lambda i: i["currency"])
-        ]
+
+        try:
+            response = self.log_curl(requests.get(url))
+        except Exception as e:
+            raise exceptions.RequestError(str(e)) from e
+
+        try:
+            response.raise_for_status()
+        except Exception as e:
+            raise exceptions.BadResponse(str(e)) from e
+
+        try:
+            data = json.loads(response.content)
+            relevant = [i for i in data if i["currency"] not in ["BTC", "XBT"]]
+            results = [
+                (f"BTC/{i['currency']}", f"Bitcoin against {i['country']}")
+                for i in sorted(relevant, key=lambda i: i["currency"])
+            ]
+        except Exception as e:
+            raise exceptions.ResponseParsingError(str(e)) from e
+
+        if not results:
+            raise exceptions.ResponseParsingError("Expected data not found")
+        else:
+            return results
 
     def fetch(self, series):
-        if series.base != "BTC" or series.quote == "BTC":
-            # BTC is the only valid base. BTC as the quote will return BTC/USD.
-            logging.critical(
-                f"Invalid pair '{'/'.join([series.base, series.quote])}'. "
-                f"Run 'pricehist source {self.id()} --symbols' to list valid pairs."
-            )
-            sys.exit(1)
+        if series.base != "BTC" or series.quote in ["BTC", "XBT"]:
+            # BTC is the only valid base.
+            # BTC as the quote will return BTC/USD, which we don't want.
+            # XBT as the quote will fail with HTTP status 500.
+            raise exceptions.InvalidPair(series.base, series.quote, self)
+
         data = self._data(series)
+
         prices = []
-        for (d, v) in data["bpi"].items():
+        for (d, v) in data.get("bpi", {}).items():
             prices.append(Price(d, Decimal(str(v))))
+
         return dataclasses.replace(series, prices=prices)
 
     def _data(self, series):
@@ -67,5 +85,37 @@ class CoinDesk(BaseSource):
             "start": series.start,
             "end": series.end,
         }
-        response = self.log_curl(requests.get(url, params=params))
-        return json.loads(response.content)
+
+        try:
+            response = self.log_curl(requests.get(url, params=params))
+        except Exception as e:
+            raise exceptions.RequestError(str(e)) from e
+
+        code = response.status_code
+        text = response.text
+        if code == 404 and "currency was not found" in text:
+            raise exceptions.InvalidPair(series.base, series.quote, self)
+        elif code == 404 and "only covers data from" in text:
+            raise exceptions.BadResponse(text)
+        elif code == 404 and "end date is before" in text and series.end < series.start:
+            raise exceptions.BadResponse("End date is before start date.")
+        elif code == 404 and "end date is before" in text:
+            raise exceptions.BadResponse("The start date must be in the past.")
+        elif code == 500 and "No results returned from database" in text:
+            raise exceptions.BadResponse(
+                "No results returned from database. This can happen when data "
+                "for a valid quote currency (e.g. CUP) doesn't go all the way "
+                "back to the start date, or potentially for other reasons."
+            )
+        else:
+            try:
+                response.raise_for_status()
+            except Exception as e:
+                raise exceptions.BadResponse(str(e)) from e
+
+        try:
+            result = json.loads(response.content)
+        except Exception as e:
+            raise exceptions.ResponseParsingError(str(e)) from e
+
+        return result

tests/pricehist/sources/test_coindesk.py

@@ -0,0 +1,326 @@
+import logging
+import os
+from datetime import datetime
+from decimal import Decimal
+from pathlib import Path
+
+import pytest
+import requests
+import responses
+
+from pricehist import exceptions
+from pricehist.price import Price
+from pricehist.series import Series
+from pricehist.sources.coindesk import CoinDesk
+
+
+@pytest.fixture
+def src():
+    return CoinDesk()
+
+
+@pytest.fixture
+def type(src):
+    return src.types()[0]
+
+
+@pytest.fixture
+def requests_mock():
+    with responses.RequestsMock() as mock:
+        yield mock
+
+
+@pytest.fixture
+def currencies_url():
+    return "https://api.coindesk.com/v1/bpi/supported-currencies.json"
+
+
+@pytest.fixture
+def currencies_json():
+    dir = Path(os.path.splitext(__file__)[0])
+    return (dir / "supported-currencies-partial.json").read_text()
+
+
+@pytest.fixture
+def currencies_response_ok(requests_mock, currencies_url, currencies_json):
+    requests_mock.add(
+        responses.GET,
+        currencies_url,
+        body=currencies_json,
+        status=200,
+    )
+    yield requests_mock
+
+
+@pytest.fixture
+def recent_json():
+    dir = Path(os.path.splitext(__file__)[0])
+    return (dir / "recent.json").read_text()
+
+
+@pytest.fixture
+def recent_response_ok(requests_mock, recent_json):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        body=recent_json,
+        status=200,
+    )
+    yield requests_mock
+
+
+@pytest.fixture
+def all_json():
+    dir = Path(os.path.splitext(__file__)[0])
+    return (dir / "all-partial.json").read_text()
+
+
+@pytest.fixture
+def all_response_ok(requests_mock, all_json):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        body=all_json,
+        status=200,
+    )
+    yield requests_mock
+
+
+@pytest.fixture
+def not_found_response(requests_mock):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        status=404,
+        body="Sorry, that currency was not found",
+    )
+
+
+def test_normalizesymbol(src):
+    assert src.normalizesymbol("btc") == "BTC"
+    assert src.normalizesymbol("usd") == "USD"
+
+
+def test_metadata(src):
+    assert isinstance(src.id(), str)
+    assert len(src.id()) > 0
+
+    assert isinstance(src.name(), str)
+    assert len(src.name()) > 0
+
+    assert isinstance(src.description(), str)
+    assert len(src.description()) > 0
+
+    assert isinstance(src.source_url(), str)
+    assert src.source_url().startswith("http")
+
+    assert datetime.strptime(src.start(), "%Y-%m-%d")
+
+    assert isinstance(src.types(), list)
+    assert len(src.types()) > 0
+    assert isinstance(src.types()[0], str)
+    assert len(src.types()[0]) > 0
+
+    assert isinstance(src.notes(), str)
+
+
+def test_symbols(src, currencies_response_ok):
+    syms = src.symbols()
+    assert ("BTC/AUD", "Bitcoin against Australian Dollar") in syms
+    assert len(syms) > 3
+
+
+def test_symbols_requests_logged(src, currencies_response_ok, caplog):
+    with caplog.at_level(logging.DEBUG):
+        src.symbols()
+    assert any(
+        ["DEBUG" == r.levelname and " curl " in r.message for r in caplog.records]
+    )
+
+
+def test_symbols_not_found(src, requests_mock, currencies_url):
+    requests_mock.add(
+        responses.GET,
+        currencies_url,
+        body="[]",
+        status=200,
+    )
+    with pytest.raises(exceptions.ResponseParsingError) as e:
+        src.symbols()
+    assert "data not found" in str(e.value)
+
+
+def test_symbols_network_issue(src, requests_mock, currencies_url):
+    requests_mock.add(
+        responses.GET,
+        currencies_url,
+        body=requests.exceptions.ConnectionError("Network issue"),
+    )
+    with pytest.raises(exceptions.RequestError) as e:
+        src.symbols()
+    assert "Network issue" in str(e.value)
+
+
+def test_symbols_bad_status(src, requests_mock, currencies_url):
+    requests_mock.add(
+        responses.GET,
+        currencies_url,
+        status=500,
+    )
+    with pytest.raises(exceptions.BadResponse) as e:
+        src.symbols()
+    assert "Server Error" in str(e.value)
+
+
+def test_symbols_parsing_error(src, requests_mock, currencies_url):
+    requests_mock.add(
+        responses.GET,
+        currencies_url,
+        body="NOT JSON",
+    )
+    with pytest.raises(exceptions.ResponseParsingError) as e:
+        src.symbols()
+    assert "while parsing data" in str(e.value)
+
+
+def test_fetch_known_pair(src, type, recent_response_ok):
+    series = src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07"))
+    req = recent_response_ok.calls[0].request
+    assert req.params["currency"] == "AUD"
+    assert req.params["start"] == "2021-01-01"
+    assert req.params["end"] == "2021-01-07"
+    assert series.prices[0] == Price("2021-01-01", Decimal("38204.8987"))
+    assert series.prices[-1] == Price("2021-01-07", Decimal("50862.227"))
+    assert len(series.prices) == 7
+
+
+def test_fetch_requests_logged(src, type, recent_response_ok, caplog):
+    with caplog.at_level(logging.DEBUG):
+        src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07"))
+    assert any(
+        ["DEBUG" == r.levelname and " curl " in r.message for r in caplog.records]
+    )
+
+
+def test_fetch_long_hist_from_start(src, type, all_response_ok):
+    series = src.fetch(Series("BTC", "AUD", type, src.start(), "2021-01-07"))
+    assert series.prices[0] == Price("2010-07-18", Decimal("0.0984"))
+    assert series.prices[-1] == Price("2021-01-07", Decimal("50862.227"))
+    assert len(series.prices) > 13
+
+
+def test_fetch_from_before_start(src, type, requests_mock):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        status=404,
+        body="Sorry, the CoinDesk BPI only covers data from 2010-07-17 onwards.",
+    )
+    with pytest.raises(exceptions.BadResponse) as e:
+        src.fetch(Series("BTC", "AUD", type, "2010-01-01", "2010-07-24"))
+    assert "only covers data from" in str(e.value)
+
+
+def test_fetch_to_future(src, type, all_response_ok):
+    series = src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2100-01-01"))
+    assert len(series.prices) > 0
+
+
+def test_wrong_dates_order(src, type, requests_mock):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        status=404,
+        body="Sorry, but your specified end date is before your start date.",
+    )
+    with pytest.raises(exceptions.BadResponse) as e:
+        src.fetch(Series("BTC", "AUD", type, "2021-01-07", "2021-01-01"))
+    assert "End date is before start date." in str(e.value)
+
+
+def test_fetch_in_future(src, type, requests_mock):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        status=404,
+        body="Sorry, but your specified end date is before your start date.",
+    )
+    with pytest.raises(exceptions.BadResponse) as e:
+        src.fetch(Series("BTC", "AUD", type, "2030-01-01", "2030-01-07"))
+    assert "start date must be in the past" in str(e.value)
+
+
+def test_fetch_empty(src, type, requests_mock):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        body="{}",
+    )
+    series = src.fetch(Series("BTC", "AUD", type, "2010-07-17", "2010-07-17"))
+    assert len(series.prices) == 0
+
+
+def test_fetch_known_pair_no_data(src, type, requests_mock):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        status=500,
+        body="No results returned from database",
+    )
+    with pytest.raises(exceptions.BadResponse) as e:
+        src.fetch(Series("BTC", "CUP", type, "2010-07-17", "2010-07-23"))
+    assert "No results returned from database" in str(e.value)
+
+
+def test_fetch_non_btc_base(src, type):
+    with pytest.raises(exceptions.InvalidPair):
+        src.fetch(Series("USD", "AUD", type, "2021-01-01", "2021-01-07"))
+
+
+def test_fetch_unknown_quote(src, type, not_found_response):
+    with pytest.raises(exceptions.InvalidPair):
+        src.fetch(Series("BTC", "XZY", type, "2021-01-01", "2021-01-07"))
+
+
+def test_fetch_no_quote(src, type, not_found_response):
+    with pytest.raises(exceptions.InvalidPair):
+        src.fetch(Series("BTC", "", type, "2021-01-01", "2021-01-07"))
+
+
+def test_fetch_unknown_pair(src, type):
+    with pytest.raises(exceptions.InvalidPair):
+        src.fetch(Series("ABC", "XZY", type, "2021-01-01", "2021-01-07"))
+
+
+def test_fetch_network_issue(src, type, requests_mock):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        body=requests.exceptions.ConnectionError("Network issue"),
+    )
+    with pytest.raises(exceptions.RequestError) as e:
+        src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07"))
+    assert "Network issue" in str(e.value)
+
+
+def test_fetch_bad_status(src, type, requests_mock):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        status=500,
+        body="Some other reason",
+    )
+    with pytest.raises(exceptions.BadResponse) as e:
+        src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07"))
+    assert "Internal Server Error" in str(e.value)
+
+
+def test_fetch_parsing_error(src, type, requests_mock):
+    requests_mock.add(
+        responses.GET,
+        "https://api.coindesk.com/v1/bpi/historical/close.json",
+        body="NOT JSON",
+    )
+    with pytest.raises(exceptions.ResponseParsingError) as e:
+        src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07"))
+    assert "while parsing data" in str(e.value)

tests/pricehist/sources/test_coindesk/all-partial.json

@@ -0,0 +1,23 @@
+{
+  "bpi": {
+    "2010-07-18": 0.0984,
+    "2010-07-19": 0.093,
+    "2010-07-20": 0.0851,
+    "2010-07-21": 0.0898,
+    "2010-07-22": 0.0567,
+    "2010-07-23": 0.07,
+    "2010-07-24": 0.0609,
+    "2021-01-01": 38204.8987,
+    "2021-01-02": 41853.1942,
+    "2021-01-03": 42925.6366,
+    "2021-01-04": 41751.2249,
+    "2021-01-05": 43890.3534,
+    "2021-01-06": 47190.09,
+    "2021-01-07": 50862.227
+  },
+  "disclaimer": "This data was produced from the CoinDesk Bitcoin Price Index. BPI value data returned as AUD.",
+  "time": {
+    "updated": "Jan 8, 2021 00:03:00 UTC",
+    "updatedISO": "2021-01-08T00:03:00+00:00"
+  }
+}

tests/pricehist/sources/test_coindesk/recent.json

@@ -0,0 +1,16 @@
+{
+  "bpi": {
+    "2021-01-01": 38204.8987,
+    "2021-01-02": 41853.1942,
+    "2021-01-03": 42925.6366,
+    "2021-01-04": 41751.2249,
+    "2021-01-05": 43890.3534,
+    "2021-01-06": 47190.09,
+    "2021-01-07": 50862.227
+  },
+  "disclaimer": "This data was produced from the CoinDesk Bitcoin Price Index. BPI value data returned as AUD.",
+  "time": {
+    "updated": "Jan 8, 2021 00:03:00 UTC",
+    "updatedISO": "2021-01-08T00:03:00+00:00"
+  }
+}

tests/pricehist/sources/test_coindesk/supported-currencies-partial.json

@@ -0,0 +1,26 @@
+[
+  {
+    "currency": "AUD",
+    "country": "Australian Dollar"
+  },
+  {
+    "currency": "BTC",
+    "country": "Bitcoin"
+  },
+  {
+    "currency": "CUP",
+    "country": "Cuban Peso"
+  },
+  {
+    "currency": "EUR",
+    "country": "Euro"
+  },
+  {
+    "currency": "USD",
+    "country": "United States Dollar"
+  },
+  {
+    "currency": "XBT",
+    "country": "Bitcoin"
+  }
+]