Support Coinbase Pro.

This commit is contained in:
Chris Berkhout 2021-08-23 18:40:35 +02:00
parent ca63a435bd
commit 7b53204bcf
11 changed files with 821 additions and 1 deletions

View file

@ -22,6 +22,7 @@ pipx install pricehist
## Sources
- **`alphavantage`**: [Alpha Vantage](https://www.alphavantage.co/)
- **`coinbasepro`**: [Coinbase Pro](https://pro.coinbase.com/)
- **`coindesk`**: [CoinDesk Bitcoin Price Index](https://www.coindesk.com/coindesk-api)
- **`coinmarketcap`**: [CoinMarketCap](https://coinmarketcap.com/)
- **`ecb`**: [European Central Bank Euro foreign exchange reference rates](https://www.ecb.europa.eu/stats/exchange/eurofxref/html/index.en.html)

View file

@ -0,0 +1,4 @@
from pricehist import beanprice
from pricehist.sources.coinbasepro import CoinbasePro
Source = beanprice.source(CoinbasePro())

View file

@ -1,4 +1,5 @@
from .alphavantage import AlphaVantage
from .coinbasepro import CoinbasePro
from .coindesk import CoinDesk
from .coinmarketcap import CoinMarketCap
from .ecb import ECB
@ -6,7 +7,14 @@ from .yahoo import Yahoo
by_id = {
source.id(): source
for source in [AlphaVantage(), CoinDesk(), CoinMarketCap(), ECB(), Yahoo()]
for source in [
AlphaVantage(),
CoinbasePro(),
CoinDesk(),
CoinMarketCap(),
ECB(),
Yahoo(),
]
}

View file

@ -0,0 +1,164 @@
import dataclasses
import json
from datetime import datetime, timedelta, timezone
from decimal import Decimal
import requests
from pricehist import exceptions
from pricehist.price import Price
from .basesource import BaseSource
class CoinbasePro(BaseSource):
def id(self):
return "coinbasepro"
def name(self):
return "Coinbase Pro"
def description(self):
return "The Coinbase Pro feed API provides market data to the public."
def source_url(self):
return "https://docs.pro.coinbase.com/"
def start(self):
return "2015-07-20"
def types(self):
return ["mid", "open", "high", "low", "close"]
def notes(self):
return (
"This source uses Coinbase's Pro APIs, not the v2 API.\n"
"No key or other authentication is requried because it only uses "
"the feed APIs that provide market data and are public."
)
def symbols(self):
products_url = "https://api.pro.coinbase.com/products"
currencies_url = "https://api.pro.coinbase.com/currencies"
try:
products_response = self.log_curl(requests.get(products_url))
currencies_response = self.log_curl(requests.get(currencies_url))
except Exception as e:
raise exceptions.RequestError(str(e)) from e
try:
products_response.raise_for_status()
currencies_response.raise_for_status()
except Exception as e:
raise exceptions.BadResponse(str(e)) from e
try:
products_data = json.loads(products_response.content)
currencies_data = json.loads(currencies_response.content)
currencies = {c["id"]: c for c in currencies_data}
results = []
for i in sorted(products_data, key=lambda i: i["id"]):
base = i["base_currency"]
quote = i["quote_currency"]
base_name = currencies[base]["name"] if currencies[base] else base
quote_name = currencies[quote]["name"] if currencies[quote] else quote
results.append((f"{base}/{quote}", f"{base_name} against {quote_name}"))
except Exception as e:
raise exceptions.ResponseParsingError(str(e)) from e
if not results:
raise exceptions.ResponseParsingError("Expected data not found")
else:
return results
def fetch(self, series):
data = []
for seg_start, seg_end in self._segments(series.start, series.end):
data.extend(self._data(series.base, series.quote, seg_start, seg_end))
prices = []
for item in data:
prices.append(Price(item["date"], self._amount(item, series.type)))
return dataclasses.replace(series, prices=prices)
def _segments(self, start, end, length=290):
start = datetime.fromisoformat(start).date()
end = max(datetime.fromisoformat(end).date(), start)
segments = []
seg_start = start
while seg_start <= end:
seg_end = min(seg_start + timedelta(days=length - 1), end)
segments.append((seg_start.isoformat(), seg_end.isoformat()))
seg_start = seg_end + timedelta(days=1)
return segments
def _data(self, base, quote, start, end):
product = f"{base}-{quote}"
url = f"https://api.pro.coinbase.com/products/{product}/candles"
params = {
"start": start,
"end": end,
"granularity": "86400",
}
try:
response = self.log_curl(requests.get(url, params=params))
except Exception as e:
raise exceptions.RequestError(str(e)) from e
code = response.status_code
text = response.text
if code == 400 and "aggregations requested exceeds" in text:
raise exceptions.BadResponse("Too many data points requested.")
elif code == 400 and "start must be before end" in text:
raise exceptions.BadResponse("The end can't preceed the start.")
elif code == 400 and "is too old" in text:
raise exceptions.BadResponse("The requested interval is too early.")
elif code == 404 and "NotFound" in text:
raise exceptions.InvalidPair(base, quote, self)
elif code == 429:
raise exceptions.RateLimit(
"The rate limit has been exceeded. For more information see "
"https://docs.pro.coinbase.com/#rate-limit."
)
else:
try:
response.raise_for_status()
except Exception as e:
raise exceptions.BadResponse(str(e)) from e
try:
result = reversed(
[
{
"date": self._ts_to_date(candle[0]),
"low": candle[1],
"high": candle[2],
"open": candle[3],
"close": candle[4],
}
for candle in json.loads(response.content)
if start <= self._ts_to_date(candle[0]) <= end
]
)
except Exception as e:
raise exceptions.ResponseParsingError(str(e)) from e
return result
def _ts_to_date(self, ts):
return datetime.fromtimestamp(ts, tz=timezone.utc).date().isoformat()
def _amount(self, item, type):
if type in ["mid"]:
high = Decimal(str(item["high"]))
low = Decimal(str(item["low"]))
return sum([high, low]) / 2
else:
return Decimal(str(item[type]))

View file

@ -80,6 +80,18 @@ date,base,quote,amount,source,type
END
run_test "$name" "$cmd" "$expected"
name="Coinbase Pro"
cmd="pricehist fetch coinbasepro BTC/EUR -s 2021-01-04 -e 2021-01-08"
read -r -d '' expected <<END
date,base,quote,amount,source,type
2021-01-04,BTC,EUR,24127,coinbasepro,mid
2021-01-05,BTC,EUR,26201.31,coinbasepro,mid
2021-01-06,BTC,EUR,28527.005,coinbasepro,mid
2021-01-07,BTC,EUR,31208.49,coinbasepro,mid
2021-01-08,BTC,EUR,32019,coinbasepro,mid
END
run_test "$name" "$cmd" "$expected"
name="CoinDesk Bitcoin Price Index"
cmd="pricehist fetch coindesk BTC/EUR -s 2021-01-04 -e 2021-01-08"
read -r -d '' expected <<END

View file

@ -0,0 +1,334 @@
import logging
import os
import re
from datetime import datetime
from decimal import Decimal
from pathlib import Path
import pytest
import requests
import responses
from pricehist import exceptions
from pricehist.price import Price
from pricehist.series import Series
from pricehist.sources.coinbasepro import CoinbasePro
@pytest.fixture
def src():
return CoinbasePro()
@pytest.fixture
def type(src):
return src.types()[0]
@pytest.fixture
def requests_mock():
with responses.RequestsMock() as mock:
yield mock
@pytest.fixture
def products_url():
return "https://api.pro.coinbase.com/products"
@pytest.fixture
def currencies_url():
return "https://api.pro.coinbase.com/currencies"
def product_url(base, quote):
return f"https://api.pro.coinbase.com/products/{base}-{quote}/candles"
@pytest.fixture
def products_json():
return (Path(os.path.splitext(__file__)[0]) / "products-partial.json").read_text()
@pytest.fixture
def currencies_json():
return (Path(os.path.splitext(__file__)[0]) / "currencies-partial.json").read_text()
@pytest.fixture
def products_response_ok(requests_mock, products_url, products_json):
requests_mock.add(responses.GET, products_url, body=products_json, status=200)
yield requests_mock
@pytest.fixture
def currencies_response_ok(requests_mock, currencies_url, currencies_json):
requests_mock.add(responses.GET, currencies_url, body=currencies_json, status=200)
yield requests_mock
@pytest.fixture
def recent_response_ok(requests_mock):
json = (Path(os.path.splitext(__file__)[0]) / "recent.json").read_text()
requests_mock.add(responses.GET, product_url("BTC", "EUR"), body=json, status=200)
yield requests_mock
@pytest.fixture
def multi_response_ok(requests_mock):
url1 = re.compile(
r"https://api\.pro\.coinbase\.com/products/BTC-EUR/candles\?start=2020-01-01.*"
)
url2 = re.compile(
r"https://api\.pro\.coinbase\.com/products/BTC-EUR/candles\?start=2020-10-17.*"
)
json1 = (
Path(os.path.splitext(__file__)[0]) / "2020-01-01--2020-10-16.json"
).read_text()
json2 = (
Path(os.path.splitext(__file__)[0]) / "2020-10-17--2021-01-07.json"
).read_text()
requests_mock.add(responses.GET, url1, body=json1, status=200)
requests_mock.add(responses.GET, url2, body=json2, status=200)
yield requests_mock
@pytest.fixture
def response_empty(requests_mock):
requests_mock.add(
responses.GET,
product_url("BTC", "EUR"),
status=200,
body="[]",
)
def test_normalizesymbol(src):
assert src.normalizesymbol("btc") == "BTC"
assert src.normalizesymbol("usd") == "USD"
def test_metadata(src):
assert isinstance(src.id(), str)
assert len(src.id()) > 0
assert isinstance(src.name(), str)
assert len(src.name()) > 0
assert isinstance(src.description(), str)
assert len(src.description()) > 0
assert isinstance(src.source_url(), str)
assert src.source_url().startswith("http")
assert datetime.strptime(src.start(), "%Y-%m-%d")
assert isinstance(src.types(), list)
assert len(src.types()) > 0
assert isinstance(src.types()[0], str)
assert len(src.types()[0]) > 0
assert isinstance(src.notes(), str)
def test_symbols(src, products_response_ok, currencies_response_ok):
syms = src.symbols()
assert ("BTC/EUR", "Bitcoin against Euro") in syms
assert len(syms) > 2
def test_symbols_requests_logged(
src, products_response_ok, currencies_response_ok, caplog
):
with caplog.at_level(logging.DEBUG):
src.symbols()
matching = filter(
lambda r: "DEBUG" == r.levelname and "curl " in r.message,
caplog.records,
)
assert len(list(matching)) == 2
def test_symbols_not_found(src, requests_mock, products_url, currencies_response_ok):
requests_mock.add(responses.GET, products_url, body="[]", status=200)
with pytest.raises(exceptions.ResponseParsingError) as e:
src.symbols()
assert "data not found" in str(e.value)
def test_symbols_network_issue(
src, requests_mock, products_response_ok, currencies_url
):
requests_mock.add(
responses.GET,
currencies_url,
body=requests.exceptions.ConnectionError("Network issue"),
)
with pytest.raises(exceptions.RequestError) as e:
src.symbols()
assert "Network issue" in str(e.value)
def test_symbols_bad_status(src, requests_mock, products_url, currencies_response_ok):
requests_mock.add(responses.GET, products_url, status=500)
with pytest.raises(exceptions.BadResponse) as e:
src.symbols()
assert "Server Error" in str(e.value)
def test_symbols_parsing_error(
src, requests_mock, products_response_ok, currencies_url
):
requests_mock.add(responses.GET, currencies_url, body="NOT JSON")
with pytest.raises(exceptions.ResponseParsingError) as e:
src.symbols()
assert "while parsing data" in str(e.value)
def test_fetch_known_pair(src, type, recent_response_ok):
series = src.fetch(Series("BTC", "EUR", type, "2021-01-01", "2021-01-07"))
req = recent_response_ok.calls[0].request
assert req.params["granularity"] == "86400"
assert req.params["start"] == "2021-01-01"
assert req.params["end"] == "2021-01-07"
assert series.prices[0] == Price("2021-01-01", Decimal("23881.35"))
assert series.prices[-1] == Price("2021-01-07", Decimal("31208.49"))
assert len(series.prices) == 7
def test_fetch_types_all_available(src, recent_response_ok):
mid = src.fetch(Series("BTC", "EUR", "mid", "2021-01-01", "2021-01-07"))
opn = src.fetch(Series("BTC", "EUR", "open", "2021-01-01", "2021-01-07"))
hgh = src.fetch(Series("BTC", "EUR", "high", "2021-01-01", "2021-01-07"))
low = src.fetch(Series("BTC", "EUR", "low", "2021-01-01", "2021-01-07"))
cls = src.fetch(Series("BTC", "EUR", "close", "2021-01-01", "2021-01-07"))
assert mid.prices[0].amount == Decimal("23881.35")
assert opn.prices[0].amount == Decimal("23706.73")
assert hgh.prices[0].amount == Decimal("24250")
assert low.prices[0].amount == Decimal("23512.7")
assert cls.prices[0].amount == Decimal("24070.97")
def test_fetch_type_mid_is_mean_of_low_and_high(src, recent_response_ok):
mid = src.fetch(Series("BTC", "EUR", "mid", "2021-01-01", "2021-01-07")).prices
low = src.fetch(Series("BTC", "EUR", "low", "2021-01-01", "2021-01-07")).prices
hgh = src.fetch(Series("BTC", "EUR", "high", "2021-01-01", "2021-01-07")).prices
assert all(
[
mid[i].amount == (sum([low[i].amount, hgh[i].amount]) / 2)
for i in range(0, 7)
]
)
def test_fetch_requests_logged(src, type, recent_response_ok, caplog):
with caplog.at_level(logging.DEBUG):
src.fetch(Series("BTC", "EUR", type, "2021-01-01", "2021-01-07"))
assert any(
["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records]
)
def test_fetch_long_hist_multi_segment(src, type, multi_response_ok):
series = src.fetch(Series("BTC", "EUR", type, "2020-01-01", "2021-01-07"))
assert series.prices[0] == Price("2020-01-01", Decimal("6430.175"))
assert series.prices[-1] == Price("2021-01-07", Decimal("31208.49"))
assert len(series.prices) > 3
def test_fetch_from_before_start(src, type, requests_mock):
body = '{"message":"End is too old"}'
requests_mock.add(responses.GET, product_url("BTC", "EUR"), status=400, body=body)
with pytest.raises(exceptions.BadResponse) as e:
src.fetch(Series("BTC", "EUR", type, "1960-01-01", "1960-01-07"))
assert "too early" in str(e.value)
def test_fetch_in_future(src, type, response_empty):
series = src.fetch(Series("BTC", "EUR", type, "2100-01-01", "2100-01-07"))
assert len(series.prices) == 0
def test_fetch_wrong_dates_order_alledged(src, type, requests_mock):
# Is actually prevented in argument parsing and inside the source.
body = '{"message":"start must be before end"}'
requests_mock.add(responses.GET, product_url("BTC", "EUR"), status=400, body=body)
with pytest.raises(exceptions.BadResponse) as e:
src.fetch(Series("BTC", "EUR", type, "2021-01-07", "2021-01-01"))
assert "end can't preceed" in str(e.value)
def test_fetch_too_many_data_points_alledged(src, type, requests_mock):
# Should only happen if limit is reduced or calculated segments lengthened
body = "aggregations requested exceeds"
requests_mock.add(responses.GET, product_url("BTC", "EUR"), status=400, body=body)
with pytest.raises(exceptions.BadResponse) as e:
src.fetch(Series("BTC", "EUR", type, "2021-01-07", "2021-01-01"))
assert "Too many data points" in str(e.value)
def test_fetch_rate_limit(src, type, requests_mock):
body = "Too many requests"
requests_mock.add(responses.GET, product_url("BTC", "EUR"), status=429, body=body)
with pytest.raises(exceptions.RateLimit) as e:
src.fetch(Series("BTC", "EUR", type, "2021-01-07", "2021-01-01"))
assert "rate limit has been exceeded" in str(e.value)
def test_fetch_empty(src, type, response_empty):
series = src.fetch(Series("BTC", "EUR", type, "2000-01-01", "2000-01-07"))
assert len(series.prices) == 0
def test_fetch_unknown_base(src, type, requests_mock):
body = '{"message":"NotFound"}'
requests_mock.add(
responses.GET, product_url("UNKNOWN", "EUR"), status=404, body=body
)
with pytest.raises(exceptions.InvalidPair):
src.fetch(Series("UNKNOWN", "EUR", type, "2021-01-01", "2021-01-07"))
def test_fetch_unknown_quote(src, type, requests_mock):
body = '{"message":"NotFound"}'
requests_mock.add(responses.GET, product_url("BTC", "XZY"), status=404, body=body)
with pytest.raises(exceptions.InvalidPair):
src.fetch(Series("BTC", "XZY", type, "2021-01-01", "2021-01-07"))
def test_fetch_no_quote(src, type, requests_mock):
body = '{"message":"NotFound"}'
requests_mock.add(responses.GET, product_url("BTC", ""), status=404, body=body)
with pytest.raises(exceptions.InvalidPair):
src.fetch(Series("BTC", "", type, "2021-01-01", "2021-01-07"))
def test_fetch_unknown_pair(src, type, requests_mock):
body = '{"message":"NotFound"}'
requests_mock.add(responses.GET, product_url("ABC", "XZY"), status=404, body=body)
with pytest.raises(exceptions.InvalidPair):
src.fetch(Series("ABC", "XZY", type, "2021-01-01", "2021-01-07"))
def test_fetch_network_issue(src, type, requests_mock):
body = requests.exceptions.ConnectionError("Network issue")
requests_mock.add(responses.GET, product_url("BTC", "EUR"), body=body)
with pytest.raises(exceptions.RequestError) as e:
src.fetch(Series("BTC", "EUR", type, "2021-01-01", "2021-01-07"))
assert "Network issue" in str(e.value)
def test_fetch_bad_status(src, type, requests_mock):
requests_mock.add(
responses.GET, product_url("BTC", "EUR"), status=500, body="Some other reason"
)
with pytest.raises(exceptions.BadResponse) as e:
src.fetch(Series("BTC", "EUR", type, "2021-01-01", "2021-01-07"))
assert "Internal Server Error" in str(e.value)
def test_fetch_parsing_error(src, type, requests_mock):
requests_mock.add(responses.GET, product_url("BTC", "EUR"), body="NOT JSON")
with pytest.raises(exceptions.ResponseParsingError) as e:
src.fetch(Series("BTC", "EUR", type, "2021-01-01", "2021-01-07"))
assert "while parsing data" in str(e.value)

View file

@ -0,0 +1,18 @@
[
[
1602806400,
9588,
9860,
9828.84,
9672.41,
1068.08144123
],
[
1577836800,
6388.91,
6471.44,
6400.02,
6410.22,
491.94797816
]
]

View file

@ -0,0 +1,18 @@
[
[
1609977600,
29516.98,
32900,
29818.73,
32120.19,
5957.46980324
],
[
1602892800,
9630.1,
9742.61,
9675.29,
9706.33,
385.03505036
]
]

View file

@ -0,0 +1,141 @@
[
{
"id": "BTC",
"name": "Bitcoin",
"min_size": "0.00000001",
"status": "online",
"message": "",
"max_precision": "0.00000001",
"convertible_to": [],
"details": {
"type": "crypto",
"symbol": "₿",
"network_confirmations": 3,
"sort_order": 20,
"crypto_address_link": "https://live.blockcypher.com/btc/address/{{address}}",
"crypto_transaction_link": "https://live.blockcypher.com/btc/tx/{{txId}}",
"push_payment_methods": [
"crypto"
],
"group_types": [
"btc",
"crypto"
],
"display_name": "",
"processing_time_seconds": 0,
"min_withdrawal_amount": 0.0001,
"max_withdrawal_amount": 2400
}
},
{
"id": "DOGE",
"name": "Dogecoin",
"min_size": "1",
"status": "online",
"message": "",
"max_precision": "0.1",
"convertible_to": [],
"details": {
"type": "crypto",
"symbol": "",
"network_confirmations": 60,
"sort_order": 29,
"crypto_address_link": "https://dogechain.info/address/{{address}}",
"crypto_transaction_link": "",
"push_payment_methods": [
"crypto"
],
"group_types": [],
"display_name": "",
"processing_time_seconds": 0,
"min_withdrawal_amount": 1,
"max_withdrawal_amount": 17391300
}
},
{
"id": "ETH",
"name": "Ether",
"min_size": "0.00000001",
"status": "online",
"message": "",
"max_precision": "0.00000001",
"convertible_to": [],
"details": {
"type": "crypto",
"symbol": "Ξ",
"network_confirmations": 35,
"sort_order": 25,
"crypto_address_link": "https://etherscan.io/address/{{address}}",
"crypto_transaction_link": "https://etherscan.io/tx/0x{{txId}}",
"push_payment_methods": [
"crypto"
],
"group_types": [
"eth",
"crypto"
],
"display_name": "",
"processing_time_seconds": 0,
"min_withdrawal_amount": 0.001,
"max_withdrawal_amount": 7450
}
},
{
"id": "EUR",
"name": "Euro",
"min_size": "0.01",
"status": "online",
"message": "",
"max_precision": "0.01",
"convertible_to": [],
"details": {
"type": "fiat",
"symbol": "€",
"network_confirmations": 0,
"sort_order": 2,
"crypto_address_link": "",
"crypto_transaction_link": "",
"push_payment_methods": [
"sepa_bank_account"
],
"group_types": [
"fiat",
"eur"
],
"display_name": "",
"processing_time_seconds": 0,
"min_withdrawal_amount": 0,
"max_withdrawal_amount": 0
}
},
{
"id": "GBP",
"name": "British Pound",
"min_size": "0.01",
"status": "online",
"message": "",
"max_precision": "0.01",
"convertible_to": [],
"details": {
"type": "fiat",
"symbol": "£",
"network_confirmations": 0,
"sort_order": 3,
"crypto_address_link": "",
"crypto_transaction_link": "",
"push_payment_methods": [
"uk_bank_account",
"swift_lhv",
"swift"
],
"group_types": [
"fiat",
"gbp"
],
"display_name": "",
"processing_time_seconds": 0,
"min_withdrawal_amount": 0,
"max_withdrawal_amount": 0
}
}
]

View file

@ -0,0 +1,62 @@
[
{
"id": "BTC-EUR",
"base_currency": "BTC",
"quote_currency": "EUR",
"base_min_size": "0.0001",
"base_max_size": "200",
"quote_increment": "0.01",
"base_increment": "0.00000001",
"display_name": "BTC/EUR",
"min_market_funds": "10",
"max_market_funds": "600000",
"margin_enabled": false,
"fx_stablecoin": false,
"post_only": false,
"limit_only": false,
"cancel_only": false,
"trading_disabled": false,
"status": "online",
"status_message": ""
},
{
"id": "ETH-GBP",
"base_currency": "ETH",
"quote_currency": "GBP",
"base_min_size": "0.001",
"base_max_size": "1400",
"quote_increment": "0.01",
"base_increment": "0.00000001",
"display_name": "ETH/GBP",
"min_market_funds": "10",
"max_market_funds": "1000000",
"margin_enabled": false,
"fx_stablecoin": false,
"post_only": false,
"limit_only": false,
"cancel_only": false,
"trading_disabled": false,
"status": "online",
"status_message": ""
},
{
"id": "DOGE-EUR",
"base_currency": "DOGE",
"quote_currency": "EUR",
"base_min_size": "1",
"base_max_size": "690000",
"quote_increment": "0.0001",
"base_increment": "0.1",
"display_name": "DOGE/EUR",
"min_market_funds": "5.0",
"max_market_funds": "100000",
"margin_enabled": false,
"fx_stablecoin": false,
"post_only": false,
"limit_only": false,
"cancel_only": false,
"trading_disabled": false,
"status": "online",
"status_message": ""
}
]

View file

@ -0,0 +1,58 @@
[
[
1609977600,
29516.98,
32900,
29818.73,
32120.19,
5957.46980324
],
[
1609891200,
27105.01,
29949,
27655.04,
29838.52,
4227.05067035
],
[
1609804800,
24413.62,
27989,
26104.4,
27654.01,
4036.27720179
],
[
1609718400,
22055,
26199,
25624.7,
26115.94,
6304.41029978
],
[
1609632000,
24500,
27195.46,
25916.75,
25644.41,
4975.13927959
],
[
1609545600,
22000,
27000,
24071.26,
25907.35,
7291.88538639
],
[
1609459200,
23512.7,
24250,
23706.73,
24070.97,
1830.04655405
]
]