From c4df91d0667e91174a8b946c60fdab1bc1532409 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 27 May 2021 16:31:05 +0200 Subject: [PATCH] Log requests as curls. --- poetry.lock | 16 +++++++++++++++- pyproject.toml | 1 + src/pricehist/sources/basesource.py | 15 +++++++++++++++ src/pricehist/sources/coindesk.py | 4 ++-- src/pricehist/sources/coinmarketcap.py | 10 ++++++---- src/pricehist/sources/ecb.py | 9 +++++---- 6 files changed, 44 insertions(+), 11 deletions(-) diff --git a/poetry.lock b/poetry.lock index d947d1d..16da670 100644 --- a/poetry.lock +++ b/poetry.lock @@ -90,6 +90,17 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "curlify" +version = "2.2.1" +description = "Library to convert python requests object to curl command." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +requests = "*" + [[package]] name = "flake8" version = "3.9.1" @@ -311,7 +322,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "8674071f4ee6a2bd084bc4aa02e2d3abaca72eeba749ee5ac67de9a8001ac8db" +content-hash = "a0b4419f4cb60c138eeecce88efa4e617c55f4bd2b83e2c22da471bbc7863d36" [metadata.files] appdirs = [ @@ -349,6 +360,9 @@ cssselect = [ {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"}, {file = "cssselect-1.1.0.tar.gz", hash = "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc"}, ] +curlify = [ + {file = "curlify-2.2.1.tar.gz", hash = "sha256:0d3f02e7235faf952de8ef45ef469845196d30632d5838bcd5aee217726ddd6d"}, +] flake8 = [ {file = "flake8-3.9.1-py2.py3-none-any.whl", hash = "sha256:3b9f848952dddccf635be78098ca75010f073bfe14d2c6bda867154bea728d2a"}, {file = "flake8-3.9.1.tar.gz", hash = "sha256:1aa8990be1e689d96c745c5682b687ea49f2e05a443aff1f8251092b0014e378"}, diff --git a/pyproject.toml b/pyproject.toml index 038f363..94c1815 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ python = "^3.9" requests = "^2.25.1" lxml = "^4.6.2" cssselect = "^1.1.0" +curlify = "^2.2.1" [tool.poetry.dev-dependencies] pytest = "^6.2.2" diff --git a/src/pricehist/sources/basesource.py b/src/pricehist/sources/basesource.py index 88394c0..8e04c09 100644 --- a/src/pricehist/sources/basesource.py +++ b/src/pricehist/sources/basesource.py @@ -1,5 +1,8 @@ +import logging from abc import ABC, abstractmethod +import curlify + from pricehist.series import Series @@ -12,23 +15,35 @@ class BaseSource(ABC): def name(self) -> str: pass + @abstractmethod def description(self) -> str: pass + @abstractmethod def source_url(self) -> str: pass + @abstractmethod def start(self) -> str: pass + @abstractmethod def types(self) -> list[str]: pass + @abstractmethod def notes(self) -> str: pass + @abstractmethod def symbols(self) -> list[str]: pass + @abstractmethod def fetch(self, series: Series) -> Series: pass + + def log_curl(self, response): + curl = curlify.to_curl(response.request, compressed=True) + logging.debug(f"Request to {self.id()}: {curl}") + return response diff --git a/src/pricehist/sources/coindesk.py b/src/pricehist/sources/coindesk.py index 1f6783f..aa9aa02 100644 --- a/src/pricehist/sources/coindesk.py +++ b/src/pricehist/sources/coindesk.py @@ -36,7 +36,7 @@ class CoinDesk(BaseSource): def symbols(self): url = "https://api.coindesk.com/v1/bpi/supported-currencies.json" - response = requests.get(url) + response = self.log_curl(requests.get(url)) data = json.loads(response.content) relevant = [i for i in data if i["currency"] not in ["XBT", "BTC"]] symbols = sorted( @@ -58,5 +58,5 @@ class CoinDesk(BaseSource): "start": series.start, "end": series.end, } - response = requests.get(url, params=params) + response = self.log_curl(requests.get(url, params=params)) return json.loads(response.content) diff --git a/src/pricehist/sources/coinmarketcap.py b/src/pricehist/sources/coinmarketcap.py index f498187..37e49a8 100644 --- a/src/pricehist/sources/coinmarketcap.py +++ b/src/pricehist/sources/coinmarketcap.py @@ -7,8 +7,10 @@ import requests from pricehist.price import Price +from .basesource import BaseSource -class CoinMarketCap: + +class CoinMarketCap(BaseSource): def id(self): return "coinmarketcap" @@ -83,7 +85,7 @@ class CoinMarketCap: int(datetime.strptime(series.end, "%Y-%m-%d").timestamp()) + 24 * 60 * 60 ) # round up to include the last day - response = requests.get(url, params=params) + response = self.log_curl(requests.get(url, params=params)) return json.loads(response.content) @@ -106,11 +108,11 @@ class CoinMarketCap: def _symbol_data(self): fiat_url = "https://web-api.coinmarketcap.com/v1/fiat/map?include_metals=true" - fiat_res = requests.get(fiat_url) + fiat_res = self.log_curl(requests.get(fiat_url)) fiat = json.loads(fiat_res.content) crypto_url = ( "https://web-api.coinmarketcap.com/v1/cryptocurrency/map?sort=cmc_rank" ) - crypto_res = requests.get(crypto_url) + crypto_res = self.log_curl(requests.get(crypto_url)) crypto = json.loads(crypto_res.content) return crypto["data"] + fiat["data"] diff --git a/src/pricehist/sources/ecb.py b/src/pricehist/sources/ecb.py index 3ac6498..c8ed92c 100644 --- a/src/pricehist/sources/ecb.py +++ b/src/pricehist/sources/ecb.py @@ -8,8 +8,10 @@ from lxml import etree from pricehist import isocurrencies from pricehist.price import Price +from .basesource import BaseSource -class ECB: + +class ECB(BaseSource): def id(self): return "ecb" @@ -32,8 +34,7 @@ class ECB: return "" def symbols(self): - data = self._raw_data(more_than_90_days=True) - root = etree.fromstring(data) + root = self._data(more_than_90_days=True) nodes = root.cssselect("[currency]") currencies = sorted(set([n.attrib["currency"] for n in nodes])) iso = isocurrencies.by_code() @@ -65,6 +66,6 @@ class ECB: else: source_url = f"{url_base}/eurofxref-hist-90d.xml" # last 90 days - response = requests.get(source_url) + response = self.log_curl(requests.get(source_url)) root = etree.fromstring(response.content) return root