Log requests as curls.

poetry.lock

@@ -90,6 +90,17 @@ category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 
+[[package]]
+name = "curlify"
+version = "2.2.1"
+description = "Library to convert python requests object to curl command."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+requests = "*"
+
 [[package]]
 name = "flake8"
 version = "3.9.1"
@@ -311,7 +322,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.9"
-content-hash = "8674071f4ee6a2bd084bc4aa02e2d3abaca72eeba749ee5ac67de9a8001ac8db"
+content-hash = "a0b4419f4cb60c138eeecce88efa4e617c55f4bd2b83e2c22da471bbc7863d36"
 
 [metadata.files]
 appdirs = [
@@ -349,6 +360,9 @@ cssselect = [
     {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"},
     {file = "cssselect-1.1.0.tar.gz", hash = "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc"},
 ]
+curlify = [
+    {file = "curlify-2.2.1.tar.gz", hash = "sha256:0d3f02e7235faf952de8ef45ef469845196d30632d5838bcd5aee217726ddd6d"},
+]
 flake8 = [
     {file = "flake8-3.9.1-py2.py3-none-any.whl", hash = "sha256:3b9f848952dddccf635be78098ca75010f073bfe14d2c6bda867154bea728d2a"},
     {file = "flake8-3.9.1.tar.gz", hash = "sha256:1aa8990be1e689d96c745c5682b687ea49f2e05a443aff1f8251092b0014e378"},

pyproject.toml

@@ -10,6 +10,7 @@ python = "^3.9"
 requests = "^2.25.1"
 lxml = "^4.6.2"
 cssselect = "^1.1.0"
+curlify = "^2.2.1"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.2"

src/pricehist/sources/basesource.py

@@ -1,5 +1,8 @@
+import logging
 from abc import ABC, abstractmethod
 
+import curlify
+
 from pricehist.series import Series
 
 
@@ -12,23 +15,35 @@ class BaseSource(ABC):
     def name(self) -> str:
         pass
 
+    @abstractmethod
     def description(self) -> str:
         pass
 
+    @abstractmethod
     def source_url(self) -> str:
         pass
 
+    @abstractmethod
     def start(self) -> str:
         pass
 
+    @abstractmethod
     def types(self) -> list[str]:
         pass
 
+    @abstractmethod
     def notes(self) -> str:
         pass
 
+    @abstractmethod
     def symbols(self) -> list[str]:
         pass
 
+    @abstractmethod
     def fetch(self, series: Series) -> Series:
         pass
+
+    def log_curl(self, response):
+        curl = curlify.to_curl(response.request, compressed=True)
+        logging.debug(f"Request to {self.id()}: {curl}")
+        return response

src/pricehist/sources/coindesk.py

@@ -36,7 +36,7 @@ class CoinDesk(BaseSource):
 
     def symbols(self):
         url = "https://api.coindesk.com/v1/bpi/supported-currencies.json"
-        response = requests.get(url)
+        response = self.log_curl(requests.get(url))
         data = json.loads(response.content)
         relevant = [i for i in data if i["currency"] not in ["XBT", "BTC"]]
         symbols = sorted(
@@ -58,5 +58,5 @@ class CoinDesk(BaseSource):
             "start": series.start,
             "end": series.end,
         }
-        response = requests.get(url, params=params)
+        response = self.log_curl(requests.get(url, params=params))
         return json.loads(response.content)

src/pricehist/sources/coinmarketcap.py

@@ -7,8 +7,10 @@ import requests
 
 from pricehist.price import Price
 
+from .basesource import BaseSource
 
-class CoinMarketCap:
+
+class CoinMarketCap(BaseSource):
     def id(self):
         return "coinmarketcap"
 
@@ -83,7 +85,7 @@ class CoinMarketCap:
             int(datetime.strptime(series.end, "%Y-%m-%d").timestamp()) + 24 * 60 * 60
         )  # round up to include the last day
 
-        response = requests.get(url, params=params)
+        response = self.log_curl(requests.get(url, params=params))
 
         return json.loads(response.content)
 
@@ -106,11 +108,11 @@ class CoinMarketCap:
 
     def _symbol_data(self):
         fiat_url = "https://web-api.coinmarketcap.com/v1/fiat/map?include_metals=true"
-        fiat_res = requests.get(fiat_url)
+        fiat_res = self.log_curl(requests.get(fiat_url))
         fiat = json.loads(fiat_res.content)
         crypto_url = (
             "https://web-api.coinmarketcap.com/v1/cryptocurrency/map?sort=cmc_rank"
         )
-        crypto_res = requests.get(crypto_url)
+        crypto_res = self.log_curl(requests.get(crypto_url))
         crypto = json.loads(crypto_res.content)
         return crypto["data"] + fiat["data"]

src/pricehist/sources/ecb.py

@@ -8,8 +8,10 @@ from lxml import etree
 from pricehist import isocurrencies
 from pricehist.price import Price
 
+from .basesource import BaseSource
 
-class ECB:
+
+class ECB(BaseSource):
     def id(self):
         return "ecb"
 
@@ -32,8 +34,7 @@ class ECB:
         return ""
 
     def symbols(self):
-        data = self._raw_data(more_than_90_days=True)
-        root = etree.fromstring(data)
+        root = self._data(more_than_90_days=True)
         nodes = root.cssselect("[currency]")
         currencies = sorted(set([n.attrib["currency"] for n in nodes]))
         iso = isocurrencies.by_code()
@@ -65,6 +66,6 @@ class ECB:
         else:
             source_url = f"{url_base}/eurofxref-hist-90d.xml"  # last 90 days
 
-        response = requests.get(source_url)
+        response = self.log_curl(requests.get(source_url))
         root = etree.fromstring(response.content)
         return root