Introduce series.

This commit is contained in:
Chris Berkhout 2021-05-27 12:47:12 +02:00
parent 06e0a32514
commit 2c657307a8
10 changed files with 103 additions and 91 deletions

View file

@ -1,4 +1,5 @@
import argparse import argparse
import dataclasses
import logging import logging
import shutil import shutil
from datetime import datetime, timedelta from datetime import datetime, timedelta
@ -6,6 +7,8 @@ from textwrap import TextWrapper
from pricehist import __version__, outputs, sources from pricehist import __version__, outputs, sources
from pricehist.format import Format from pricehist.format import Format
from pricehist.price import Price
from pricehist.series import Series
def cli(args=None): def cli(args=None):
@ -95,21 +98,23 @@ def cmd_fetch(args):
f"source start date of {source.start()}." f"source start date of {source.start()}."
) )
prices = source.fetch(args.pair, type, start, args.end) base, quote = args.pair.split("/")
series = source.fetch(Series(base, quote, type, start, args.end))
if args.invert:
series = dataclasses.replace(
series,
base=series.quote,
quote=series.base,
prices=[Price(date=p.date, amount=(1 / p.amount)) for p in series.prices],
)
if args.renamebase or args.renamequote: if args.renamebase or args.renamequote:
prices = [ series = dataclasses.replace(
p._replace( series,
base=(args.renamebase or p.base), base=(args.renamebase or base),
quote=(args.renamequote or p.quote), quote=(args.renamequote or quote),
) )
for p in prices
]
if args.invert:
prices = [
p._replace(base=p.quote, quote=p.base, amount=(1 / p.amount))
for p in prices
]
default = Format() default = Format()
@ -128,7 +133,7 @@ def cmd_fetch(args):
decimal_places=if_not_none(args.quantize, default.decimal_places), decimal_places=if_not_none(args.quantize, default.decimal_places),
) )
print(output.format(prices, source, type, fmt=fmt), end="") print(output.format(series, source, fmt=fmt), end="")
def build_parser(): def build_parser():

View file

@ -2,9 +2,9 @@ from pricehist.format import Format
class Beancount: class Beancount:
def format(self, prices, source=None, type=None, fmt=Format()): def format(self, series, source=None, fmt=Format()):
lines = [] lines = []
for price in prices: for price in series.prices:
amount_parts = f"{fmt.quantize(price.amount):,}".split(".") amount_parts = f"{fmt.quantize(price.amount):,}".split(".")
amount_parts[0] = amount_parts[0].replace(",", fmt.thousands) amount_parts[0] = amount_parts[0].replace(",", fmt.thousands)
@ -12,13 +12,13 @@ class Beancount:
qa_parts = [amount] qa_parts = [amount]
if fmt.symbol == "right": if fmt.symbol == "right":
qa_parts = qa_parts + [price.quote] qa_parts = qa_parts + [series.quote]
else: else:
qa_parts = qa_parts + [" ", price.quote] qa_parts = qa_parts + [" ", series.quote]
quote_amount = "".join(qa_parts) quote_amount = "".join(qa_parts)
date = str(price.date).replace("-", fmt.datesep) date = str(price.date).replace("-", fmt.datesep)
lines.append(f"{date} price {price.base} {quote_amount}") lines.append(f"{date} price {series.base} {quote_amount}")
return "\n".join(lines) + "\n" return "\n".join(lines) + "\n"

View file

@ -2,13 +2,15 @@ from pricehist.format import Format
class CSV: class CSV:
def format(self, prices, source=None, type=None, fmt=Format()): def format(self, series, source=None, fmt=Format()):
lines = ["date,base,quote,amount,source,type"] lines = ["date,base,quote,amount,source,type"]
for price in prices: for price in series.prices:
date = str(price.date).replace("-", fmt.datesep) date = str(price.date).replace("-", fmt.datesep)
amount_parts = f"{fmt.quantize(price.amount):,}".split(".") amount_parts = f"{fmt.quantize(price.amount):,}".split(".")
amount_parts[0] = amount_parts[0].replace(",", fmt.thousands) amount_parts[0] = amount_parts[0].replace(",", fmt.thousands)
amount = fmt.decimal.join(amount_parts) amount = fmt.decimal.join(amount_parts)
line = ",".join([date, price.base, price.quote, amount, source.id(), type]) line = ",".join(
[date, series.base, series.quote, amount, source.id(), series.type]
)
lines.append(line) lines.append(line)
return "\n".join(lines) + "\n" return "\n".join(lines) + "\n"

View file

@ -7,18 +7,18 @@ from pricehist.format import Format
class GnuCashSQL: class GnuCashSQL:
def format(self, prices, source=None, type=None, fmt=Format()): def format(self, series, source=None, fmt=Format()):
src = f"pricehist:{source.id()}" src = f"pricehist:{source.id()}"
values_parts = [] values_parts = []
for price in prices: for price in series.prices:
date = f"{price.date} {fmt.time}" date = f"{price.date} {fmt.time}"
amount = fmt.quantize(price.amount) amount = fmt.quantize(price.amount)
m = hashlib.sha256() m = hashlib.sha256()
m.update( m.update(
"".join([date, price.base, price.quote, src, type, str(amount)]).encode( "".join(
"utf-8" [date, series.base, series.quote, src, series.type, str(amount)]
) ).encode("utf-8")
) )
guid = m.hexdigest()[0:32] guid = m.hexdigest()[0:32]
value_num = str(amount).replace(".", "") value_num = str(amount).replace(".", "")
@ -27,10 +27,10 @@ class GnuCashSQL:
"(" "("
f"'{guid}', " f"'{guid}', "
f"'{date}', " f"'{date}', "
f"'{price.base}', " f"'{series.base}', "
f"'{price.quote}', " f"'{series.quote}', "
f"'{src}', " f"'{src}', "
f"'{type}', " f"'{series.type}', "
f"{value_num}, " f"{value_num}, "
f"{value_denom}" f"{value_denom}"
")" ")"
@ -41,8 +41,8 @@ class GnuCashSQL:
sql = read_text("pricehist.resources", "gnucash.sql").format( sql = read_text("pricehist.resources", "gnucash.sql").format(
version=__version__, version=__version__,
timestamp=datetime.utcnow().isoformat() + "Z", timestamp=datetime.utcnow().isoformat() + "Z",
base=price.base, base=series.base,
quote=price.quote, quote=series.quote,
values=values, values=values,
) )

View file

@ -2,9 +2,9 @@ from pricehist.format import Format
class Ledger: class Ledger:
def format(self, prices, source=None, type=None, fmt=Format()): def format(self, series, source=None, fmt=Format()):
lines = [] lines = []
for price in prices: for price in series.prices:
date = str(price.date).replace("-", fmt.datesep) date = str(price.date).replace("-", fmt.datesep)
amount_parts = f"{fmt.quantize(price.amount):,}".split(".") amount_parts = f"{fmt.quantize(price.amount):,}".split(".")
@ -13,16 +13,16 @@ class Ledger:
qa_parts = [amount] qa_parts = [amount]
if fmt.symbol == "left": if fmt.symbol == "left":
qa_parts = [price.quote] + qa_parts qa_parts = [series.quote] + qa_parts
elif fmt.symbol == "leftspace": elif fmt.symbol == "leftspace":
qa_parts = [price.quote, " "] + qa_parts qa_parts = [series.quote, " "] + qa_parts
elif fmt.symbol == "right": elif fmt.symbol == "right":
qa_parts = qa_parts + [price.quote] qa_parts = qa_parts + [series.quote]
else: else:
qa_parts = qa_parts + [" ", price.quote] qa_parts = qa_parts + [" ", series.quote]
quote_amount = "".join(qa_parts) quote_amount = "".join(qa_parts)
lines.append(f"P {date} {fmt.time} {price.base} {quote_amount}") lines.append(f"P {date} {fmt.time} {series.base} {quote_amount}")
return "\n".join(lines) + "\n" return "\n".join(lines) + "\n"
# TODO support additional details of the format: # TODO support additional details of the format:

View file

@ -4,7 +4,5 @@ from decimal import Decimal
@dataclass(frozen=True) @dataclass(frozen=True)
class Price: class Price:
base: str
quote: str
date: str date: str
amount: Decimal amount: Decimal

13
src/pricehist/series.py Normal file
View file

@ -0,0 +1,13 @@
from dataclasses import dataclass, field
from pricehist.price import Price
@dataclass(frozen=True)
class Series:
base: str
quote: str
type: str
start: str
end: str
prices: list[Price] = field(default_factory=list)

View file

@ -1,3 +1,4 @@
import dataclasses
import json import json
from decimal import Decimal from decimal import Decimal
@ -41,19 +42,17 @@ class CoinDesk:
) )
return symbols return symbols
def fetch(self, pair, type, start, end): def fetch(self, series):
base, quote = pair.split("/")
url = "https://api.coindesk.com/v1/bpi/historical/close.json" url = "https://api.coindesk.com/v1/bpi/historical/close.json"
params = { params = {
"currency": quote, "currency": series.quote,
"start": start, "start": series.start,
"end": end, "end": series.end,
} }
response = requests.get(url, params=params) response = requests.get(url, params=params)
data = json.loads(response.content) data = json.loads(response.content)
prices = [] prices = []
for (d, v) in data["bpi"].items(): for (d, v) in data["bpi"].items():
prices.append(Price(base, quote, d, Decimal(str(v)))) prices.append(Price(d, Decimal(str(v))))
return prices return dataclasses.replace(series, prices=prices)

View file

@ -1,3 +1,4 @@
import dataclasses
import json import json
from datetime import datetime from datetime import datetime
from decimal import Decimal from decimal import Decimal
@ -8,32 +9,25 @@ from pricehist.price import Price
class CoinMarketCap: class CoinMarketCap:
@staticmethod def id(self):
def id():
return "coinmarketcap" return "coinmarketcap"
@staticmethod def name(self):
def name():
return "CoinMarketCap" return "CoinMarketCap"
@staticmethod def description(self):
def description():
return "The world's most-referenced price-tracking website for cryptoassets" return "The world's most-referenced price-tracking website for cryptoassets"
@staticmethod def source_url(self):
def source_url():
return "https://coinmarketcap.com/" return "https://coinmarketcap.com/"
@staticmethod def start(self):
def start():
return "2013-04-28" return "2013-04-28"
@staticmethod def types(self):
def types():
return ["mid", "open", "high", "low", "close"] return ["mid", "open", "high", "low", "close"]
@staticmethod def notes(self):
def notes():
return ( return (
"This source makes unoffical use of endpoints that power CoinMarketCap's " "This source makes unoffical use of endpoints that power CoinMarketCap's "
"public web interface. The price data comes from a public equivalent of " "public web interface. The price data comes from a public equivalent of "
@ -52,36 +46,36 @@ class CoinMarketCap:
rows = [i.ljust(id_width + 4) + d for i, d in zip(ids, descriptions)] rows = [i.ljust(id_width + 4) + d for i, d in zip(ids, descriptions)]
return rows return rows
def fetch(self, pair, type, start, end): def fetch(self, series):
base, quote = pair.split("/")
url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical" url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical"
params = {} params = {}
if base.startswith("id=") or quote.startswith("id="): if series.base.startswith("id=") or series.quote.startswith("id="):
symbols = {} symbols = {}
for i in self._symbol_data(): for i in self._symbol_data():
symbols[str(i["id"])] = i["symbol"] or i["code"] symbols[str(i["id"])] = i["symbol"] or i["code"]
if base.startswith("id="): if series.base.startswith("id="):
params["id"] = base[3:] params["id"] = series.base[3:]
output_base = symbols[base[3:]] output_base = symbols[series.base[3:]]
else: else:
params["symbol"] = base params["symbol"] = series.base
output_base = base output_base = series.base
if quote.startswith("id="): if series.quote.startswith("id="):
params["convert_id"] = quote[3:] params["convert_id"] = series.quote[3:]
quote_key = quote[3:] quote_key = series.quote[3:]
output_quote = symbols[quote[3:]] output_quote = symbols[series.quote[3:]]
else: else:
params["convert"] = quote params["convert"] = series.quote
quote_key = quote quote_key = series.quote
output_quote = quote output_quote = series.quote
params["time_start"] = int(datetime.strptime(start, "%Y-%m-%d").timestamp()) params["time_start"] = int(
datetime.strptime(series.start, "%Y-%m-%d").timestamp()
)
params["time_end"] = ( params["time_end"] = (
int(datetime.strptime(end, "%Y-%m-%d").timestamp()) + 24 * 60 * 60 int(datetime.strptime(series.end, "%Y-%m-%d").timestamp()) + 24 * 60 * 60
) # round up to include the last day ) # round up to include the last day
response = requests.get(url, params=params) response = requests.get(url, params=params)
@ -90,10 +84,12 @@ class CoinMarketCap:
prices = [] prices = []
for item in data["data"]["quotes"]: for item in data["data"]["quotes"]:
d = item["time_open"][0:10] d = item["time_open"][0:10]
amount = self._amount(item["quote"][quote_key], type) amount = self._amount(item["quote"][quote_key], series.type)
prices.append(Price(output_base, output_quote, d, amount)) prices.append(Price(d, amount))
return prices return dataclasses.replace(
series, base=output_base, quote=output_quote, prices=prices
)
def _symbol_data(self): def _symbol_data(self):
fiat_url = "https://web-api.coinmarketcap.com/v1/fiat/map?include_metals=true" fiat_url = "https://web-api.coinmarketcap.com/v1/fiat/map?include_metals=true"

View file

@ -1,3 +1,4 @@
import dataclasses
from datetime import datetime, timedelta from datetime import datetime, timedelta
from decimal import Decimal from decimal import Decimal
@ -39,11 +40,9 @@ class ECB:
pairs = [f"EUR/{c} Euro against {iso[c].name}" for c in currencies] pairs = [f"EUR/{c} Euro against {iso[c].name}" for c in currencies]
return pairs return pairs
def fetch(self, pair, type, start, end): def fetch(self, series):
base, quote = pair.split("/")
almost_90_days_ago = str(datetime.now().date() - timedelta(days=85)) almost_90_days_ago = str(datetime.now().date() - timedelta(days=85))
data = self._raw_data(start < almost_90_days_ago) data = self._raw_data(series.start < almost_90_days_ago)
root = etree.fromstring(data) root = etree.fromstring(data)
all_rows = [] all_rows = []
@ -51,14 +50,14 @@ class ECB:
date = day.attrib["time"] date = day.attrib["time"]
# TODO what if it's not found for that day? # TODO what if it's not found for that day?
# (some quotes aren't in the earliest data) # (some quotes aren't in the earliest data)
for row in day.cssselect(f"[currency='{quote}']"): for row in day.cssselect(f"[currency='{series.quote}']"):
rate = Decimal(row.attrib["rate"]) rate = Decimal(row.attrib["rate"])
all_rows.insert(0, (date, rate)) all_rows.insert(0, (date, rate))
selected = [ selected = [
Price(base, quote, d, r) for d, r in all_rows if d >= start and d <= end Price(d, r) for d, r in all_rows if d >= series.start and d <= series.end
] ]
return selected return dataclasses.replace(series, prices=selected)
def _raw_data(self, more_than_90_days=False): def _raw_data(self, more_than_90_days=False):
url_base = "https://www.ecb.europa.eu/stats/eurofxref" url_base = "https://www.ecb.europa.eu/stats/eurofxref"