Introduce series.
This commit is contained in:
parent
06e0a32514
commit
2c657307a8
10 changed files with 103 additions and 91 deletions
|
@ -1,4 +1,5 @@
|
||||||
import argparse
|
import argparse
|
||||||
|
import dataclasses
|
||||||
import logging
|
import logging
|
||||||
import shutil
|
import shutil
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
@ -6,6 +7,8 @@ from textwrap import TextWrapper
|
||||||
|
|
||||||
from pricehist import __version__, outputs, sources
|
from pricehist import __version__, outputs, sources
|
||||||
from pricehist.format import Format
|
from pricehist.format import Format
|
||||||
|
from pricehist.price import Price
|
||||||
|
from pricehist.series import Series
|
||||||
|
|
||||||
|
|
||||||
def cli(args=None):
|
def cli(args=None):
|
||||||
|
@ -95,21 +98,23 @@ def cmd_fetch(args):
|
||||||
f"source start date of {source.start()}."
|
f"source start date of {source.start()}."
|
||||||
)
|
)
|
||||||
|
|
||||||
prices = source.fetch(args.pair, type, start, args.end)
|
base, quote = args.pair.split("/")
|
||||||
|
series = source.fetch(Series(base, quote, type, start, args.end))
|
||||||
|
|
||||||
|
if args.invert:
|
||||||
|
series = dataclasses.replace(
|
||||||
|
series,
|
||||||
|
base=series.quote,
|
||||||
|
quote=series.base,
|
||||||
|
prices=[Price(date=p.date, amount=(1 / p.amount)) for p in series.prices],
|
||||||
|
)
|
||||||
|
|
||||||
if args.renamebase or args.renamequote:
|
if args.renamebase or args.renamequote:
|
||||||
prices = [
|
series = dataclasses.replace(
|
||||||
p._replace(
|
series,
|
||||||
base=(args.renamebase or p.base),
|
base=(args.renamebase or base),
|
||||||
quote=(args.renamequote or p.quote),
|
quote=(args.renamequote or quote),
|
||||||
)
|
)
|
||||||
for p in prices
|
|
||||||
]
|
|
||||||
if args.invert:
|
|
||||||
prices = [
|
|
||||||
p._replace(base=p.quote, quote=p.base, amount=(1 / p.amount))
|
|
||||||
for p in prices
|
|
||||||
]
|
|
||||||
|
|
||||||
default = Format()
|
default = Format()
|
||||||
|
|
||||||
|
@ -128,7 +133,7 @@ def cmd_fetch(args):
|
||||||
decimal_places=if_not_none(args.quantize, default.decimal_places),
|
decimal_places=if_not_none(args.quantize, default.decimal_places),
|
||||||
)
|
)
|
||||||
|
|
||||||
print(output.format(prices, source, type, fmt=fmt), end="")
|
print(output.format(series, source, fmt=fmt), end="")
|
||||||
|
|
||||||
|
|
||||||
def build_parser():
|
def build_parser():
|
||||||
|
|
|
@ -2,9 +2,9 @@ from pricehist.format import Format
|
||||||
|
|
||||||
|
|
||||||
class Beancount:
|
class Beancount:
|
||||||
def format(self, prices, source=None, type=None, fmt=Format()):
|
def format(self, series, source=None, fmt=Format()):
|
||||||
lines = []
|
lines = []
|
||||||
for price in prices:
|
for price in series.prices:
|
||||||
|
|
||||||
amount_parts = f"{fmt.quantize(price.amount):,}".split(".")
|
amount_parts = f"{fmt.quantize(price.amount):,}".split(".")
|
||||||
amount_parts[0] = amount_parts[0].replace(",", fmt.thousands)
|
amount_parts[0] = amount_parts[0].replace(",", fmt.thousands)
|
||||||
|
@ -12,13 +12,13 @@ class Beancount:
|
||||||
|
|
||||||
qa_parts = [amount]
|
qa_parts = [amount]
|
||||||
if fmt.symbol == "right":
|
if fmt.symbol == "right":
|
||||||
qa_parts = qa_parts + [price.quote]
|
qa_parts = qa_parts + [series.quote]
|
||||||
else:
|
else:
|
||||||
qa_parts = qa_parts + [" ", price.quote]
|
qa_parts = qa_parts + [" ", series.quote]
|
||||||
quote_amount = "".join(qa_parts)
|
quote_amount = "".join(qa_parts)
|
||||||
|
|
||||||
date = str(price.date).replace("-", fmt.datesep)
|
date = str(price.date).replace("-", fmt.datesep)
|
||||||
lines.append(f"{date} price {price.base} {quote_amount}")
|
lines.append(f"{date} price {series.base} {quote_amount}")
|
||||||
return "\n".join(lines) + "\n"
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,13 +2,15 @@ from pricehist.format import Format
|
||||||
|
|
||||||
|
|
||||||
class CSV:
|
class CSV:
|
||||||
def format(self, prices, source=None, type=None, fmt=Format()):
|
def format(self, series, source=None, fmt=Format()):
|
||||||
lines = ["date,base,quote,amount,source,type"]
|
lines = ["date,base,quote,amount,source,type"]
|
||||||
for price in prices:
|
for price in series.prices:
|
||||||
date = str(price.date).replace("-", fmt.datesep)
|
date = str(price.date).replace("-", fmt.datesep)
|
||||||
amount_parts = f"{fmt.quantize(price.amount):,}".split(".")
|
amount_parts = f"{fmt.quantize(price.amount):,}".split(".")
|
||||||
amount_parts[0] = amount_parts[0].replace(",", fmt.thousands)
|
amount_parts[0] = amount_parts[0].replace(",", fmt.thousands)
|
||||||
amount = fmt.decimal.join(amount_parts)
|
amount = fmt.decimal.join(amount_parts)
|
||||||
line = ",".join([date, price.base, price.quote, amount, source.id(), type])
|
line = ",".join(
|
||||||
|
[date, series.base, series.quote, amount, source.id(), series.type]
|
||||||
|
)
|
||||||
lines.append(line)
|
lines.append(line)
|
||||||
return "\n".join(lines) + "\n"
|
return "\n".join(lines) + "\n"
|
||||||
|
|
|
@ -7,18 +7,18 @@ from pricehist.format import Format
|
||||||
|
|
||||||
|
|
||||||
class GnuCashSQL:
|
class GnuCashSQL:
|
||||||
def format(self, prices, source=None, type=None, fmt=Format()):
|
def format(self, series, source=None, fmt=Format()):
|
||||||
src = f"pricehist:{source.id()}"
|
src = f"pricehist:{source.id()}"
|
||||||
|
|
||||||
values_parts = []
|
values_parts = []
|
||||||
for price in prices:
|
for price in series.prices:
|
||||||
date = f"{price.date} {fmt.time}"
|
date = f"{price.date} {fmt.time}"
|
||||||
amount = fmt.quantize(price.amount)
|
amount = fmt.quantize(price.amount)
|
||||||
m = hashlib.sha256()
|
m = hashlib.sha256()
|
||||||
m.update(
|
m.update(
|
||||||
"".join([date, price.base, price.quote, src, type, str(amount)]).encode(
|
"".join(
|
||||||
"utf-8"
|
[date, series.base, series.quote, src, series.type, str(amount)]
|
||||||
)
|
).encode("utf-8")
|
||||||
)
|
)
|
||||||
guid = m.hexdigest()[0:32]
|
guid = m.hexdigest()[0:32]
|
||||||
value_num = str(amount).replace(".", "")
|
value_num = str(amount).replace(".", "")
|
||||||
|
@ -27,10 +27,10 @@ class GnuCashSQL:
|
||||||
"("
|
"("
|
||||||
f"'{guid}', "
|
f"'{guid}', "
|
||||||
f"'{date}', "
|
f"'{date}', "
|
||||||
f"'{price.base}', "
|
f"'{series.base}', "
|
||||||
f"'{price.quote}', "
|
f"'{series.quote}', "
|
||||||
f"'{src}', "
|
f"'{src}', "
|
||||||
f"'{type}', "
|
f"'{series.type}', "
|
||||||
f"{value_num}, "
|
f"{value_num}, "
|
||||||
f"{value_denom}"
|
f"{value_denom}"
|
||||||
")"
|
")"
|
||||||
|
@ -41,8 +41,8 @@ class GnuCashSQL:
|
||||||
sql = read_text("pricehist.resources", "gnucash.sql").format(
|
sql = read_text("pricehist.resources", "gnucash.sql").format(
|
||||||
version=__version__,
|
version=__version__,
|
||||||
timestamp=datetime.utcnow().isoformat() + "Z",
|
timestamp=datetime.utcnow().isoformat() + "Z",
|
||||||
base=price.base,
|
base=series.base,
|
||||||
quote=price.quote,
|
quote=series.quote,
|
||||||
values=values,
|
values=values,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -2,9 +2,9 @@ from pricehist.format import Format
|
||||||
|
|
||||||
|
|
||||||
class Ledger:
|
class Ledger:
|
||||||
def format(self, prices, source=None, type=None, fmt=Format()):
|
def format(self, series, source=None, fmt=Format()):
|
||||||
lines = []
|
lines = []
|
||||||
for price in prices:
|
for price in series.prices:
|
||||||
date = str(price.date).replace("-", fmt.datesep)
|
date = str(price.date).replace("-", fmt.datesep)
|
||||||
|
|
||||||
amount_parts = f"{fmt.quantize(price.amount):,}".split(".")
|
amount_parts = f"{fmt.quantize(price.amount):,}".split(".")
|
||||||
|
@ -13,16 +13,16 @@ class Ledger:
|
||||||
|
|
||||||
qa_parts = [amount]
|
qa_parts = [amount]
|
||||||
if fmt.symbol == "left":
|
if fmt.symbol == "left":
|
||||||
qa_parts = [price.quote] + qa_parts
|
qa_parts = [series.quote] + qa_parts
|
||||||
elif fmt.symbol == "leftspace":
|
elif fmt.symbol == "leftspace":
|
||||||
qa_parts = [price.quote, " "] + qa_parts
|
qa_parts = [series.quote, " "] + qa_parts
|
||||||
elif fmt.symbol == "right":
|
elif fmt.symbol == "right":
|
||||||
qa_parts = qa_parts + [price.quote]
|
qa_parts = qa_parts + [series.quote]
|
||||||
else:
|
else:
|
||||||
qa_parts = qa_parts + [" ", price.quote]
|
qa_parts = qa_parts + [" ", series.quote]
|
||||||
quote_amount = "".join(qa_parts)
|
quote_amount = "".join(qa_parts)
|
||||||
|
|
||||||
lines.append(f"P {date} {fmt.time} {price.base} {quote_amount}")
|
lines.append(f"P {date} {fmt.time} {series.base} {quote_amount}")
|
||||||
return "\n".join(lines) + "\n"
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
# TODO support additional details of the format:
|
# TODO support additional details of the format:
|
||||||
|
|
|
@ -4,7 +4,5 @@ from decimal import Decimal
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class Price:
|
class Price:
|
||||||
base: str
|
|
||||||
quote: str
|
|
||||||
date: str
|
date: str
|
||||||
amount: Decimal
|
amount: Decimal
|
||||||
|
|
13
src/pricehist/series.py
Normal file
13
src/pricehist/series.py
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
from pricehist.price import Price
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Series:
|
||||||
|
base: str
|
||||||
|
quote: str
|
||||||
|
type: str
|
||||||
|
start: str
|
||||||
|
end: str
|
||||||
|
prices: list[Price] = field(default_factory=list)
|
|
@ -1,3 +1,4 @@
|
||||||
|
import dataclasses
|
||||||
import json
|
import json
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
|
|
||||||
|
@ -41,19 +42,17 @@ class CoinDesk:
|
||||||
)
|
)
|
||||||
return symbols
|
return symbols
|
||||||
|
|
||||||
def fetch(self, pair, type, start, end):
|
def fetch(self, series):
|
||||||
base, quote = pair.split("/")
|
|
||||||
|
|
||||||
url = "https://api.coindesk.com/v1/bpi/historical/close.json"
|
url = "https://api.coindesk.com/v1/bpi/historical/close.json"
|
||||||
params = {
|
params = {
|
||||||
"currency": quote,
|
"currency": series.quote,
|
||||||
"start": start,
|
"start": series.start,
|
||||||
"end": end,
|
"end": series.end,
|
||||||
}
|
}
|
||||||
response = requests.get(url, params=params)
|
response = requests.get(url, params=params)
|
||||||
data = json.loads(response.content)
|
data = json.loads(response.content)
|
||||||
prices = []
|
prices = []
|
||||||
for (d, v) in data["bpi"].items():
|
for (d, v) in data["bpi"].items():
|
||||||
prices.append(Price(base, quote, d, Decimal(str(v))))
|
prices.append(Price(d, Decimal(str(v))))
|
||||||
|
|
||||||
return prices
|
return dataclasses.replace(series, prices=prices)
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import dataclasses
|
||||||
import json
|
import json
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
|
@ -8,32 +9,25 @@ from pricehist.price import Price
|
||||||
|
|
||||||
|
|
||||||
class CoinMarketCap:
|
class CoinMarketCap:
|
||||||
@staticmethod
|
def id(self):
|
||||||
def id():
|
|
||||||
return "coinmarketcap"
|
return "coinmarketcap"
|
||||||
|
|
||||||
@staticmethod
|
def name(self):
|
||||||
def name():
|
|
||||||
return "CoinMarketCap"
|
return "CoinMarketCap"
|
||||||
|
|
||||||
@staticmethod
|
def description(self):
|
||||||
def description():
|
|
||||||
return "The world's most-referenced price-tracking website for cryptoassets"
|
return "The world's most-referenced price-tracking website for cryptoassets"
|
||||||
|
|
||||||
@staticmethod
|
def source_url(self):
|
||||||
def source_url():
|
|
||||||
return "https://coinmarketcap.com/"
|
return "https://coinmarketcap.com/"
|
||||||
|
|
||||||
@staticmethod
|
def start(self):
|
||||||
def start():
|
|
||||||
return "2013-04-28"
|
return "2013-04-28"
|
||||||
|
|
||||||
@staticmethod
|
def types(self):
|
||||||
def types():
|
|
||||||
return ["mid", "open", "high", "low", "close"]
|
return ["mid", "open", "high", "low", "close"]
|
||||||
|
|
||||||
@staticmethod
|
def notes(self):
|
||||||
def notes():
|
|
||||||
return (
|
return (
|
||||||
"This source makes unoffical use of endpoints that power CoinMarketCap's "
|
"This source makes unoffical use of endpoints that power CoinMarketCap's "
|
||||||
"public web interface. The price data comes from a public equivalent of "
|
"public web interface. The price data comes from a public equivalent of "
|
||||||
|
@ -52,36 +46,36 @@ class CoinMarketCap:
|
||||||
rows = [i.ljust(id_width + 4) + d for i, d in zip(ids, descriptions)]
|
rows = [i.ljust(id_width + 4) + d for i, d in zip(ids, descriptions)]
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
def fetch(self, pair, type, start, end):
|
def fetch(self, series):
|
||||||
base, quote = pair.split("/")
|
|
||||||
|
|
||||||
url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical"
|
url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical"
|
||||||
|
|
||||||
params = {}
|
params = {}
|
||||||
if base.startswith("id=") or quote.startswith("id="):
|
if series.base.startswith("id=") or series.quote.startswith("id="):
|
||||||
symbols = {}
|
symbols = {}
|
||||||
for i in self._symbol_data():
|
for i in self._symbol_data():
|
||||||
symbols[str(i["id"])] = i["symbol"] or i["code"]
|
symbols[str(i["id"])] = i["symbol"] or i["code"]
|
||||||
|
|
||||||
if base.startswith("id="):
|
if series.base.startswith("id="):
|
||||||
params["id"] = base[3:]
|
params["id"] = series.base[3:]
|
||||||
output_base = symbols[base[3:]]
|
output_base = symbols[series.base[3:]]
|
||||||
else:
|
else:
|
||||||
params["symbol"] = base
|
params["symbol"] = series.base
|
||||||
output_base = base
|
output_base = series.base
|
||||||
|
|
||||||
if quote.startswith("id="):
|
if series.quote.startswith("id="):
|
||||||
params["convert_id"] = quote[3:]
|
params["convert_id"] = series.quote[3:]
|
||||||
quote_key = quote[3:]
|
quote_key = series.quote[3:]
|
||||||
output_quote = symbols[quote[3:]]
|
output_quote = symbols[series.quote[3:]]
|
||||||
else:
|
else:
|
||||||
params["convert"] = quote
|
params["convert"] = series.quote
|
||||||
quote_key = quote
|
quote_key = series.quote
|
||||||
output_quote = quote
|
output_quote = series.quote
|
||||||
|
|
||||||
params["time_start"] = int(datetime.strptime(start, "%Y-%m-%d").timestamp())
|
params["time_start"] = int(
|
||||||
|
datetime.strptime(series.start, "%Y-%m-%d").timestamp()
|
||||||
|
)
|
||||||
params["time_end"] = (
|
params["time_end"] = (
|
||||||
int(datetime.strptime(end, "%Y-%m-%d").timestamp()) + 24 * 60 * 60
|
int(datetime.strptime(series.end, "%Y-%m-%d").timestamp()) + 24 * 60 * 60
|
||||||
) # round up to include the last day
|
) # round up to include the last day
|
||||||
|
|
||||||
response = requests.get(url, params=params)
|
response = requests.get(url, params=params)
|
||||||
|
@ -90,10 +84,12 @@ class CoinMarketCap:
|
||||||
prices = []
|
prices = []
|
||||||
for item in data["data"]["quotes"]:
|
for item in data["data"]["quotes"]:
|
||||||
d = item["time_open"][0:10]
|
d = item["time_open"][0:10]
|
||||||
amount = self._amount(item["quote"][quote_key], type)
|
amount = self._amount(item["quote"][quote_key], series.type)
|
||||||
prices.append(Price(output_base, output_quote, d, amount))
|
prices.append(Price(d, amount))
|
||||||
|
|
||||||
return prices
|
return dataclasses.replace(
|
||||||
|
series, base=output_base, quote=output_quote, prices=prices
|
||||||
|
)
|
||||||
|
|
||||||
def _symbol_data(self):
|
def _symbol_data(self):
|
||||||
fiat_url = "https://web-api.coinmarketcap.com/v1/fiat/map?include_metals=true"
|
fiat_url = "https://web-api.coinmarketcap.com/v1/fiat/map?include_metals=true"
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import dataclasses
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
|
|
||||||
|
@ -39,11 +40,9 @@ class ECB:
|
||||||
pairs = [f"EUR/{c} Euro against {iso[c].name}" for c in currencies]
|
pairs = [f"EUR/{c} Euro against {iso[c].name}" for c in currencies]
|
||||||
return pairs
|
return pairs
|
||||||
|
|
||||||
def fetch(self, pair, type, start, end):
|
def fetch(self, series):
|
||||||
base, quote = pair.split("/")
|
|
||||||
|
|
||||||
almost_90_days_ago = str(datetime.now().date() - timedelta(days=85))
|
almost_90_days_ago = str(datetime.now().date() - timedelta(days=85))
|
||||||
data = self._raw_data(start < almost_90_days_ago)
|
data = self._raw_data(series.start < almost_90_days_ago)
|
||||||
root = etree.fromstring(data)
|
root = etree.fromstring(data)
|
||||||
|
|
||||||
all_rows = []
|
all_rows = []
|
||||||
|
@ -51,14 +50,14 @@ class ECB:
|
||||||
date = day.attrib["time"]
|
date = day.attrib["time"]
|
||||||
# TODO what if it's not found for that day?
|
# TODO what if it's not found for that day?
|
||||||
# (some quotes aren't in the earliest data)
|
# (some quotes aren't in the earliest data)
|
||||||
for row in day.cssselect(f"[currency='{quote}']"):
|
for row in day.cssselect(f"[currency='{series.quote}']"):
|
||||||
rate = Decimal(row.attrib["rate"])
|
rate = Decimal(row.attrib["rate"])
|
||||||
all_rows.insert(0, (date, rate))
|
all_rows.insert(0, (date, rate))
|
||||||
selected = [
|
selected = [
|
||||||
Price(base, quote, d, r) for d, r in all_rows if d >= start and d <= end
|
Price(d, r) for d, r in all_rows if d >= series.start and d <= series.end
|
||||||
]
|
]
|
||||||
|
|
||||||
return selected
|
return dataclasses.replace(series, prices=selected)
|
||||||
|
|
||||||
def _raw_data(self, more_than_90_days=False):
|
def _raw_data(self, more_than_90_days=False):
|
||||||
url_base = "https://www.ecb.europa.eu/stats/eurofxref"
|
url_base = "https://www.ecb.europa.eu/stats/eurofxref"
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue