fz0x1/pricehist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

After running Black.

Browse source
This commit is contained in:
Chris Berkhout 2021-04-20 19:54:01 +02:00
parent 4f9353013e
commit d2416ebb0c
13 changed files with 203 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/pricehist/__init__.py
View file

@ -1 +1 @@
__version__ = '0.1.0'
__version__ = "0.1.0"

118
src/pricehist/cli.py
View file

@ -5,33 +5,37 @@ from datetime import datetime, timedelta
from pricehist import sources
from pricehist import outputs
def cli(args=None):
parser = build_parser()
args = parser.parse_args()
if (args.command == 'sources'):
if args.command == "sources":
cmd_sources(args)
elif (args.command == 'source'):
elif args.command == "source":
cmd_source(args)
elif (args.command == 'fetch'):
elif args.command == "fetch":
cmd_fetch(args)
else:
parser.print_help()
def cmd_sources(args):
width = max([len(identifier) for identifier in sources.by_id.keys()])
for identifier, source in sources.by_id.items():
print(f'{identifier.ljust(width)} {source.name()}')
print(f"{identifier.ljust(width)} {source.name()}")
def cmd_source(args):
source = sources.by_id[args.identifier]
print(f'ID : {source.id()}')
print(f'Name : {source.name()}')
print(f'Description : {source.description()}')
print(f'URL : {source.source_url()}')
print(f"ID : {source.id()}")
print(f"Name : {source.name()}")
print(f"Description : {source.description()}")
print(f"URL : {source.source_url()}")
print(f'Bases : {", ".join(source.bases())}')
print(f'Quotes : {", ".join(source.quotes())}')
def cmd_fetch(args):
source = sources.by_id[args.source]()
start = args.start or args.after
@ -39,9 +43,10 @@ def cmd_fetch(args):
prices = source.fetch(args.pair, args.start, args.end)
print(output.format(prices))
def build_parser():
def valid_date(s):
if s == 'today':
if s == "today":
return today()
try:
datetime.strptime(s, "%Y-%m-%d")
@ -51,40 +56,83 @@ def build_parser():
raise argparse.ArgumentTypeError(msg)
def following_valid_date(s):
return str(datetime.strptime(valid_date(s), "%Y-%m-%d").date() + timedelta(days=1))
return str(
datetime.strptime(valid_date(s), "%Y-%m-%d").date() + timedelta(days=1)
)
def today():
return str(datetime.now().date())
parser = argparse.ArgumentParser(description='Fetch historical price data')
parser = argparse.ArgumentParser(description="Fetch historical price data")
subparsers = parser.add_subparsers(title='commands', dest='command')
subparsers = parser.add_subparsers(title="commands", dest="command")
sources_parser = subparsers.add_parser('sources', help='list sources')
sources_parser = subparsers.add_parser("sources", help="list sources")
source_parser = subparsers.add_parser('source', help='show source details')
source_parser.add_argument('identifier', metavar='ID', type=str,
choices=sources.by_id.keys(),
help='the source identifier')
source_parser = subparsers.add_parser("source", help="show source details")
source_parser.add_argument(
"identifier",
metavar="ID",
type=str,
choices=sources.by_id.keys(),
help="the source identifier",
)
fetch_parser = subparsers.add_parser('fetch', help='fetch prices',
usage='pricehist fetch ID [-h] -p PAIR (-s DATE | -sx DATE) [-e DATE] [-o FMT]')
fetch_parser.add_argument('source', metavar='ID', type=str,
choices=sources.by_id.keys(),
help='the source identifier')
fetch_parser.add_argument('-p', '--pair', dest='pair', type=str, required=True,
help='pair, usually BASE/QUOTE, e.g. BTC/USD')
fetch_parser = subparsers.add_parser(
"fetch",
help="fetch prices",
usage="pricehist fetch ID [-h] -p PAIR (-s DATE | -sx DATE) [-e DATE] [-o FMT]",
)
fetch_parser.add_argument(
"source",
metavar="ID",
type=str,
choices=sources.by_id.keys(),
help="the source identifier",
)
fetch_parser.add_argument(
"-p",
"--pair",
dest="pair",
type=str,
required=True,
help="pair, usually BASE/QUOTE, e.g. BTC/USD",
)
fetch_start_group = fetch_parser.add_mutually_exclusive_group(required=True)
fetch_start_group.add_argument('-s', '--start', dest='start', metavar='DATE', type=valid_date,
help='start date, inclusive')
fetch_start_group.add_argument('-sx', '--startx', dest='start', metavar='DATE', type=following_valid_date,
help='start date, exclusive')
fetch_parser.add_argument('-e', '--end', dest='end', metavar='DATE', type=valid_date,
default=today(),
help='end date, inclusive (default: today)')
fetch_parser.add_argument('-o', '--output', dest='output', metavar='FMT', type=str,
choices=outputs.by_type.keys(),
default=outputs.default,
help=f'output format (default: {outputs.default})')
fetch_start_group.add_argument(
"-s",
"--start",
dest="start",
metavar="DATE",
type=valid_date,
help="start date, inclusive",
)
fetch_start_group.add_argument(
"-sx",
"--startx",
dest="start",
metavar="DATE",
type=following_valid_date,
help="start date, exclusive",
)
fetch_parser.add_argument(
"-e",
"--end",
dest="end",
metavar="DATE",
type=valid_date,
default=today(),
help="end date, inclusive (default: today)",
)
fetch_parser.add_argument(
"-o",
"--output",
dest="output",
metavar="FMT",
type=str,
choices=outputs.by_type.keys(),
default=outputs.default,
help=f"output format (default: {outputs.default})",
)
return parser

10
src/pricehist/outputs/__init__.py
View file

@ -3,11 +3,11 @@ from .csv import CSV
from .gnucashsql import GnuCashSQL
from .ledger import Ledger
default = 'ledger'
default = "ledger"
by_type = {
'beancount': Beancount,
'csv': CSV,
'gnucash-sql': GnuCashSQL,
'ledger': Ledger
"beancount": Beancount,
"csv": CSV,
"gnucash-sql": GnuCashSQL,
"ledger": Ledger,
}

10
src/pricehist/outputs/beancount.py
View file

@ -1,12 +1,14 @@
class Beancount():
class Beancount:
def format(self, prices):
lines = []
for price in prices:
date = str(price.date).translate(str.maketrans('-','/'))
lines.append(f"{price.date} price {price.base} {price.amount} {price.quote}")
date = str(price.date).translate(str.maketrans("-", "/"))
lines.append(
f"{price.date} price {price.base} {price.amount} {price.quote}"
)
return "\n".join(lines)
# https://beancount.github.io/docs/fetching_prices_in_beancount.html
# https://beancount.github.io/docs/beancount_language_syntax.html#commodities-currencies
# https://beancount.github.io/docs/beancount_language_syntax.html#comments

7
src/pricehist/outputs/csv.py
View file

@ -1,9 +1,8 @@
class CSV():
class CSV:
def format(self, prices):
lines = ["date,base,quote,amount"]
for price in prices:
date = str(price.date).translate(str.maketrans('-','/'))
line = ','.join([price.date, price.base, price.quote, str(price.amount)])
date = str(price.date).translate(str.maketrans("-", "/"))
line = ",".join([price.date, price.base, price.quote, str(price.amount)])
lines.append(line)
return "\n".join(lines)

22
src/pricehist/outputs/gnucashsql.py
View file

@ -3,25 +3,29 @@ import hashlib
from pricehist import __version__
class GnuCashSQL():
class GnuCashSQL:
def format(self, prices):
source = 'pricehist'
typ = 'unknown'
source = "pricehist"
typ = "unknown"
values = []
for price in prices:
date = f'{price.date} 00:00:00'
date = f"{price.date} 00:00:00"
m = hashlib.sha256()
m.update("".join([date, price.base, price.quote, source, typ, str(price.amount)]).encode('utf-8'))
m.update(
"".join(
[date, price.base, price.quote, source, typ, str(price.amount)]
).encode("utf-8")
)
guid = m.hexdigest()[0:32]
value_num = str(price.amount).replace('.', '')
value_denom = 10 ** len(f'{price.amount}.'.split('.')[1])
value_num = str(price.amount).replace(".", "")
value_denom = 10 ** len(f"{price.amount}.".split(".")[1])
v = f"('{guid}', '{date}', '{price.base}', '{price.quote}', '{source}', '{typ}', {value_num}, {value_denom})"
values.append(v)
comma_newline = ",\n"
sql = f'''\
sql = f"""\
-- Created by pricehist v{__version__} at {datetime.utcnow().isoformat()}Z
BEGIN;
@ -66,6 +70,6 @@ SELECT * FROM summary;
SELECT 'final' AS status, p.* FROM prices p WHERE p.guid IN (SELECT guid FROM new_prices) ORDER BY p.date;
COMMIT;
'''
"""
return sql

6
src/pricehist/outputs/ledger.py
View file

@ -1,13 +1,11 @@
class Ledger():
class Ledger:
def format(self, prices):
lines = []
for price in prices:
date = str(price.date).translate(str.maketrans('-','/'))
date = str(price.date).translate(str.maketrans("-", "/"))
lines.append(f"P {date} 00:00:00 {price.base} {price.amount} {price.quote}")
return "\n".join(lines)
# TODO support additional details of the format:
# https://www.ledger-cli.org/3.0/doc/ledger3.html#Commodities-and-Currencies
# https://www.ledger-cli.org/3.0/doc/ledger3.html#Commoditized-Amounts

2
src/pricehist/price.py
View file

@ -1,3 +1,3 @@
from collections import namedtuple
Price = namedtuple('Price', ['base', 'quote', 'date', 'amount'])
Price = namedtuple("Price", ["base", "quote", "date", "amount"])

6
src/pricehist/sources/__init__.py
View file

@ -2,8 +2,4 @@ from .coindesk import CoinDesk
from .coinmarketcap import CoinMarketCap
from .ecb import ECB
by_id = {
CoinDesk.id(): CoinDesk,
CoinMarketCap.id(): CoinMarketCap,
ECB.id(): ECB
}
by_id = {CoinDesk.id(): CoinDesk, CoinMarketCap.id(): CoinMarketCap, ECB.id(): ECB}

32
src/pricehist/sources/coindesk.py
View file

@ -4,52 +4,54 @@ import requests
from pricehist.price import Price
class CoinDesk():
class CoinDesk:
@staticmethod
def id():
return 'coindesk'
return "coindesk"
@staticmethod
def name():
return 'CoinDesk Bitcoin Price Index'
return "CoinDesk Bitcoin Price Index"
@staticmethod
def description():
return 'An average of bitcoin prices across leading global exchanges. Powered by CoinDesk, https://www.coindesk.com/price/bitcoin'
return "An average of bitcoin prices across leading global exchanges. Powered by CoinDesk, https://www.coindesk.com/price/bitcoin"
@staticmethod
def source_url():
return 'https://www.coindesk.com/coindesk-api'
return "https://www.coindesk.com/coindesk-api"
@staticmethod
def bases():
return ['BTC']
return ["BTC"]
@staticmethod
def quotes():
url = 'https://api.coindesk.com/v1/bpi/supported-currencies.json'
url = "https://api.coindesk.com/v1/bpi/supported-currencies.json"
response = requests.get(url)
data = json.loads(response.content)
symbols = sorted([item['currency'] for item in data])
symbols = sorted([item["currency"] for item in data])
return symbols
def fetch(self, pair, start, end):
base, quote = pair.split('/')
base, quote = pair.split("/")
if base not in self.bases():
exit(f'Invalid base {base}')
exit(f"Invalid base {base}")
if quote not in self.quotes():
exit(f'Invalid quote {quote}')
exit(f"Invalid quote {quote}")
min_start = '2010-07-17'
min_start = "2010-07-17"
if start < min_start:
exit(f'start {start} too early. The CoinDesk BPI only covers data from {min_start} onwards.')
exit(
f"start {start} too early. The CoinDesk BPI only covers data from {min_start} onwards."
)
url = f'https://api.coindesk.com/v1/bpi/historical/close.json?currency={quote}&start={start}&end={end}'
url = f"https://api.coindesk.com/v1/bpi/historical/close.json?currency={quote}&start={start}&end={end}"
response = requests.get(url)
data = json.loads(response.content)
prices = []
for (d, v) in data['bpi'].items():
for (d, v) in data["bpi"].items():
prices.append(Price(base, quote, d, Decimal(str(v))))
return prices

29
src/pricehist/sources/coinmarketcap.py
View file

@ -6,15 +6,15 @@ from xml.etree import ElementTree
from pricehist.price import Price
class CoinMarketCap():
class CoinMarketCap:
@staticmethod
def id():
return 'coinmarketcap'
return "coinmarketcap"
@staticmethod
def name():
return 'CoinMarketCap'
return "CoinMarketCap"
@staticmethod
def description():
@ -22,7 +22,7 @@ class CoinMarketCap():
@staticmethod
def source_url():
return 'https://coinmarketcap.com/'
return "https://coinmarketcap.com/"
# # currency metadata - these may max out at 5k items (crypto data is currently 4720 items)
# curl 'https://web-api.coinmarketcap.com/v1/fiat/map?include_metals=true' | jq . | tee fiat-map.json
@ -37,24 +37,25 @@ class CoinMarketCap():
return []
def fetch(self, pair, start, end):
base, quote = pair.split('/')
base, quote = pair.split("/")
url = f'https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical'
url = f"https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical"
params = {
'symbol': base,
'convert': quote,
'time_start': int(datetime.strptime(start, '%Y-%m-%d').timestamp()),
'time_end': int(datetime.strptime(end, '%Y-%m-%d').timestamp()) + 24*60*60 # round up to include the last day
"symbol": base,
"convert": quote,
"time_start": int(datetime.strptime(start, "%Y-%m-%d").timestamp()),
"time_end": int(datetime.strptime(end, "%Y-%m-%d").timestamp())
+ 24 * 60 * 60, # round up to include the last day
}
response = requests.get(url, params=params)
data = json.loads(response.content)
prices = []
for item in data['data']['quotes']:
d = item['time_open'][0:10]
high = Decimal(str(item['quote'][quote]['high']))
low = Decimal(str(item['quote'][quote]['low']))
for item in data["data"]["quotes"]:
d = item["time_open"][0:10]
high = Decimal(str(item["quote"][quote]["high"]))
low = Decimal(str(item["quote"][quote]["low"]))
mid = sum([high, low]) / 2
prices.append(Price(base, quote, d, mid))

78
src/pricehist/sources/ecb.py
View file

@ -6,51 +6,81 @@ from xml.etree import ElementTree
from pricehist.price import Price
class ECB():
class ECB:
@staticmethod
def id():
return 'ecb'
return "ecb"
@staticmethod
def name():
return 'European Central Bank'
return "European Central Bank"
@staticmethod
def description():
return 'European Central Bank Euro foreign exchange reference rates'
return "European Central Bank Euro foreign exchange reference rates"
@staticmethod
def source_url():
return 'https://www.ecb.europa.eu/stats/exchange/eurofxref/html/index.en.html'
return "https://www.ecb.europa.eu/stats/exchange/eurofxref/html/index.en.html"
@staticmethod
def bases():
return ['EUR']
return ["EUR"]
@staticmethod
def quotes():
return ['AUD', 'BGN', 'BRL', 'CAD', 'CHF', 'CNY', 'CZK', 'DKK', 'GBP',
'HKD', 'HRK', 'HUF', 'IDR', 'ILS', 'INR', 'ISK', 'JPY', 'KRW',
'MXN', 'MYR', 'NOK', 'NZD', 'PHP', 'PLN', 'RON', 'RUB', 'SEK',
'SGD', 'THB', 'TRY', 'USD', 'ZAR']
return [
"AUD",
"BGN",
"BRL",
"CAD",
"CHF",
"CNY",
"CZK",
"DKK",
"GBP",
"HKD",
"HRK",
"HUF",
"IDR",
"ILS",
"INR",
"ISK",
"JPY",
"KRW",
"MXN",
"MYR",
"NOK",
"NZD",
"PHP",
"PLN",
"RON",
"RUB",
"SEK",
"SGD",
"THB",
"TRY",
"USD",
"ZAR",
]
def fetch(self, pair, start, end):
base, quote = pair.split('/')
base, quote = pair.split("/")
if base not in self.bases():
exit(f'Invalid base {base}')
exit(f"Invalid base {base}")
if quote not in self.quotes():
exit(f'Invalid quote {quote}')
exit(f"Invalid quote {quote}")
min_start = '1999-01-04'
min_start = "1999-01-04"
if start < min_start:
exit(f'start {start} too early. Minimum is {min_start}')
exit(f"start {start} too early. Minimum is {min_start}")
almost_90_days_ago = str(datetime.now().date() - timedelta(days=85))
if start > almost_90_days_ago:
source_url = 'https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml' # last 90 days
source_url = "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml" # last 90 days
else:
source_url = 'https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml' # since 1999
source_url = "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml" # since 1999
response = requests.get(source_url)
data = response.content
@ -58,16 +88,18 @@ class ECB():
# TODO consider changing from xml.etree to lxml
root = ElementTree.fromstring(data)
namespaces = {
'default': 'http://www.ecb.int/vocabulary/2002-08-01/eurofxref',
'gesmes': 'http://www.gesmes.org/xml/2002-08-01'
"default": "http://www.ecb.int/vocabulary/2002-08-01/eurofxref",
"gesmes": "http://www.gesmes.org/xml/2002-08-01",
}
all_rows = []
for day in root.find('default:Cube', namespaces):
date = day.attrib['time']
for day in root.find("default:Cube", namespaces):
date = day.attrib["time"]
rate_xpath = f"./*[@currency='{quote}']"
# TODO what if it's not found for that day? (some quotes aren't in the earliest data)
rate = Decimal(day.find(rate_xpath).attrib['rate'])
rate = Decimal(day.find(rate_xpath).attrib["rate"])
all_rows.insert(0, (date, rate))
selected = [ Price(base, quote, d, r) for d, r in all_rows if d >= start and d <= end ]
selected = [
Price(base, quote, d, r) for d, r in all_rows if d >= start and d <= end
]
return selected

2
tests/test_pricehist.py
View file

@ -2,4 +2,4 @@ from pricehist import __version__
def test_version():
assert __version__ == '0.1.0'
assert __version__ == "0.1.0"