fz0x1/pricehist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Set User-Agent when talking to Yahoo.

Browse source
This commit is contained in:
Chris Berkhout 2021-07-14 13:05:20 +02:00
parent c0c7e546a3
commit fcce842d82
1 changed files with 7 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/pricehist/sources/yahoo.py
View file

@ -8,6 +8,7 @@ from decimal import Decimal
import requests import requests
from pricehist.price import Price from pricehist.price import Price
from pricehist import __version__
from .basesource import BaseSource from .basesource import BaseSource
@ -62,6 +63,7 @@ class Yahoo(BaseSource):
return [] return []
def fetch(self, series): def fetch(self, series):
# TODO fail if quote isn't empty - yahoo symbols don't have a slash
spark, history = self._data(series) spark, history = self._data(series)
output_quote = spark["spark"]["result"][0]["response"][0]["meta"]["currency"] output_quote = spark["spark"]["result"][0]["response"][0]["meta"]["currency"]
@ -84,6 +86,7 @@ class Yahoo(BaseSource):
def _data(self, series) -> (dict, csv.DictReader): def _data(self, series) -> (dict, csv.DictReader):
base_url = "https://query1.finance.yahoo.com/v7/finance" base_url = "https://query1.finance.yahoo.com/v7/finance"
headers = {"User-Agent": f"pricehist/{__version__}"}
spark_url = f"{base_url}/spark" spark_url = f"{base_url}/spark"
spark_params = { spark_params = {
@ -94,7 +97,9 @@ class Yahoo(BaseSource):
"includeTimestamps": "false", "includeTimestamps": "false",
"includePrePost": "false", "includePrePost": "false",
} }
spark_response = self.log_curl(requests.get(spark_url, params=spark_params)) spark_response = self.log_curl(
requests.get(spark_url, params=spark_params, headers=headers)
)
spark = json.loads(spark_response.content) spark = json.loads(spark_response.content)
start_ts = int(datetime.strptime(series.start, "%Y-%m-%d").timestamp()) start_ts = int(datetime.strptime(series.start, "%Y-%m-%d").timestamp())
@ -111,7 +116,7 @@ class Yahoo(BaseSource):
"includeAdjustedClose": "true", "includeAdjustedClose": "true",
} }
history_response = self.log_curl( history_response = self.log_curl(
requests.get(history_url, params=history_params) requests.get(history_url, params=history_params, headers=headers)
) )
history_lines = history_response.content.decode("utf-8").splitlines() history_lines = history_response.content.decode("utf-8").splitlines()
history_lines[0] = history_lines[0].lower().replace(" ", "") history_lines[0] = history_lines[0].lower().replace(" ", "")