From 2787c212d23e73db2973843a6f367df057ebcc71 Mon Sep 17 00:00:00 2001
From: Chris Berkhout <chrisberkhout@gmail.com>
Date: Mon, 27 Dec 2021 11:25:55 +1100
Subject: [PATCH] Fix AlphaVantage to handle adjusted endpoint being premium.

---
 src/pricehist/exceptions.py                   |  4 +-
 src/pricehist/sources/alphavantage.py         | 19 ++++-
 tests/live.sh                                 | 12 +--
 tests/pricehist/sources/test_alphavantage.py  | 39 ++++++++-
 .../test_alphavantage/ibm-partial-adj.json    | 81 +++++++++++++++++++
 .../test_alphavantage/ibm-partial.json        | 42 ++--------
 6 files changed, 149 insertions(+), 48 deletions(-)
 create mode 100644 tests/pricehist/sources/test_alphavantage/ibm-partial-adj.json

diff --git a/src/pricehist/exceptions.py b/src/pricehist/exceptions.py
index 5ac7aa7..e207537 100644
--- a/src/pricehist/exceptions.py
+++ b/src/pricehist/exceptions.py
@@ -52,7 +52,7 @@ class InvalidType(SourceError, ValueError):
 class CredentialsError(SourceError):
     """Access credentials are unavailable or invalid."""
 
-    def __init__(self, keys, source):
+    def __init__(self, keys, source, msg=""):
         self.keys = keys
         self.source = source
         message = (
@@ -61,6 +61,8 @@ class CredentialsError(SourceError):
             f"correctly. Run 'pricehist source {source.id()}' for more "
             f"information about credentials."
         )
+        if msg:
+            message += f" {msg}"
         super(CredentialsError, self).__init__(message)
 
 
diff --git a/src/pricehist/sources/alphavantage.py b/src/pricehist/sources/alphavantage.py
index 75cda5d..84df4d9 100644
--- a/src/pricehist/sources/alphavantage.py
+++ b/src/pricehist/sources/alphavantage.py
@@ -51,7 +51,8 @@ class AlphaVantage(BaseSource):
             "will list all digital and physical currency symbols.\n"
             "The PAIR for stocks is the stock symbol only. The quote currency "
             f"will be determined automatically. {self._stock_symbols_message()}\n"
-            "The price type 'adjclose' is only available for stocks.\n"
+            "The price type 'adjclose' is only available for stocks, and "
+            "requires an access key for which premium endpoints are unlocked.\n"
             "Beware that digital currencies quoted in non-USD currencies may "
             "be converted from USD data at one recent exchange rate rather "
             "than using historical rates.\n"
@@ -186,8 +187,13 @@ class AlphaVantage(BaseSource):
     def _stock_data(self, series):
         output_quote = self._stock_currency(series.base) or "UNKNOWN"
 
+        if series.type == "adjclose":
+            function = "TIME_SERIES_DAILY_ADJUSTED"
+        else:
+            function = "TIME_SERIES_DAILY"
+
         params = {
-            "function": "TIME_SERIES_DAILY_ADJUSTED",
+            "function": function,
             "symbol": series.base,
             "outputsize": self._outputsize(series.start),
             "apikey": self._apikey(),
@@ -225,7 +231,8 @@ class AlphaVantage(BaseSource):
                     "high": entries["2. high"],
                     "low": entries["3. low"],
                     "close": entries["4. close"],
-                    "adjclose": entries["5. adjusted close"],
+                    "adjclose": "5. adjusted close" in entries
+                    and entries["5. adjusted close"],
                 }
                 for day, entries in reversed(data["Time Series (Daily)"].items())
             }
@@ -332,6 +339,12 @@ class AlphaVantage(BaseSource):
         if type(data) == dict:
             if "Note" in data and "call frequency" in data["Note"]:
                 raise exceptions.RateLimit(data["Note"])
+            if (
+                "Information" in data
+                and "ways to unlock premium" in data["Information"]
+            ):
+                msg = "You were denied access to a premium endpoint."
+                raise exceptions.CredentialsError([self.API_KEY_NAME], self, msg)
             if "Error Message" in data and "apikey " in data["Error Message"]:
                 raise exceptions.CredentialsError([self.API_KEY_NAME], self)
 
diff --git a/tests/live.sh b/tests/live.sh
index 543d634..33e66c3 100755
--- a/tests/live.sh
+++ b/tests/live.sh
@@ -47,11 +47,11 @@ name="Alpha Vantage stocks"
 cmd="pricehist fetch alphavantage TSLA -s 2021-01-04 -e 2021-01-08"
 read -r -d '' expected <<END
 date,base,quote,amount,source,type
-2021-01-04,TSLA,USD,729.77,alphavantage,close
-2021-01-05,TSLA,USD,735.11,alphavantage,close
-2021-01-06,TSLA,USD,755.98,alphavantage,close
-2021-01-07,TSLA,USD,816.04,alphavantage,close
-2021-01-08,TSLA,USD,880.02,alphavantage,close
+2021-01-04,TSLA,USD,729.7700,alphavantage,close
+2021-01-05,TSLA,USD,735.1100,alphavantage,close
+2021-01-06,TSLA,USD,755.9800,alphavantage,close
+2021-01-07,TSLA,USD,816.0400,alphavantage,close
+2021-01-08,TSLA,USD,880.0200,alphavantage,close
 END
 run_test "$name" "$cmd" "$expected"
 
@@ -64,7 +64,7 @@ date,base,quote,amount,source,type
 2021-01-05,AUD,EUR,0.63086,alphavantage,close
 2021-01-06,AUD,EUR,0.63306,alphavantage,close
 2021-01-07,AUD,EUR,0.63284,alphavantage,close
-2021-01-08,AUD,EUR,0.63360,alphavantage,close
+2021-01-08,AUD,EUR,0.63530,alphavantage,close
 END
 run_test "$name" "$cmd" "$expected"
 
diff --git a/tests/pricehist/sources/test_alphavantage.py b/tests/pricehist/sources/test_alphavantage.py
index 334bbd4..615f4fb 100644
--- a/tests/pricehist/sources/test_alphavantage.py
+++ b/tests/pricehist/sources/test_alphavantage.py
@@ -48,6 +48,9 @@ search_url = re.compile(
     r"https://www\.alphavantage\.co/query\?function=SYMBOL_SEARCH.*"
 )
 stock_url = re.compile(
+    r"https://www\.alphavantage\.co/query\?function=TIME_SERIES_DAILY&.*"
+)
+adj_stock_url = re.compile(
     r"https://www\.alphavantage\.co/query\?function=TIME_SERIES_DAILY_ADJUSTED.*"
 )
 physical_url = re.compile(r"https://www\.alphavantage\.co/query\?function=FX_DAILY.*")
@@ -64,6 +67,18 @@ rate_limit_json = (
     '" }'
 )
 
+premium_json = (
+    '{ "Information": "Thank you for using Alpha Vantage! This is a premium '
+    "endpoint and there are multiple ways to unlock premium endpoints: (1) "
+    "become a holder of Alpha Vantage Coin (AVC), an Ethereum-based "
+    "cryptocurrency that provides various utility & governance functions "
+    "within the Alpha Vantage ecosystem (AVC mining guide: "
+    "https://www.alphatournament.com/avc_mining_guide/) to unlock all "
+    "premium endpoints, (2) subscribe to any of the premium plans at "
+    "https://www.alphavantage.co/premium/ to instantly unlock all premium "
+    'endpoints" }'
+)
+
 
 @pytest.fixture
 def physical_list_ok(requests_mock):
@@ -99,6 +114,13 @@ def ibm_ok(requests_mock):
     yield requests_mock
 
 
+@pytest.fixture
+def ibm_adj_ok(requests_mock):
+    json = (Path(os.path.splitext(__file__)[0]) / "ibm-partial-adj.json").read_text()
+    requests_mock.add(responses.GET, adj_stock_url, body=json, status=200)
+    yield requests_mock
+
+
 @pytest.fixture
 def euraud_ok(requests_mock):
     json = (Path(os.path.splitext(__file__)[0]) / "eur-aud-partial.json").read_text()
@@ -282,7 +304,7 @@ def test_fetch_stock_known(src, type, search_ok, ibm_ok):
     stock_req = ibm_ok.calls[1].request
     assert search_req.params["function"] == "SYMBOL_SEARCH"
     assert search_req.params["keywords"] == "IBM"
-    assert stock_req.params["function"] == "TIME_SERIES_DAILY_ADJUSTED"
+    assert stock_req.params["function"] == "TIME_SERIES_DAILY"
     assert stock_req.params["symbol"] == "IBM"
     assert stock_req.params["outputsize"] == "full"
     assert (series.base, series.quote) == ("IBM", "USD")
@@ -315,16 +337,19 @@ def test_fetch_stock_types_all_available(src, search_ok, ibm_ok):
     opn = src.fetch(Series("IBM", "", "open", "2021-01-04", "2021-01-08"))
     hgh = src.fetch(Series("IBM", "", "high", "2021-01-04", "2021-01-08"))
     low = src.fetch(Series("IBM", "", "low", "2021-01-04", "2021-01-08"))
-    adj = src.fetch(Series("IBM", "", "adjclose", "2021-01-04", "2021-01-08"))
     mid = src.fetch(Series("IBM", "", "mid", "2021-01-04", "2021-01-08"))
     assert cls.prices[0].amount == Decimal("123.94")
     assert opn.prices[0].amount == Decimal("125.85")
     assert hgh.prices[0].amount == Decimal("125.9174")
     assert low.prices[0].amount == Decimal("123.04")
-    assert adj.prices[0].amount == Decimal("120.943645029")
     assert mid.prices[0].amount == Decimal("124.4787")
 
 
+def test_fetch_stock_types_adj_available(src, search_ok, ibm_adj_ok):
+    adj = src.fetch(Series("IBM", "", "adjclose", "2021-01-04", "2021-01-08"))
+    assert adj.prices[0].amount == Decimal("120.943645029")
+
+
 def test_fetch_stock_type_mid_is_mean_of_low_and_high(src, search_ok, ibm_ok):
     hgh = src.fetch(Series("IBM", "", "high", "2021-01-04", "2021-01-08")).prices
     low = src.fetch(Series("IBM", "", "low", "2021-01-04", "2021-01-08")).prices
@@ -401,6 +426,14 @@ def test_fetch_stock_rate_limit(src, type, search_ok, requests_mock):
     assert "rate limit" in str(e.value)
 
 
+# TODO
+def test_fetch_stock_premium(src, search_ok, requests_mock):
+    requests_mock.add(responses.GET, adj_stock_url, body=premium_json)
+    with pytest.raises(exceptions.CredentialsError) as e:
+        src.fetch(Series("IBM", "", "adjclose", "2021-01-04", "2021-01-08"))
+    assert "denied access to a premium endpoint" in str(e.value)
+
+
 def test_fetch_physical_known(src, type, physical_list_ok, euraud_ok):
     series = src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08"))
     req = euraud_ok.calls[1].request
diff --git a/tests/pricehist/sources/test_alphavantage/ibm-partial-adj.json b/tests/pricehist/sources/test_alphavantage/ibm-partial-adj.json
new file mode 100644
index 0000000..7329475
--- /dev/null
+++ b/tests/pricehist/sources/test_alphavantage/ibm-partial-adj.json
@@ -0,0 +1,81 @@
+{
+  "Meta Data": {
+    "1. Information": "Daily Time Series with Splits and Dividend Events",
+    "2. Symbol": "IBM",
+    "3. Last Refreshed": "2021-07-20",
+    "4. Output Size": "Full size",
+    "5. Time Zone": "US/Eastern"
+  },
+  "Time Series (Daily)": {
+    "2021-01-11": {
+      "1. open": "127.95",
+      "2. high": "129.675",
+      "3. low": "127.66",
+      "4. close": "128.58",
+      "5. adjusted close": "125.471469081",
+      "6. volume": "5602466",
+      "7. dividend amount": "0.0000",
+      "8. split coefficient": "1.0"
+    },
+    "2021-01-08": {
+      "1. open": "128.57",
+      "2. high": "129.32",
+      "3. low": "126.98",
+      "4. close": "128.53",
+      "5. adjusted close": "125.422677873",
+      "6. volume": "4676487",
+      "7. dividend amount": "0.0000",
+      "8. split coefficient": "1.0"
+    },
+    "2021-01-07": {
+      "1. open": "130.04",
+      "2. high": "130.46",
+      "3. low": "128.26",
+      "4. close": "128.99",
+      "5. adjusted close": "125.871556982",
+      "6. volume": "4507382",
+      "7. dividend amount": "0.0000",
+      "8. split coefficient": "1.0"
+    },
+    "2021-01-06": {
+      "1. open": "126.9",
+      "2. high": "131.88",
+      "3. low": "126.72",
+      "4. close": "129.29",
+      "5. adjusted close": "126.164304226",
+      "6. volume": "7956740",
+      "7. dividend amount": "0.0000",
+      "8. split coefficient": "1.0"
+    },
+    "2021-01-05": {
+      "1. open": "125.01",
+      "2. high": "126.68",
+      "3. low": "124.61",
+      "4. close": "126.14",
+      "5. adjusted close": "123.090458157",
+      "6. volume": "6114619",
+      "7. dividend amount": "0.0000",
+      "8. split coefficient": "1.0"
+    },
+    "2021-01-04": {
+      "1. open": "125.85",
+      "2. high": "125.9174",
+      "3. low": "123.04",
+      "4. close": "123.94",
+      "5. adjusted close": "120.943645029",
+      "6. volume": "5179161",
+      "7. dividend amount": "0.0000",
+      "8. split coefficient": "1.0"
+    },
+    "2020-12-31": {
+      "1. open": "124.22",
+      "2. high": "126.03",
+      "3. low": "123.99",
+      "4. close": "125.88",
+      "5. adjusted close": "122.836743878",
+      "6. volume": "3574696",
+      "7. dividend amount": "0.0000",
+      "8. split coefficient": "1.0"
+    }
+  }
+}
diff --git a/tests/pricehist/sources/test_alphavantage/ibm-partial.json b/tests/pricehist/sources/test_alphavantage/ibm-partial.json
index 7329475..8a1ce0b 100644
--- a/tests/pricehist/sources/test_alphavantage/ibm-partial.json
+++ b/tests/pricehist/sources/test_alphavantage/ibm-partial.json
@@ -11,71 +11,43 @@
       "1. open": "127.95",
       "2. high": "129.675",
       "3. low": "127.66",
-      "4. close": "128.58",
-      "5. adjusted close": "125.471469081",
-      "6. volume": "5602466",
-      "7. dividend amount": "0.0000",
-      "8. split coefficient": "1.0"
+      "4. close": "128.58"
     },
     "2021-01-08": {
       "1. open": "128.57",
       "2. high": "129.32",
       "3. low": "126.98",
-      "4. close": "128.53",
-      "5. adjusted close": "125.422677873",
-      "6. volume": "4676487",
-      "7. dividend amount": "0.0000",
-      "8. split coefficient": "1.0"
+      "4. close": "128.53"
     },
     "2021-01-07": {
       "1. open": "130.04",
       "2. high": "130.46",
       "3. low": "128.26",
-      "4. close": "128.99",
-      "5. adjusted close": "125.871556982",
-      "6. volume": "4507382",
-      "7. dividend amount": "0.0000",
-      "8. split coefficient": "1.0"
+      "4. close": "128.99"
     },
     "2021-01-06": {
       "1. open": "126.9",
       "2. high": "131.88",
       "3. low": "126.72",
-      "4. close": "129.29",
-      "5. adjusted close": "126.164304226",
-      "6. volume": "7956740",
-      "7. dividend amount": "0.0000",
-      "8. split coefficient": "1.0"
+      "4. close": "129.29"
     },
     "2021-01-05": {
       "1. open": "125.01",
       "2. high": "126.68",
       "3. low": "124.61",
-      "4. close": "126.14",
-      "5. adjusted close": "123.090458157",
-      "6. volume": "6114619",
-      "7. dividend amount": "0.0000",
-      "8. split coefficient": "1.0"
+      "4. close": "126.14"
     },
     "2021-01-04": {
       "1. open": "125.85",
       "2. high": "125.9174",
       "3. low": "123.04",
-      "4. close": "123.94",
-      "5. adjusted close": "120.943645029",
-      "6. volume": "5179161",
-      "7. dividend amount": "0.0000",
-      "8. split coefficient": "1.0"
+      "4. close": "123.94"
     },
     "2020-12-31": {
       "1. open": "124.22",
       "2. high": "126.03",
       "3. low": "123.99",
-      "4. close": "125.88",
-      "5. adjusted close": "122.836743878",
-      "6. volume": "3574696",
-      "7. dividend amount": "0.0000",
-      "8. split coefficient": "1.0"
+      "4. close": "125.88"
     }
   }
 }