Refactor of caching & data storage part 4 #118

sigrefdataproviders/file_download_sig_ref_data_provider.py

@@ -3,11 +3,11 @@ from datetime import datetime
 from threading import Thread, Event
 
 import pytz
-import requests
 from requests import ReadTimeout
 from requests.exceptions import ConnectionError, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
+from core.url_data_cache import URL_DATA_CACHE
 from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
 
 
@@ -15,6 +15,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
     """Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
 
     def __init__(self, sig_name, provider_config, url, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
         super().__init__(sig_name, provider_config)
         self._url = url
         self._poll_interval = poll_interval
@@ -25,7 +26,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info(
-            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " seconds.")
+            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " days.")
         self._thread = Thread(target=self._run, daemon=True)
         self._thread.start()
 
@@ -35,14 +36,15 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
     def _run(self):
         while True:
             self._poll()
-            if self._stop_event.wait(timeout=self._poll_interval):
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
                 break
 
     def _poll(self):
         try:
-            # Request data from API
+            # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
             logging.debug("Downloading " + self.sig_name + " SIG ref data...")
-            http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            http_response = URL_DATA_CACHE.get(self._url, headers=HTTP_HEADERS)
             # Check response code was good
             if http_response.ok:
                 # Pass off to the subclass for processing