Refactor of caching & data storage part 9 #118

providers/staticdata/file_download_static_data_provider.py

@@ -0,0 +1,77 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from providers.staticdata.static_data_provider import StaticDataProvider
+
+
+class FileDownloadStaticDataProvider(StaticDataProvider):
+    """Generic static reference data provider class for providers that fetch their data from the web by downloading a
+     file."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
+        super().__init__(name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+        self._url_data_cache = URLDataCache("staticdata_" + name)
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info(
+            "Set up query of " + self.name + " static reference data every " + str(self._poll_interval) + " days.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
+                break
+
+    def _poll(self):
+        try:
+            # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
+            logging.debug("Downloading " + self.name + " static reference data...")
+            http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                ok = self._handle_http_response(http_response)
+                if ok:
+                    self.status = "OK"
+                    self.last_update_time = datetime.now(pytz.UTC)
+                    logging.info("Updated static reference data for " + self.name)
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when downloading static reference data for {self.name}.")
+
+        except ConnectionError:
+            self.status = "Error"
+            logging.warning(f"Connection error when downloading static reference data for {self.name}.")
+        except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
+            logging.warning(f"Timeout when downloading static reference data for {self.name}.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP static reference data provider (" + self.name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _handle_http_response(self, http_response):
+        """Handle an HTTP response returned by the server and load the data from it. Return true if successful,
+         false otherwise."""
+
+        raise NotImplementedError("Subclasses must implement this method")