import logging from datetime import datetime from threading import Thread, Event import pytz from requests import ReadTimeout from requests.exceptions import ConnectionError, ConnectTimeout from core.constants import HTTP_HEADERS from core.url_data_cache import URLDataCache from staticdataproviders.static_data_provider import StaticDataProvider class FileDownloadStaticDataProvider(StaticDataProvider): """Generic static reference data provider class for providers that fetch their data from the web by downloading a file.""" def __init__(self, name, provider_config, url, poll_interval): """ Set up the provider, note poll_interval is in *days*.""" super().__init__(name, provider_config) self._url = url self._poll_interval = poll_interval self._thread = None self._stop_event = Event() self._url_data_cache = URLDataCache("staticdata_" + name) def start(self): # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between # subsequent polls, so start() returns immediately and the application can continue starting. logging.info( "Set up query of " + self.name + " static reference data every " + str(self._poll_interval) + " days.") self._thread = Thread(target=self._run, daemon=True) self._thread.start() def stop(self): self._stop_event.set() def _run(self): while True: self._poll() if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24): break def _poll(self): try: # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for # caching, but just so continual restarts of the software during testing don't hammer the servers. logging.debug("Downloading " + self.name + " static reference data...") http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS) # Check response code was good if http_response.ok: # Pass off to the subclass for processing ok = self._handle_http_response(http_response) if ok: self.status = "OK" self.last_update_time = datetime.now(pytz.UTC) logging.info("Updated static reference data for " + self.name) else: self.status = "Error" logging.warning(f"HTTP {http_response.status_code} when downloading static reference data for {self.name}.") except ConnectionError: self.status = "Error" logging.warning(f"Connection error when downloading static reference data for {self.name}.") except (ConnectTimeout, ReadTimeout): self.status = "Error" logging.warning(f"Timeout when downloading static reference data for {self.name}.") except Exception: self.status = "Error" logging.exception("Exception in HTTP static reference data provider (" + self.name + ")") self._stop_event.wait(timeout=1) def _handle_http_response(self, http_response): """Handle an HTTP response returned by the server and load the data from it. Return true if successful, false otherwise.""" raise NotImplementedError("Subclasses must implement this method")