import logging from datetime import datetime from threading import Event, Thread import pytz from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout from core.constants import HTTP_HEADERS from core.url_data_cache import URLDataCache from providers.staticdata.static_data_provider import StaticDataProvider logger = logging.getLogger(__name__) class FileDownloadStaticDataProvider(StaticDataProvider): """Generic static reference data provider class for providers that fetch their data from the web by downloading a file.""" def __init__(self, name, provider_config, url, poll_interval): """Set up the provider, note poll_interval is in *days*.""" super().__init__(name, provider_config) self._url = url self._poll_interval = poll_interval self._thread = None self._stop_event = Event() self._url_data_cache = URLDataCache(f"staticdata_{name}") def start(self): # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between # subsequent polls, so start() returns immediately and the application can continue starting. logger.info(f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.") self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}", daemon=True) self._thread.start() def stop(self): self._stop_event.set() if self._thread: self._thread.join(timeout=12) if self._thread.is_alive(): logger.warning(f"{self.name} static data worker thread did not exit on time and will be killed.") def _run(self): while True: self._poll() if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24): break def _poll(self): try: # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for # caching, but just so continual restarts of the software during testing don't hammer the servers. logger.debug(f"Downloading {self.name} static reference data...") http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS) # Check response code was good if http_response.ok: # Pass off to the subclass for processing ok = self._handle_http_response(http_response) if ok: self.status = "OK" self.last_update_time = datetime.now(pytz.UTC) logger.info(f"Updated static reference data for {self.name}") else: self.status = "Error" logger.warning( f"HTTP {http_response.status_code} when downloading static reference data for {self.name}." ) except ConnectionError: self.status = "Error" logger.warning(f"Connection error when downloading static reference data for {self.name}.") except (ConnectTimeout, ReadTimeout): self.status = "Error" logger.warning(f"Timeout when downloading static reference data for {self.name}.") except Exception: self.status = "Error" logger.exception(f"Exception in HTTP static reference data provider ({self.name})") self._stop_event.wait(timeout=1) def _handle_http_response(self, http_response): """Handle an HTTP response returned by the server and load the data from it. Return true if successful, false otherwise.""" raise NotImplementedError("Subclasses must implement this method")