from __future__ import annotations import logging from datetime import datetime from threading import Event, Thread from typing import Any import pytz from requests import Response from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout from core.constants import HTTP_HEADERS from core.url_data_cache import URLDataCache from providers.staticdata.static_data_provider import StaticDataProvider logger = logging.getLogger(__name__) class FileDownloadStaticDataProvider(StaticDataProvider): """Generic static reference data provider class for providers that fetch their data from the web by downloading a file.""" def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None: """Set up the provider, note poll_interval is in *days*.""" super().__init__(name, provider_config) self._url = url self._poll_interval = poll_interval self._thread: Thread | None = None self._stop_event = Event() self._url_data_cache = URLDataCache(f"staticdata_{name}") def start(self) -> None: # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between # subsequent polls, so start() returns immediately and the application can continue starting. logger.info(f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.") self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}", daemon=True) self._thread.start() def stop(self) -> None: self._stop_event.set() if self._thread: self._thread.join(timeout=12) if self._thread.is_alive(): logger.warning(f"{self.name} static data worker thread did not exit on time and will be killed.") def _run(self) -> None: while True: self._poll() if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24): break def _poll(self) -> None: try: # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for # caching, but just so continual restarts of the software during testing don't hammer the servers. logger.debug(f"Downloading {self.name} static reference data...") http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS) # Check response code was good if http_response.ok: # Pass off to the subclass for processing ok = self._handle_http_response(http_response) if ok: self.status = "OK" self.last_update_time = datetime.now(pytz.UTC) logger.info(f"Updated static reference data for {self.name}") else: self.status = "Error" logger.warning( f"HTTP {http_response.status_code} when downloading static reference data for {self.name}." ) except ConnectionError: self.status = "Error" logger.warning(f"Connection error when downloading static reference data for {self.name}.") except (ConnectTimeout, ReadTimeout): self.status = "Error" logger.warning(f"Timeout when downloading static reference data for {self.name}.") except Exception: self.status = "Error" logger.exception(f"Exception in HTTP static reference data provider ({self.name})") self._stop_event.wait(timeout=1) def _handle_http_response(self, http_response: Response) -> bool: """Handle an HTTP response returned by the server and load the data from it. Return true if successful, false otherwise.""" raise NotImplementedError("Subclasses must implement this method")