import logging from datetime import datetime from threading import Thread, Event import pytz from requests import ReadTimeout from requests.exceptions import ConnectionError, ConnectTimeout from core.constants import HTTP_HEADERS from core.url_data_cache import URLDataCache from providers.callsigndata.callsign_data_provider import CallsignDataProvider class FileDownloadCallsignDataProvider(CallsignDataProvider): """Generic static reference data provider class for providers that fetch their data from the web by downloading a file.""" def __init__(self, name, provider_config, url, cache_file_path, poll_interval): """ Set up the provider, note poll_interval is in *days*.""" super().__init__(name, provider_config) self._url = url self._cache_file_path = cache_file_path self._poll_interval = poll_interval self._thread = None self._stop_event = Event() self._url_data_cache = URLDataCache("callsigndata_" + name) def start(self): # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between # subsequent polls, so start() returns immediately and the application can continue starting. logging.info( "Set up query of " + self.name + " callsign reference data every " + str(self._poll_interval) + " days.") self._thread = Thread(target=self._run, daemon=True) self._thread.start() def stop(self): self._stop_event.set() def _run(self): while True: self._poll() if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24): break def _poll(self): try: # Request the file. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for # caching, but just so continual restarts of the software during testing don't hammer the servers. logging.debug("Downloading " + self.name + " callsign reference data...") http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS) # Check response code was good if http_response.ok: # Save the data to a local file with open(self._cache_file_path, "wb") as f: f.write(http_response.content) f.flush() # Pass off to the subclass for processing ok = self._handle_file(self._cache_file_path) if ok: self.status = "OK" self.last_update_time = datetime.now(pytz.UTC) logging.info("Updated callsign reference data from " + self.name) else: self.status = "Error" logging.warning(f"Error updating callsign reference data from {self.name}.") else: self.status = "Error" logging.warning(f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}.") except ConnectionError: self.status = "Error" logging.warning(f"Connection error when downloading callsign reference data from {self.name}.") except (ConnectTimeout, ReadTimeout): self.status = "Error" logging.warning(f"Timeout when downloading callsign reference data from {self.name}.") except Exception: self.status = "Error" logging.exception("Exception in callsign reference data provider (" + self.name + ")") self._stop_event.wait(timeout=1) def _handle_file(self, path): """Handle an updated file downloaded from the server. Return true if successful, false otherwise.""" raise NotImplementedError("Subclasses must implement this method")