Refactor of caching & data storage part 10 #118

providers/callsigndata/callsign_data_provider.py

@@ -0,0 +1,36 @@
+from datetime import datetime
+
+import pytz
+
+
+class CallsignDataProvider:
+    """Generic callsign reference data provider class. Subclasses of this set up the various mechanisms via which
+    Spothole can look up data for callsigns."""
+
+    def __init__(self, name, provider_config):
+        """Constructor"""
+
+        self.name = name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access remote resources, if
+        needed."""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def lookup(self, callsign, lookup_credentials):
+        """Looks up data for the provided callsign. Takes a LookupCredentials object, which provides any credentials
+        that have been provided by the user for this session (QRZ.com/HamQTH) to allow us to look up using those
+        services on the user's behalf. (Clublog is looked up using an API key owned by the server and provided in its
+        config file, so users need not provide their own.) Returns a Callsign object with as much data populated as
+        possible."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/callsigndata/clublogxml.py

@@ -0,0 +1,68 @@
+import gzip
+import logging
+
+from pyhamtools import LookupLib, Callinfo
+
+from data.callsign import Callsign
+from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
+
+
+class ClublogXML(FileDownloadCallsignDataProvider):
+    """Callsign data provider for ClubLog's Country File, which provides basic callsign to DXCC entity mapping."""
+
+    POLL_INTERVAL_DAYS = 30
+    DATA_URL = "https://cdn.clublog.org/cty.php"
+    CACHE_PATH_ZIPPED = "cache/cty.xml.gz"
+    CACHE_PATH_UNZIPPED = "cache/cty.xml"
+    _callinfo = None
+
+    def __init__(self, provider_config):
+        # API key required for this provider
+        self._api_key = provider_config.get("api-key", "")
+        if self._api_key == "":
+            provider_config["enabled"] = False
+            logging.warning(
+                "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
+
+        super().__init__("Clublog XML", provider_config, self.DATA_URL + "?api=" + self._api_key,
+                         self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS)
+
+    def _handle_file(self, path):
+        try:
+            # The download from Clublog is gzipped so we need to uncompress that and re-save as a separate file that
+            # the LookupLib can actually use.
+            with gzip.open(path, "rb") as uncompressed:
+                file_content = uncompressed.read()
+            assert isinstance(file_content, bytes)
+            with open(self.CACHE_PATH_UNZIPPED, "wb") as f:
+                f.write(file_content)
+                f.flush()
+
+            # Now load the data
+            lookuplib = LookupLib(lookuptype="clublogxml", filename=self.CACHE_PATH_UNZIPPED)
+            self._callinfo = Callinfo(lookuplib)
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading Clublog XML.", e, exc_info=True)
+            return False
+
+    def lookup(self, callsign, lookup_credentials):
+        # Lookup credentials are not required for this source.
+        # Lat/lon will only be centre of country or capital city from this source
+        ll = self._callinfo.get_lat_long(callsign)
+        lat = None
+        lon = None
+        if ll and "latitude" in ll and "longitude" in ll:
+            lat = float(ll["latitude"])
+            lon = float(ll["longitude"])
+
+        return Callsign(call=callsign,
+                        home_call=self._callinfo.get_homecall(callsign),
+                        country=self._callinfo.get_country_name(callsign),
+                        dxcc_id=self._callinfo.get_adif_id(callsign),
+                        continent=self._callinfo.get_continent(callsign),
+                        cq_zone=self._callinfo.get_cqz(callsign),
+                        itu_zone=self._callinfo.get_ituz(callsign),
+                        latitude=lat,
+                        longitude=lon)

providers/callsigndata/countryfiles.py

@@ -0,0 +1,48 @@
+import logging
+
+from pyhamtools import LookupLib, Callinfo
+
+from data.callsign import Callsign
+from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
+
+
+class CountryFiles(FileDownloadCallsignDataProvider):
+    """Callsign data provider for Country-files.com, which provides basic callsign to DXCC entity mapping."""
+
+    POLL_INTERVAL_DAYS = 30
+    DATA_URL = "https://www.country-files.com/cty/cty.plist"
+    CACHE_PATH = "cache/cty.plist"
+    _callinfo = None
+
+    def __init__(self, provider_config):
+        super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS)
+
+    def _handle_file(self, path):
+        try:
+            lookuplib = LookupLib(lookuptype="countryfile", filename=path)
+            self._callinfo = Callinfo(lookuplib)
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading Country Files cty.plist.", e, exc_info=True)
+            return False
+
+    def lookup(self, callsign, lookup_credentials):
+        # Lookup credentials are not required for this source.
+        # Lat/lon will only be centre of country or capital city from this source
+        ll = self._callinfo.get_lat_long(callsign)
+        lat = None
+        lon = None
+        if ll and "latitude" in ll and "longitude" in ll:
+            lat = float(ll["latitude"])
+            lon = float(ll["longitude"])
+
+        return Callsign(call=callsign,
+                        home_call=self._callinfo.get_homecall(callsign),
+                        country=self._callinfo.get_country_name(callsign),
+                        dxcc_id=self._callinfo.get_adif_id(callsign),
+                        continent=self._callinfo.get_continent(callsign),
+                        cq_zone=self._callinfo.get_cqz(callsign),
+                        itu_zone=self._callinfo.get_ituz(callsign),
+                        latitude=lat,
+                        longitude=lon)

providers/callsigndata/file_download_callsign_data_provider.py

@@ -0,0 +1,85 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from providers.callsigndata.callsign_data_provider import CallsignDataProvider
+
+
+class FileDownloadCallsignDataProvider(CallsignDataProvider):
+    """Generic static reference data provider class for providers that fetch their data from the web by downloading a
+     file."""
+
+    def __init__(self, name, provider_config, url, cache_file_path, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
+        super().__init__(name, provider_config)
+        self._url = url
+        self._cache_file_path = cache_file_path
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+        self._url_data_cache = URLDataCache("callsigndata_" + name)
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info(
+            "Set up query of " + self.name + " callsign reference data every " + str(self._poll_interval) + " days.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
+                break
+
+    def _poll(self):
+        try:
+            # Request the file. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
+            logging.debug("Downloading " + self.name + " callsign reference data...")
+            http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
+            # Check response code was good
+            if http_response.ok:
+                # Save the data to a local file
+                with open(self._cache_file_path, "wb") as f:
+                    f.write(http_response.content)
+                    f.flush()
+                    # Pass off to the subclass for processing
+                    ok = self._handle_file(self._cache_file_path)
+                    if ok:
+                        self.status = "OK"
+                        self.last_update_time = datetime.now(pytz.UTC)
+                        logging.info("Updated callsign reference data from " + self.name)
+                    else:
+                        self.status = "Error"
+                        logging.warning(f"Error updating callsign reference data from {self.name}.")
+
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}.")
+
+        except ConnectionError:
+            self.status = "Error"
+            logging.warning(f"Connection error when downloading callsign reference data from {self.name}.")
+        except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
+            logging.warning(f"Timeout when downloading callsign reference data from {self.name}.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in callsign reference data provider (" + self.name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _handle_file(self, path):
+        """Handle an updated file downloaded from the server. Return true if successful, false otherwise."""
+
+        raise NotImplementedError("Subclasses must implement this method")