Refactor of caching & data storage part 12 #118

providers/callsigndata/api_query_callsign_data_provider.py

@@ -14,9 +14,9 @@ from providers.callsigndata.callsign_data_provider import CallsignDataProvider
 class APIQueryCallsignDataProvider(CallsignDataProvider):
     """Generic callsign data provider class for providers that fetch their data from the web on-demand using an API."""
 
-    def __init__(self, name, provider_config):
+    def __init__(self, name, provider_config, storage):
         """ Set up the provider."""
-        super().__init__(name, provider_config)
+        super().__init__(name, provider_config, storage)
 
     def start(self):
         pass

providers/callsigndata/callsign_data_provider.py

@@ -2,18 +2,22 @@ from datetime import datetime
 
 import pytz
 
+from core.data_store import DATA_STORE
+
 
 class CallsignDataProvider:
     """Generic callsign reference data provider class. Subclasses of this set up the various mechanisms via which
     Spothole can look up data for callsigns."""
 
-    def __init__(self, name, provider_config):
-        """Constructor"""
+    def __init__(self, name, provider_config, storage):
+        """Constructor. As well as name and config, provide the storage object from DATA_STORE that will be used to
+        store the result of lookups to speed up future access."""
 
         self.name = name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
+        self._storage = storage
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access remote resources, if
@@ -31,6 +35,19 @@ class CallsignDataProvider:
         that have been provided by the user for this session (QRZ.com/HamQTH) to allow us to look up using those
         services on the user's behalf. (Clublog is looked up using an API key owned by the server and provided in its
         config file, so users need not provide their own.) Returns a Callsign object with as much data populated as
-        possible."""
+        possible. Data is cached internally for a set period of 30 days to avoid the need to request data from servers
+        each time."""
+
+        if callsign in self._storage:
+            return self._storage[callsign]
+        else:
+            c = self._perform_new_lookup(callsign, lookup_credentials)
+            if c:
+                self._storage.set(callsign, c, expire=DATA_STORE.CALLSIGN_DATA_TTL_SEC)
+            return c
+
+
+    def _perform_new_lookup(self, callsign, lookup_credentials):
+        """Makes a new request to the data source for callsign data."""
 
         raise NotImplementedError("Subclasses must implement this method")

providers/callsigndata/clublogapi.py

@@ -4,6 +4,7 @@ from datetime import datetime
 import pytz
 from pyhamtools import LookupLib, Callinfo
 
+from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
 from data.callsign import Callsign
 from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
@@ -25,10 +26,12 @@ class ClublogAPI(APIQueryCallsignDataProvider):
             logging.warning(
                 "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
 
-        super().__init__("Clublog API", provider_config)
+        super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
+
+        self.status = "Ready"
 
 
-    def lookup(self, callsign, lookup_credentials):
+    def _perform_new_lookup(self, callsign, lookup_credentials):
         callsign_data = Callsign(call=callsign)
 
         try:

providers/callsigndata/clublogxml.py

@@ -3,8 +3,8 @@ import logging
 
 from pyhamtools import LookupLib, Callinfo
 
+from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
-from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -26,7 +26,7 @@ class ClublogXML(FileDownloadCallsignDataProvider):
                 "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
 
         super().__init__("Clublog XML", provider_config, self.DATA_URL + "?api=" + self._api_key,
-                         self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS)
+                         self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS, DATA_STORE.callsign_data_clublogxml)
 
     def _handle_file(self, path):
         try:
@@ -48,5 +48,5 @@ class ClublogXML(FileDownloadCallsignDataProvider):
             logging.error("Exception when loading Clublog XML.", e, exc_info=True)
             return False
 
-    def lookup(self, callsign, lookup_credentials):
+    def _perform_new_lookup(self, callsign, lookup_credentials):
         return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)

providers/callsigndata/countryfiles.py

@@ -2,8 +2,8 @@ import logging
 
 from pyhamtools import LookupLib, Callinfo
 
+from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
-from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -16,7 +16,8 @@ class CountryFiles(FileDownloadCallsignDataProvider):
     _callinfo = None
 
     def __init__(self, provider_config):
-        super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS)
+        super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS,
+                         DATA_STORE.callsign_data_countryfiles)
 
     def _handle_file(self, path):
         try:
@@ -28,5 +29,5 @@ class CountryFiles(FileDownloadCallsignDataProvider):
             logging.error("Exception when loading Country Files cty.plist.", e, exc_info=True)
             return False
 
-    def lookup(self, callsign, lookup_credentials):
+    def _perform_new_lookup(self, callsign, lookup_credentials):
         return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)

providers/callsigndata/file_download_callsign_data_provider.py

@@ -14,9 +14,9 @@ from providers.callsigndata.callsign_data_provider import CallsignDataProvider
 class FileDownloadCallsignDataProvider(CallsignDataProvider):
     """Generic callsign data provider class for providers that fetch their data from the web by downloading a file."""
 
-    def __init__(self, name, provider_config, url, cache_file_path, poll_interval):
+    def __init__(self, name, provider_config, url, cache_file_path, poll_interval, storage):
         """ Set up the provider, note poll_interval is in *days*."""
-        super().__init__(name, provider_config)
+        super().__init__(name, provider_config, storage)
         self._url = url
         self._cache_file_path = cache_file_path
         self._poll_interval = poll_interval

providers/callsigndata/qrz.py

@@ -0,0 +1,154 @@
+import logging
+import urllib.parse
+from datetime import timedelta
+
+import xmltodict
+from pyhamtools import callinfo
+from requests import ConnectTimeout, ReadTimeout
+from requests_cache import CachedSession
+
+from core.constants import HTTP_HEADERS
+from core.data_store import DATA_STORE, CACHE_DIR
+from core.url_data_cache import URLDataCache
+from data.callsign import Callsign
+from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
+
+
+class QRZ(APIQueryCallsignDataProvider):
+    """Callsign data provider for QRZ.com."""
+
+    def __init__(self, provider_config):
+        super().__init__("QRZ.com", provider_config, DATA_STORE.callsign_data_qrz)
+        self._QRZ_BASE_URL = "https://xmldata.qrz.com/xml/current/"
+        self._URL_DATA_CACHE = URLDataCache("qrz")
+        # Separate URL cache for session key lookups. Once a session key is returned from logging in with a username
+        # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
+        self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/qrz-creds",
+                                                expire_after=timedelta(minutes=55))
+        self.status = "Ready"
+
+    def _perform_new_lookup(self, callsign, lookup_credentials):
+        # If we don't have QRZ credentials, skip this lookup
+        if not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
+                or lookup_credentials.qrz_session_key):
+            return None
+
+        try:
+            # Obtain session key from credentials, by looking it up from username & password if necessary.
+            session_key = None
+            if lookup_credentials.qrz_session_key:
+                session_key = lookup_credentials.qrz_session_key
+            elif lookup_credentials.qrz_username and lookup_credentials.qrz_password:
+                try:
+                    login_response = self._URL_DATA_CACHE.get(
+                        self._QRZ_BASE_URL + "?username=" + urllib.parse.quote_plus(lookup_credentials.qrz_username) +
+                        "&password=" + urllib.parse.quote_plus(lookup_credentials.qrz_password) + "&agent=spothole",
+                        headers=HTTP_HEADERS).content
+                    login_data = xmltodict.parse(login_response)
+                    session = login_data.get("QRZDatabase", {}).get("Session", {})
+                    if "Key" in session:
+                        session_key = str(session["Key"])
+                    else:
+                        logging.warning("QRZ.com login details incorrect, failed to look up with QRZ.")
+                        return None
+                except Exception:
+                    logging.error("Exception when getting QRZ.com session key")
+                    return None
+
+            if not session_key:
+                return None
+
+            # Try the call as given, then fall back to the base call (strips /P, /M etc.)
+            calls_to_try = [callsign]
+            try:
+                home_call = callinfo.Callinfo.get_homecall(callsign)
+                if home_call != callsign:
+                    calls_to_try.append(home_call)
+            except ValueError:
+                logging.debug("Could not look up home call for callsign %s", callsign)
+
+            # Try looking up each call using the API
+            for lookup_call in calls_to_try:
+                try:
+                    response = self._URL_DATA_CACHE.get(
+                        self._QRZ_BASE_URL + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
+                        headers=HTTP_HEADERS, timeout=10)
+                    if response.ok:
+                        qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
+                        if qrz_response:
+                            if "Callsign" in qrz_response:
+                                # Found data, convert it to our object and return it
+                                return self.qrz_response_to_callsign(callsign, qrz_response.get("Callsign"))
+
+                            elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
+                                # Errors here are normally just "callsign not in database", no need to log that ourselves
+                                # above debug level.
+                                logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call,
+                                              qrz_response.get("Session").get("Error"))
+
+                        elif not response.from_cache:
+                            logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call)
+                    elif not response.from_cache:
+                        logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call)
+
+                except (KeyError, ValueError):
+                    continue
+                except ConnectionError:
+                    logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call)
+                    continue
+                except (ConnectTimeout, ReadTimeout):
+                    logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call)
+                    continue
+                except Exception:
+                    logging.error("Exception when looking up callsign %s using QRZ", lookup_call, exc_info=True)
+                    continue
+
+            # Not found in QRZ; return a Callsign object with no data so we cache that and don't keep retrying
+            return Callsign(call=callsign)
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from QRZ.com", e, exc_info=True)
+            # Return None, this won't be cached so we will be asked to query data again for this call next time.
+            return None
+
+    @staticmethod
+    def qrz_response_to_callsign(callsign, data):
+        """Convert the "Callsign" block in QRZ's API response to our own Callsign object."""
+
+        # Get a name
+        name = None
+        if "name_fmt" in data:
+            name = data["name_fmt"]
+        if "fname" in data:
+            name = data["fname"]
+            if "nick" in data:
+                name = name + " \"" + data["nick"] + "\""
+            if "name" in data:
+                name = name + " " + data["name"]
+
+        # Check for sensible latitudes
+        lat = None
+        lon = None
+        if "latitude" in data and "longitude" in data and (
+                float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float(
+                data["latitude"]) < 89.9:
+            lat = float(data["latitude"])
+            lon = float(data["longitude"])
+
+        # Check for sensible grids
+        grid = None
+        if "grid" in data and not data["grid"].startswith("AA00"):
+            grid = data["grid"]
+
+        return Callsign(call=callsign,
+                        home_call=callinfo.Callinfo.get_homecall(callsign),
+                        name=name,
+                        qth=data["addr2"] if "addr2" in data else None,
+                        country=data["country"] if "country" in data else None,
+                        latitude=lat,
+                        longitude=lon,
+                        grid=grid,
+                        dxcc_id=int(data["adif"]) if "adif" in data else None,
+                        cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
+                        itu_zone=int(data["ituzone"]) if "adif" in data else None)