Refactor of caching & data storage part 12 #118

README.md

@@ -166,8 +166,9 @@ source .venv/bin/activate
 python3 spothole.py
 ```
 
-The software can take a few seconds to start up, mostly because it is downloading an updated file to match callsigns to
-countries. This is normal, don't panic!
+The software can take a few seconds to start up, mostly because it is downloading updated files containing the various
+bits of reference data it uses. This is normal, don't panic! Once you see `You can access your copy of Spothole at
+http://localhost:8080` in the log, your server is good to go.
 
 If you see some errors on startup, check your configuration, e.g. in case you have specified a port for the web server
 that is already in use by something else.

config-example.yml

@@ -260,12 +260,19 @@ callsign-data-providers:
     enabled: true
     # API key for Clublog to look up information. Required in order to enable this provider. You will need to request
     # one via their helpdesk portal if you want to use callsign lookups from Clublog.
-    clublog-api-key: ""
+    api-key: ""
 
   - class: "ClublogAPI"
-    enabled: true
+    # Querying the Clublog API directly doesn't provide any more data than the XML version, it just provides slightly
+    # more up-to-date information in the rare case that the prefix data changes, at a significant cost of looking up
+    # every callsign via an API call. Normally left disabled but it exists as an option.
+    enabled: false
     # API key for Clublog to look up information. Required in order to enable this provider.
-    clublog-api-key: ""
+    api-key: ""
+
+  - class: "QRZ"
+    enabled: true
+    # No server-side credentials for QRZ. Users must provide their own as per QRZ policy.
 
 
 # Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week

core/data_store.py

@@ -20,11 +20,15 @@ class DataStore:
         self._MAX_SPOT_COUNT = 100000
         self._MAX_ALERT_COUNT = 100000
         self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
-        self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
+        self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
         # Caches
         self.alerts = None
         self.spots = None
-        self.callsigns = None
+        self.callsign_data_countryfiles = None
+        self.callsign_data_clublogxml = None
+        self.callsign_data_clublogapi = None
+        self.callsign_data_qrz = None
+        self.callsign_data_hamqth = None
         self.dxcc_data = None
         self.dxcc_lookup_by_call_regex = []
         self.sigrefs = None
@@ -65,8 +69,16 @@ class DataStore:
         # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
         # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
         # we've seen, so we rely on them timing out and this triggering another lookup.
-        self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns")
-        logging.info(f"Loaded data for %d callsigns.", len(self.callsigns))
+        self.callsign_data_countryfiles = diskcache.Cache(CACHE_DIR + "callsign_data_countryfiles")
+        self.callsign_data_clublogxml = diskcache.Cache(CACHE_DIR + "callsign_data_clublogxml")
+        self.callsign_data_clublogapi = diskcache.Cache(CACHE_DIR + "callsign_data_clublogapi")
+        self.callsign_data_qrz = diskcache.Cache(CACHE_DIR + "callsign_data_qrz")
+        self.callsign_data_hamqth = diskcache.Cache(CACHE_DIR + "callsign_data_hamqth")
+        unique_keys = set()
+        for c in [self.callsign_data_countryfiles, self.callsign_data_clublogxml, self.callsign_data_clublogapi,
+                  self.callsign_data_qrz, self.callsign_data_hamqth]:
+            unique_keys.update(c)
+        logging.info(f"Loaded data for %d callsigns.", len(unique_keys))
 
         # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
         # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
@@ -97,7 +109,11 @@ class DataStore:
         self._status.close()
         self.dxcc_data.close()
         self.sigrefs.close()
-        self.callsigns.close()
+        self.callsign_data_countryfiles.close()
+        self.callsign_data_clublogxml.close()
+        self.callsign_data_clublogapi.close()
+        self.callsign_data_qrz.close()
+        self.callsign_data_hamqth.close()
 
 # Global object
 DATA_STORE = DataStore()

core/sig_lookup_helper.py

@@ -86,7 +86,9 @@ def populate_sig_ref_info(sig_ref):
                 if value is not None and sig_ref.__dict__.get(key) is None:
                     sig_ref.__dict__[key] = value
         else:
-            logging.warning("%s database did not contain data for ref %s", sig, ref_id)
+            # Maybe a super new reference we don't know about yet, but more likely a typo or a test reference,
+            # just silently ignore it.
+            logging.debug("%s database did not contain data for ref %s", sig, ref_id)
 
     except Exception:
         logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)

providers/callsigndata/api_query_callsign_data_provider.py

@@ -14,9 +14,9 @@ from providers.callsigndata.callsign_data_provider import CallsignDataProvider
 class APIQueryCallsignDataProvider(CallsignDataProvider):
     """Generic callsign data provider class for providers that fetch their data from the web on-demand using an API."""
 
-    def __init__(self, name, provider_config):
+    def __init__(self, name, provider_config, storage):
         """ Set up the provider."""
-        super().__init__(name, provider_config)
+        super().__init__(name, provider_config, storage)
 
     def start(self):
         pass

providers/callsigndata/callsign_data_provider.py

@@ -2,18 +2,22 @@ from datetime import datetime
 
 import pytz
 
+from core.data_store import DATA_STORE
+
 
 class CallsignDataProvider:
     """Generic callsign reference data provider class. Subclasses of this set up the various mechanisms via which
     Spothole can look up data for callsigns."""
 
-    def __init__(self, name, provider_config):
-        """Constructor"""
+    def __init__(self, name, provider_config, storage):
+        """Constructor. As well as name and config, provide the storage object from DATA_STORE that will be used to
+        store the result of lookups to speed up future access."""
 
         self.name = name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
+        self._storage = storage
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access remote resources, if
@@ -31,6 +35,19 @@ class CallsignDataProvider:
         that have been provided by the user for this session (QRZ.com/HamQTH) to allow us to look up using those
         services on the user's behalf. (Clublog is looked up using an API key owned by the server and provided in its
         config file, so users need not provide their own.) Returns a Callsign object with as much data populated as
-        possible."""
+        possible. Data is cached internally for a set period of 30 days to avoid the need to request data from servers
+        each time."""
+
+        if callsign in self._storage:
+            return self._storage[callsign]
+        else:
+            c = self._perform_new_lookup(callsign, lookup_credentials)
+            if c:
+                self._storage.set(callsign, c, expire=DATA_STORE.CALLSIGN_DATA_TTL_SEC)
+            return c
+
+
+    def _perform_new_lookup(self, callsign, lookup_credentials):
+        """Makes a new request to the data source for callsign data."""
 
         raise NotImplementedError("Subclasses must implement this method")

providers/callsigndata/clublogapi.py

@@ -4,6 +4,7 @@ from datetime import datetime
 import pytz
 from pyhamtools import LookupLib, Callinfo
 
+from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
 from data.callsign import Callsign
 from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
@@ -25,10 +26,12 @@ class ClublogAPI(APIQueryCallsignDataProvider):
             logging.warning(
                 "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
 
-        super().__init__("Clublog API", provider_config)
+        super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
+
+        self.status = "Ready"
 
 
-    def lookup(self, callsign, lookup_credentials):
+    def _perform_new_lookup(self, callsign, lookup_credentials):
         callsign_data = Callsign(call=callsign)
 
         try:

providers/callsigndata/clublogxml.py

@@ -3,8 +3,8 @@ import logging
 
 from pyhamtools import LookupLib, Callinfo
 
+from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
-from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -26,7 +26,7 @@ class ClublogXML(FileDownloadCallsignDataProvider):
                 "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
 
         super().__init__("Clublog XML", provider_config, self.DATA_URL + "?api=" + self._api_key,
-                         self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS)
+                         self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS, DATA_STORE.callsign_data_clublogxml)
 
     def _handle_file(self, path):
         try:
@@ -48,5 +48,5 @@ class ClublogXML(FileDownloadCallsignDataProvider):
             logging.error("Exception when loading Clublog XML.", e, exc_info=True)
             return False
 
-    def lookup(self, callsign, lookup_credentials):
+    def _perform_new_lookup(self, callsign, lookup_credentials):
         return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)

providers/callsigndata/countryfiles.py

@@ -2,8 +2,8 @@ import logging
 
 from pyhamtools import LookupLib, Callinfo
 
+from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
-from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -16,7 +16,8 @@ class CountryFiles(FileDownloadCallsignDataProvider):
     _callinfo = None
 
     def __init__(self, provider_config):
-        super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS)
+        super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS,
+                         DATA_STORE.callsign_data_countryfiles)
 
     def _handle_file(self, path):
         try:
@@ -28,5 +29,5 @@ class CountryFiles(FileDownloadCallsignDataProvider):
             logging.error("Exception when loading Country Files cty.plist.", e, exc_info=True)
             return False
 
-    def lookup(self, callsign, lookup_credentials):
+    def _perform_new_lookup(self, callsign, lookup_credentials):
         return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)

providers/callsigndata/file_download_callsign_data_provider.py

@@ -14,9 +14,9 @@ from providers.callsigndata.callsign_data_provider import CallsignDataProvider
 class FileDownloadCallsignDataProvider(CallsignDataProvider):
     """Generic callsign data provider class for providers that fetch their data from the web by downloading a file."""
 
-    def __init__(self, name, provider_config, url, cache_file_path, poll_interval):
+    def __init__(self, name, provider_config, url, cache_file_path, poll_interval, storage):
         """ Set up the provider, note poll_interval is in *days*."""
-        super().__init__(name, provider_config)
+        super().__init__(name, provider_config, storage)
         self._url = url
         self._cache_file_path = cache_file_path
         self._poll_interval = poll_interval

providers/callsigndata/qrz.py

@@ -0,0 +1,154 @@
+import logging
+import urllib.parse
+from datetime import timedelta
+
+import xmltodict
+from pyhamtools import callinfo
+from requests import ConnectTimeout, ReadTimeout
+from requests_cache import CachedSession
+
+from core.constants import HTTP_HEADERS
+from core.data_store import DATA_STORE, CACHE_DIR
+from core.url_data_cache import URLDataCache
+from data.callsign import Callsign
+from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
+
+
+class QRZ(APIQueryCallsignDataProvider):
+    """Callsign data provider for QRZ.com."""
+
+    def __init__(self, provider_config):
+        super().__init__("QRZ.com", provider_config, DATA_STORE.callsign_data_qrz)
+        self._QRZ_BASE_URL = "https://xmldata.qrz.com/xml/current/"
+        self._URL_DATA_CACHE = URLDataCache("qrz")
+        # Separate URL cache for session key lookups. Once a session key is returned from logging in with a username
+        # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
+        self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/qrz-creds",
+                                                expire_after=timedelta(minutes=55))
+        self.status = "Ready"
+
+    def _perform_new_lookup(self, callsign, lookup_credentials):
+        # If we don't have QRZ credentials, skip this lookup
+        if not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
+                or lookup_credentials.qrz_session_key):
+            return None
+
+        try:
+            # Obtain session key from credentials, by looking it up from username & password if necessary.
+            session_key = None
+            if lookup_credentials.qrz_session_key:
+                session_key = lookup_credentials.qrz_session_key
+            elif lookup_credentials.qrz_username and lookup_credentials.qrz_password:
+                try:
+                    login_response = self._URL_DATA_CACHE.get(
+                        self._QRZ_BASE_URL + "?username=" + urllib.parse.quote_plus(lookup_credentials.qrz_username) +
+                        "&password=" + urllib.parse.quote_plus(lookup_credentials.qrz_password) + "&agent=spothole",
+                        headers=HTTP_HEADERS).content
+                    login_data = xmltodict.parse(login_response)
+                    session = login_data.get("QRZDatabase", {}).get("Session", {})
+                    if "Key" in session:
+                        session_key = str(session["Key"])
+                    else:
+                        logging.warning("QRZ.com login details incorrect, failed to look up with QRZ.")
+                        return None
+                except Exception:
+                    logging.error("Exception when getting QRZ.com session key")
+                    return None
+
+            if not session_key:
+                return None
+
+            # Try the call as given, then fall back to the base call (strips /P, /M etc.)
+            calls_to_try = [callsign]
+            try:
+                home_call = callinfo.Callinfo.get_homecall(callsign)
+                if home_call != callsign:
+                    calls_to_try.append(home_call)
+            except ValueError:
+                logging.debug("Could not look up home call for callsign %s", callsign)
+
+            # Try looking up each call using the API
+            for lookup_call in calls_to_try:
+                try:
+                    response = self._URL_DATA_CACHE.get(
+                        self._QRZ_BASE_URL + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
+                        headers=HTTP_HEADERS, timeout=10)
+                    if response.ok:
+                        qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
+                        if qrz_response:
+                            if "Callsign" in qrz_response:
+                                # Found data, convert it to our object and return it
+                                return self.qrz_response_to_callsign(callsign, qrz_response.get("Callsign"))
+
+                            elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
+                                # Errors here are normally just "callsign not in database", no need to log that ourselves
+                                # above debug level.
+                                logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call,
+                                              qrz_response.get("Session").get("Error"))
+
+                        elif not response.from_cache:
+                            logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call)
+                    elif not response.from_cache:
+                        logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call)
+
+                except (KeyError, ValueError):
+                    continue
+                except ConnectionError:
+                    logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call)
+                    continue
+                except (ConnectTimeout, ReadTimeout):
+                    logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call)
+                    continue
+                except Exception:
+                    logging.error("Exception when looking up callsign %s using QRZ", lookup_call, exc_info=True)
+                    continue
+
+            # Not found in QRZ; return a Callsign object with no data so we cache that and don't keep retrying
+            return Callsign(call=callsign)
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from QRZ.com", e, exc_info=True)
+            # Return None, this won't be cached so we will be asked to query data again for this call next time.
+            return None
+
+    @staticmethod
+    def qrz_response_to_callsign(callsign, data):
+        """Convert the "Callsign" block in QRZ's API response to our own Callsign object."""
+
+        # Get a name
+        name = None
+        if "name_fmt" in data:
+            name = data["name_fmt"]
+        if "fname" in data:
+            name = data["fname"]
+            if "nick" in data:
+                name = name + " \"" + data["nick"] + "\""
+            if "name" in data:
+                name = name + " " + data["name"]
+
+        # Check for sensible latitudes
+        lat = None
+        lon = None
+        if "latitude" in data and "longitude" in data and (
+                float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float(
+                data["latitude"]) < 89.9:
+            lat = float(data["latitude"])
+            lon = float(data["longitude"])
+
+        # Check for sensible grids
+        grid = None
+        if "grid" in data and not data["grid"].startswith("AA00"):
+            grid = data["grid"]
+
+        return Callsign(call=callsign,
+                        home_call=callinfo.Callinfo.get_homecall(callsign),
+                        name=name,
+                        qth=data["addr2"] if "addr2" in data else None,
+                        country=data["country"] if "country" in data else None,
+                        latitude=lat,
+                        longitude=lon,
+                        grid=grid,
+                        dxcc_id=int(data["adif"]) if "adif" in data else None,
+                        cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
+                        itu_zone=int(data["ituzone"]) if "adif" in data else None)