Refactor of caching & data storage part 16 #118

config-example.yml

@@ -251,32 +251,44 @@ sig-ref-data-providers:
 
 
 # Callsign data providers to use. These allow Spothole to provide information about callsigns, either from static
-# lookups or from web-based services such as QRZ.
+# lookups or from web-based services such as QRZ. Note that use of QRZ and HamQTH is *on behalf of the user*, and
+# requires the user to enter their credentials into the website UI or provide them in an API call. Spothole will not
+# look up all calls via QRZ/HamQTH using the server owner's credentials, as this is against their usage policy.
 callsign-data-providers:
-  - class: "CountryFiles"
+  - class: "QRZ"
     enabled: true
+    # Callsign data providers can often provide conflicting data, e.g. a home location from QRZ vs the centre of the
+    # DXCC entity from a country file lookup. The priority flag sets which source "wins" in the event of conflict.
+    # Lower numbers take priority over higher numbers. Generally then, QRZ/HamQTH should have low numbers as they are
+    # likely to have the most accurate data.
+    priority: 1
+    # No server-side credentials for QRZ. Users must provide their own as per QRZ policy.
 
-  - class: "ClublogXML"
+  - class: "HamQTH"
     enabled: true
-    # API key for Clublog to look up information. Required in order to enable this provider. You will need to request
-    # one via their helpdesk portal if you want to use callsign lookups from Clublog.
-    api-key: ""
+    priority: 2
+    # No server-side credentials for HamQTH. Users must provide their own.
 
   - class: "ClublogAPI"
     # Querying the Clublog API directly doesn't provide any more data than the XML version, it just provides slightly
     # more up-to-date information in the rare case that the prefix data changes, at a significant cost of looking up
     # every callsign via an API call. Normally left disabled but it exists as an option.
     enabled: false
-    # API key for Clublog to look up information. Required in order to enable this provider.
+    priority: 3
+    # API key for Clublog to look up information. Required in order to enable this provider. Unlike QRZ and HamQTH,
+    # Clublog uses an API key issued to Spothole, not to the end user.
     api-key: ""
 
-  - class: "QRZ"
+  - class: "ClublogXML"
     enabled: true
-    # No server-side credentials for QRZ. Users must provide their own as per QRZ policy.
+    priority: 4
+    # API key for Clublog to look up information. Required in order to enable this provider. You will need to request
+    # one via their helpdesk portal if you want to use callsign lookups from Clublog.
+    api-key: ""
 
-  - class: "HamQTH"
+  - class: "CountryFiles"
+    priority: 5
     enabled: true
-    # No server-side credentials for HamQTH. Users must provide their own.
 
 
 # Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week

core/call_lookup_helper.py

@@ -1,21 +1,4 @@
-import gzip
-import logging
-import urllib.parse
-from datetime import timedelta
-
-import xmltodict
-from diskcache import Cache
-from pyhamtools import LookupLib, Callinfo, callinfo
-from pyhamtools.exceptions import APIKeyMissingError
-from pyhamtools.locator import latlong_to_locator
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
-from requests_cache import CachedSession
-
-from core.config import config
-from core.constants import HTTP_HEADERS, HAMQTH_PRG
 from core.data_providers import DATA_PROVIDERS
-from core.data_store import DATA_STORE
-from core.url_data_cache import URLDataCache
 from data.callsign import Callsign
 
 
@@ -24,14 +7,24 @@ def get_call_info(callsign, lookup_credentials):
     lookup_credentials is an optional object that carries the user's QRZ.com/HamQTH credentials, if they provided them,
     to enable lookup using those providers."""
 
-    callsign = Callsign(call=callsign)
-    for p in DATA_PROVIDERS.callsign_data_providers:
-        # Get new lookup data
-        data = p.lookup(callsign, lookup_credentials)
-        if data:
-            # Merge in turn, replacing any existing content where we have it.
-            for key, value in data.__dict__.items():
-                if value is not None:
-                    callsign.__dict__[key] = value
+    callsign_data = Callsign(call=callsign)
 
-    return callsign
+    if callsign:
+        # Sort callsign providers by priority order, so we query the highest priority (lowest numbers) first, and only
+        # query other providers for data we are missing as we go along.
+        for p in sorted(DATA_PROVIDERS.callsign_data_providers, key=lambda p: p.priority):
+            if p.enabled:
+                # Get new lookup data
+                data = p.lookup(callsign, lookup_credentials)
+                if data:
+                    # If we have new data for fields that were previously unpopulated, add them in
+                    for key, value in data.__dict__.items():
+                        if value is not None and callsign_data.__dict__.get(key) is None:
+                            callsign_data.__dict__[key] = value
+
+                    # If callsign data is fully populated, avoid looking up using other providers as their data will not
+                    # be used
+                    if callsign_data.fully_populated():
+                        break
+
+    return callsign_data

core/data_providers.py

@@ -29,15 +29,7 @@ class DataProviders:
 
 
     def start(self):
-        for p in self.spot_providers:
-            if p.enabled:
-                p.start()
-        for p in self.alert_providers:
-            if p.enabled:
-                p.start()
-        for p in self.solar_condition_providers:
-            if p.enabled:
-                p.start()
+        # Start data providers before spot/alert providers so the lookup data is there already for incoming spots
         for p in self.static_data_providers:
             if p.enabled:
                 p.start()
@@ -47,6 +39,15 @@ class DataProviders:
         for p in self.callsign_data_providers:
             if p.enabled:
                 p.start()
+        for p in self.spot_providers:
+            if p.enabled:
+                p.start()
+        for p in self.alert_providers:
+            if p.enabled:
+                p.start()
+        for p in self.solar_condition_providers:
+            if p.enabled:
+                p.start()
 
     def stop(self):
         for sp in self.spot_providers:

core/status_reporter.py

@@ -84,7 +84,8 @@ class StatusReporter:
         DATA_STORE.status_data["callsign_data_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
+                           "lookup_count": p.lookup_count},
                 DATA_PROVIDERS.callsign_data_providers))
         DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"],
                                                "last_api_access": WEB_SERVER.web_server_metrics[

core/utils.py

@@ -2,6 +2,7 @@ import logging
 
 import simplejson
 from pyhamtools.frequency import freq_to_band
+from pyhamtools.locator import latlong_to_locator
 
 from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES
 from core.data_store import DATA_STORE
@@ -84,23 +85,33 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
     """Utility function to take the data provided by a PyHamTools CallInfo object and populate our own Callsign data
     object from it"""
 
-    home_call = callinfo.get_homecall(callsign)
-    data = callinfo.get_all()
+    try:
+        home_call = callinfo.get_homecall(callsign)
+        data = callinfo.get_all(callsign)
 
-    country = data["country"] if "country" in data else None
-    dxcc_id = data["adif"] if "adif" in data else None
-    continent = data["continent"] if "continent" in data else None
-    cq_zone = data["cqz"] if "cqz" in data else None
-    itu_zone = data["ituz"] if "ituz" in data else None
-    lat = float(data["latitude"]) if "latitude" in data else None
-    lon = float(data["longitude"]) if "longitude" in data else None
+        country = data["country"] if "country" in data else None
+        dxcc_id = data["adif"] if "adif" in data else None
+        continent = data["continent"] if "continent" in data else None
+        cq_zone = data["cqz"] if "cqz" in data else None
+        itu_zone = data["ituz"] if "ituz" in data else None
+        lat = float(data["latitude"]) if "latitude" in data else None
+        lon = float(data["longitude"]) if "longitude" in data else None
+        grid = None
+        if lat and lon:
+            grid = latlong_to_locator(lat, lon)
 
-    return Callsign(call=callsign,
-                    home_call=home_call,
-                    country=country,
-                    dxcc_id=dxcc_id,
-                    continent=continent,
-                    cq_zone=cq_zone,
-                    itu_zone=itu_zone,
-                    latitude=lat,
-                    longitude=lon)
+        return Callsign(call=callsign,
+                        home_call=home_call,
+                        country=country,
+                        dxcc_id=dxcc_id,
+                        continent=continent,
+                        cq_zone=cq_zone,
+                        itu_zone=itu_zone,
+                        latitude=lat,
+                        longitude=lon,
+                        grid=grid,
+                        location_source="DXCC")
+
+    except (KeyError, ValueError):
+        # Unknown callsign, can't look anything up, return a Callsign object with basic data so that gets cached
+        return Callsign(call=callsign)

data/alert.py

@@ -7,7 +7,7 @@ from datetime import datetime, timedelta
 
 import pytz
 
-from core.call_lookup_helper import lookup_helper
+from core.call_lookup_helper import get_call_info
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.utils import get_flag_for_dxcc
 
@@ -86,16 +86,17 @@ class Alert:
 
             # DX country, continent, zones etc. from callsign. CQ/ITU zone are better looked up with a location but we don't
             # have a real location for alerts.
+            call_info = get_call_info(self.dx_calls[0], credentials)
             if self.dx_calls and self.dx_calls[0] and not self.dx_country:
-                self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_calls[0], credentials)
+                self.dx_country = call_info.country
             if self.dx_calls and self.dx_calls[0] and not self.dx_continent:
-                self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_calls[0], credentials)
+                self.dx_continent = call_info.continent
             if self.dx_calls and self.dx_calls[0] and not self.dx_cq_zone:
-                self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_calls[0], credentials)
+                self.dx_cq_zone = call_info.cq_zone
             if self.dx_calls and self.dx_calls[0] and not self.dx_itu_zone:
-                self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_calls[0], credentials)
+                self.dx_itu_zone = call_info.itu_zone
             if self.dx_calls and self.dx_calls[0] and not self.dx_dxcc_id:
-                self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_calls[0], credentials)
+                self.dx_dxcc_id = call_info.dxcc_id
             if self.dx_dxcc_id and not self.dx_flag:
                 self.dx_flag = get_flag_for_dxcc(self.dx_dxcc_id)
 
@@ -124,12 +125,10 @@ class Alert:
                 self_copy.received_time_iso = ""
                 self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
 
-            # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data
-            # from the actual alerting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon
-            # instead of the one from the park reference they're at.
+            # DX operator name lookup, using QRZ.com/HamQTH.
             if self.dx_calls and not self.dx_names:
                 self.dx_names = list(
-                    map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c, credentials), self.dx_calls))
+                    map(lambda c: get_call_info(c, credentials).name, self.dx_calls))
 
         except Exception as e:
             logging.error("Exception while inferring missing data from spot", e, exc_info=True)

data/callsign.py

@@ -35,4 +35,13 @@ class Callsign:
     cq_zone: int | None = None
     # ITU zone in which the callsign indicates they are operating
     itu_zone: int | None = None
+    # Location source. This can be "HOME QTH" or "DXCC" depending on which provider gave us a location
+    location_source: str | None = None
 
+    def fully_populated(self):
+        """Utility method to indicate that the callsign data is fully populated. Multiple providers can return data for
+        a callsign, and we try them in sequence until we have all the data we can, in which case there's no point
+        querying any other providers."""
+        return self.home_call is not None and self.name is not None and self.qth is not None and self.grid is not None\
+            and self.latitude is not None and self.longitude is not None and self.country is not None and self.continent\
+            is not None and self.dxcc_id is not None and self.cq_zone is not None and self.itu_zone is not None

data/spot.py

@@ -12,7 +12,7 @@ from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 from core.config import MAX_SPOT_AGE
 from core.constants import MODE_ALIASES, PROPAGATION_MODES
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
-from core.call_lookup_helper import lookup_helper
+from core.call_lookup_helper import get_call_info
 from core.sig_utils import ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.utils import infer_band_from_freq, infer_mode_from_comment, \
@@ -173,12 +173,13 @@ class Spot:
                     self.dx_ssid = split[1]
 
             # DX country, continent etc. from callsign
+            dx_call_info = get_call_info(self.dx_call, credentials)
             if self.dx_call and not self.dx_country:
-                self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_call, credentials)
+                self.dx_country = dx_call_info.country
             if self.dx_call and not self.dx_continent:
-                self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_call, credentials)
+                self.dx_continent = dx_call_info.continent
             if self.dx_call and not self.dx_dxcc_id:
-                self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_call, credentials)
+                self.dx_dxcc_id = dx_call_info.dxcc_id
             if self.dx_dxcc_id and not self.dx_flag:
                 self.dx_flag = get_flag_for_dxcc(self.dx_dxcc_id)
 
@@ -205,14 +206,15 @@ class Spot:
 
             # Spotter country, continent, zones etc. from callsign.
             # DE call with no digits, or APRS servers starting "T2" are not things we can look up location for
+            de_call_info = get_call_info(self.de_call, credentials)
             if self.de_call and any(char.isdigit() for char in self.de_call) and not (
                     self.de_call.startswith("T2") and self.source == "APRS-IS"):
                 if not self.de_country:
-                    self.de_country = lookup_helper.infer_country_from_callsign(self.de_call, credentials)
+                    self.de_country = de_call_info.country
                 if not self.de_continent:
-                    self.de_continent = lookup_helper.infer_continent_from_callsign(self.de_call, credentials)
+                    self.de_continent = de_call_info.continent
                 if not self.de_dxcc_id:
-                    self.de_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.de_call, credentials)
+                    self.de_dxcc_id = de_call_info.dxcc_id
                 if self.de_dxcc_id and not self.de_flag:
                     self.de_flag = get_flag_for_dxcc(self.de_dxcc_id)
 
@@ -359,18 +361,16 @@ class Spot:
                 self_copy.received_time_iso = ""
                 self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
 
-            # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data
-            # from the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon
+            # DX operator details lookup. This should be the last resort compared to taking the data from the actual
+            # spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon or DXCC lat/lon
             # instead of the one from the park reference they're at.
             if self.dx_call and not self.dx_name:
-                self.dx_name = lookup_helper.infer_name_from_callsign_online_lookup(self.dx_call, credentials)
+                self.dx_name = dx_call_info.name
             if self.dx_call and not self.dx_latitude:
-                latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.dx_call, credentials)
-                if latlon:
-                    self.dx_latitude = latlon[0]
-                    self.dx_longitude = latlon[1]
-                self.dx_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.dx_call, credentials)
-                self.dx_location_source = "HOME QTH"
+                self.dx_latitude = dx_call_info.latitude
+                self.dx_longitude = dx_call_info.longitude
+                self.dx_grid = dx_call_info.grid
+                self.dx_location_source = dx_call_info.location_source
 
             # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable string,
             # otherwise see what they have set on an online lookup service.
@@ -380,39 +380,19 @@ class Spot:
                     qth += " " + self.sig_refs[0].name
                 self.dx_qth = qth
             else:
-                self.dx_qth = lookup_helper.infer_qth_from_callsign_online_lookup(self.dx_call, credentials)
-
-            # Last resort for getting a DX position, use the DXCC entity.
-            if self.dx_call and not self.dx_latitude:
-                latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.dx_call)
-                if latlon:
-                    self.dx_latitude = latlon[0]
-                    self.dx_longitude = latlon[1]
-                    self.dx_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.dx_call)
-                    self.dx_location_source = "DXCC"
-
-            # It looks like we can sometimes get a string into lat/lon, so try to parse as float, reject if not valid
-            if isinstance(self.dx_latitude, str) or isinstance(self.dx_longitude, str):
-                try:
-                    self.dx_latitude = float(str(self.dx_latitude))
-                    self.dx_longitude = float(str(self.dx_longitude))
-                except (TypeError, ValueError):
-                    logging.warning("Received non-numeric strings in lat/lon (" + str(self.dx_latitude) + ", " + str(
-                        self.dx_longitude) + ") for call " + str(self.dx_call) + ", rejecting it")
-                    self.dx_latitude = None
-                    self.dx_longitude = None
+                self.dx_qth = dx_call_info.qth
 
             # CQ and ITU zone lookup, preferably from location but failing that, from callsign
             if not self.dx_cq_zone:
                 if self.dx_latitude:
                     self.dx_cq_zone = lat_lon_to_cq_zone(self.dx_latitude, self.dx_longitude)
                 elif self.dx_call:
-                    self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_call, credentials)
+                    self.dx_cq_zone = dx_call_info.cq_zone
             if not self.dx_itu_zone:
                 if self.dx_latitude:
                     self.dx_itu_zone = lat_lon_to_itu_zone(self.dx_latitude, self.dx_longitude)
                 elif self.dx_call:
-                    self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_call, credentials)
+                    self.dx_itu_zone = dx_call_info.itu_zone
 
             # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
             # is likely at home.
@@ -424,21 +404,11 @@ class Spot:
             # DE with no digits and APRS servers starting "T2" are not things we can look up location for
             if self.de_call and any(char.isdigit() for char in self.de_call) and not (
                     self.de_call.startswith("T2") and self.source == "APRS-IS"):
-                # DE operator position lookup, using QRZ.com/HamQTH.
+                # DE operator location lookup
                 if not self.de_latitude:
-                    latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.de_call, credentials)
-                    if latlon:
-                        self.de_latitude = latlon[0]
-                        self.de_longitude = latlon[1]
-                    self.de_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.de_call, credentials)
-
-                # Last resort for getting a DE position, use the DXCC entity.
-                if not self.de_latitude:
-                    latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.de_call)
-                    if latlon:
-                        self.de_latitude = latlon[0]
-                        self.de_longitude = latlon[1]
-                        self.de_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.de_call)
+                    self.de_latitude = de_call_info.latitude
+                    self.de_longitude = de_call_info.longitude
+                    self.de_grid = de_call_info.grid
 
         except Exception as e:
             logging.error("Exception while inferring missing data from spot", e, exc_info=True)

providers/callsigndata/api_query_callsign_data_provider.py

@@ -18,6 +18,9 @@ class APIQueryCallsignDataProvider(CallsignDataProvider):
         """ Set up the provider."""
         super().__init__(name, provider_config, storage)
 
+        if self.enabled:
+            self.status = "Ready"
+
     def start(self):
         pass
 

providers/callsigndata/callsign_data_provider.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 
 import pytz
@@ -15,8 +16,10 @@ class CallsignDataProvider:
 
         self.name = name
         self.enabled = provider_config["enabled"]
+        self.priority = int(provider_config["priority"])
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
+        self.lookup_count = 0
         self._storage = storage
 
     def start(self):
@@ -38,13 +41,16 @@ class CallsignDataProvider:
         possible. Data is cached internally for a set period of 30 days to avoid the need to request data from servers
         each time."""
 
-        if callsign in self._storage:
-            return self._storage[callsign]
+        if self.enabled:
+            if callsign in self._storage:
+                return self._storage[callsign]
+            else:
+                c = self._perform_new_lookup(callsign, lookup_credentials)
+                if c:
+                    self._storage.set(callsign, c, expire=DATA_STORE.CALLSIGN_DATA_TTL_SEC)
+                return c
         else:
-            c = self._perform_new_lookup(callsign, lookup_credentials)
-            if c:
-                self._storage.set(callsign, c, expire=DATA_STORE.CALLSIGN_DATA_TTL_SEC)
-            return c
+            return None
 
 
     def _perform_new_lookup(self, callsign, lookup_credentials):

providers/callsigndata/clublogapi.py

@@ -28,16 +28,18 @@ class ClublogAPI(APIQueryCallsignDataProvider):
 
         super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
 
-        self.status = "Ready"
-
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
         callsign_data = Callsign(call=callsign)
 
         try:
-            callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
-            self.status = "OK"
-            self.last_update_time = datetime.now(pytz.UTC)
+            if self._callinfo:
+                callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                self.lookup_count += 1
+            else:
+                return None
 
         except Exception as e:
             self.status = "Error"

providers/callsigndata/clublogxml.py

@@ -1,10 +1,13 @@
 import gzip
 import logging
+from datetime import datetime
 
+import pytz
 from pyhamtools import LookupLib, Callinfo
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -49,4 +52,18 @@ class ClublogXML(FileDownloadCallsignDataProvider):
             return False
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+        callsign_data = Callsign(call=callsign)
+
+        try:
+            if self._callinfo:
+                callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+                self.status = "OK"
+                self.lookup_count += 1
+            else:
+                return None
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from Clublog XML data", e, exc_info=True)
+
+        return callsign_data

providers/callsigndata/countryfiles.py

@@ -4,6 +4,7 @@ from pyhamtools import LookupLib, Callinfo
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -30,4 +31,18 @@ class CountryFiles(FileDownloadCallsignDataProvider):
             return False
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+        callsign_data = Callsign(call=callsign)
+
+        try:
+            if self._callinfo:
+                callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+                self.status = "OK"
+                self.lookup_count += 1
+            else:
+                return None
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from Country file", e, exc_info=True)
+
+        return callsign_data

providers/callsigndata/file_download_callsign_data_provider.py

@@ -24,6 +24,9 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
         self._stop_event = Event()
         self._url_data_cache = URLDataCache("callsigndata_" + name)
 
+        if self.enabled:
+            self.status = "Ready"
+
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.

providers/callsigndata/hamqth.py

@@ -1,7 +1,8 @@
 import logging
 import urllib.parse
-from datetime import timedelta
+from datetime import timedelta, datetime
 
+import pytz
 import xmltodict
 from pyhamtools import callinfo
 from requests import ConnectTimeout, ReadTimeout
@@ -27,19 +28,19 @@ class HamQTH(APIQueryCallsignDataProvider):
         # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
         self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/hamqth-creds",
                                                 expire_after=timedelta(minutes=55))
-        self.status = "Ready"
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        # If we don't have HamQTH credentials, skip this lookup
-        if not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
-                or lookup_credentials.hamqth_session_key):
+        # If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because
+        #         # someone might provide credentials next time around.
+        if not lookup_credentials or not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
+                or lookup_credentials.hamqth_session_id):
             return None
 
         try:
             # Obtain session key from credentials, by looking it up from username & password if necessary.
-            session_key = None
-            if lookup_credentials.hamqth_session_key:
-                session_key = lookup_credentials.hamqth_session_key
+            session_id = None
+            if lookup_credentials.hamqth_session_id:
+                session_id = lookup_credentials.hamqth_session_id
             elif lookup_credentials.hamqth_username and lookup_credentials.hamqth_password:
                 try:
                     session_data = self._CREDENTIALS_CACHE.get(
@@ -48,7 +49,7 @@ class HamQTH(APIQueryCallsignDataProvider):
                         headers=HTTP_HEADERS).content
                     dict_data = xmltodict.parse(session_data)
                     if "session_id" in dict_data["HamQTH"]["session"]:
-                        session_key = str(dict_data["HamQTH"]["session"]["session_id"])
+                        session_id = str(dict_data["HamQTH"]["session"]["session_id"])
                     else:
                         # Log this failure at debug level only, not our problem if user entered the wrong password.
                         logging.debug("HamQTH login details incorrect, failed to look up with HamQTH.")
@@ -57,7 +58,7 @@ class HamQTH(APIQueryCallsignDataProvider):
                     logging.error("Exception when getting HamQTH session key")
                     return None
 
-            if not session_key:
+            if not session_id:
                 return None
 
             # Try the call as given, then fall back to the base call (strips /P, /M etc.)
@@ -73,11 +74,14 @@ class HamQTH(APIQueryCallsignDataProvider):
             for lookup_call in calls_to_try:
                 try:
                     response = self._URL_DATA_CACHE.get(
-                        self._HAMQTH_BASE_URL + "?id=" + session_key + "&callsign=" + urllib.parse.quote_plus(
+                        self._HAMQTH_BASE_URL + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                             lookup_call) + "&prg=" + self._PRG, headers=HTTP_HEADERS, timeout=10)
                     if response.ok:
-                        data = xmltodict.parse(response.content)["HamQTH"]["search"]
+                        self.status = "OK"
+                        self.last_update_time = datetime.now(pytz.UTC)
+                        self.lookup_count += 1
                         # Found data, convert it to our object and return it
+                        data = xmltodict.parse(response.content)["HamQTH"]["search"]
                         return self.hamqth_response_to_callsign(callsign, data)
 
                     elif not response.from_cache:
@@ -134,4 +138,5 @@ class HamQTH(APIQueryCallsignDataProvider):
                         grid=grid,
                         dxcc_id=int(data["adif"]) if "adif" in data else None,
                         cq_zone=int(data["cq"]) if "cq" in data else None,
-                        itu_zone=int(data["itu"]) if "itu" in data else None)
+                        itu_zone=int(data["itu"]) if "itu" in data else None,
+                        location_source="HOME QTH")

providers/callsigndata/qrz.py

@@ -1,7 +1,8 @@
 import logging
 import urllib.parse
-from datetime import timedelta
+from datetime import timedelta, datetime
 
+import pytz
 import xmltodict
 from pyhamtools import callinfo
 from requests import ConnectTimeout, ReadTimeout
@@ -25,11 +26,11 @@ class QRZ(APIQueryCallsignDataProvider):
         # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
         self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/qrz-creds",
                                                 expire_after=timedelta(minutes=55))
-        self.status = "Ready"
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        # If we don't have QRZ credentials, skip this lookup
-        if not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
+        # If we don't have QRZ credentials, skip this lookup. Return None so we don't *cache* the lack of data, because
+        # someone might provide credentials next time around.
+        if not lookup_credentials or not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
                 or lookup_credentials.qrz_session_key):
             return None
 
@@ -78,6 +79,9 @@ class QRZ(APIQueryCallsignDataProvider):
                         qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
                         if qrz_response:
                             if "Callsign" in qrz_response:
+                                self.status = "OK"
+                                self.last_update_time = datetime.now(pytz.UTC)
+                                self.lookup_count += 1
                                 # Found data, convert it to our object and return it
                                 return self.qrz_response_to_callsign(callsign, qrz_response.get("Callsign"))
 
@@ -153,4 +157,5 @@ class QRZ(APIQueryCallsignDataProvider):
                         grid=grid,
                         dxcc_id=int(data["adif"]) if "adif" in data else None,
                         cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
-                        itu_zone=int(data["ituzone"]) if "ituzone" in data else None)
+                        itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
+                        location_source="HOME QTH")

server/handlers/api/lookups.py

@@ -8,15 +8,15 @@ import tornado
 from tornado import httputil
 from tornado.web import Application
 
+from core.call_lookup_helper import get_call_info
 from core.constants import SIGS
 from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.prometheus_metrics_handler import api_requests_counter
-from core.sig_utils import get_ref_regex_for_sig
 from core.sig_lookup_helper import populate_missing_sig_ref_info
+from core.sig_utils import get_ref_regex_for_sig
 from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 from data.sig_ref import SIGRef
-from data.spot import Spot
 
 
 class APILookupCallHandler(tornado.web.RequestHandler):
@@ -45,27 +45,9 @@ class APILookupCallHandler(tornado.web.RequestHandler):
             if "call" in query_params.keys():
                 call = str(query_params.get("call")).upper()
                 if re.match(r"^[A-Z0-9/\-]*$", call):
-                    # Take the callsign, make a "fake spot" so we can run infer_missing() on it, then repack the
-                    # resulting data in the correct way for the API response.
                     credentials = extract_credentials(query_params)
-                    fake_spot = Spot(dx_call=call)
-                    fake_spot.infer_missing(credentials)
-                    data = {
-                        "call": call,
-                        "name": fake_spot.dx_name,
-                        "qth": fake_spot.dx_qth,
-                        "country": fake_spot.dx_country,
-                        "flag": fake_spot.dx_flag,
-                        "continent": fake_spot.dx_continent,
-                        "dxcc_id": fake_spot.dx_dxcc_id,
-                        "cq_zone": fake_spot.dx_cq_zone,
-                        "itu_zone": fake_spot.dx_itu_zone,
-                        "grid": fake_spot.dx_grid,
-                        "latitude": fake_spot.dx_latitude,
-                        "longitude": fake_spot.dx_longitude,
-                        "location_source": fake_spot.dx_location_source
-                    }
-                    self.write(safe_json_dumps(data))
+                    callsign_data = get_call_info(call, credentials)
+                    self.write(safe_json_dumps(callsign_data))
 
                 else:
                     self.write(safe_json_dumps("Error - '" + call + "' does not look like a valid callsign."))

server/sse_broadcaster.py

@@ -11,6 +11,9 @@ class SSEBroadcaster:
     def __init__(self):
         self._handlers = set()
         self._lock = threading.Lock()
+        self._loop = None
+
+    def bind_to_web_server_loop(self):
         self._loop = IOLoop.current()
 
     def register(self, handler):
@@ -22,9 +25,9 @@ class SSEBroadcaster:
             self._handlers.discard(handler)
 
     def publish(self, value):
-        self._loop.add_callback(self._fan_out, value)
+         self._loop.add_callback(self._broadcast, value)
 
-    def _fan_out(self, value):
+    def _broadcast(self, value):
         with self._lock:
             handlers = list(self._handlers)
         for handler in handlers:

server/webserver.py

@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import os
+import threading
 
 import tornado
 from tornado.web import StaticFileHandler
@@ -61,6 +62,10 @@ class WebServer:
     async def _start_inner(self):
         """Start method (async). Sets up the Tornado application."""
 
+        # Bind the SSE broadcasters to the web server's loop, so they fire correctly
+        self._spot_broadcaster.bind_to_web_server_loop()
+        self._alert_broadcaster.bind_to_web_server_loop()
+
         # Prepare a list of common arguments that are passed in to every API & page handler. This is just a basic thing
         # to avoid copy-pasting the same thing to every route declaration below.
         handler_opts = {"web_server_metrics": self.web_server_metrics}

spothole.py

@@ -4,7 +4,6 @@ import os
 import signal
 import sys
 
-from core.call_lookup_helper import lookup_helper
 from core.config import SERVER_OWNER_CALLSIGN, LOG_LEVEL
 from core.constants import SOFTWARE_VERSION
 from core.data_providers import DATA_PROVIDERS
@@ -49,9 +48,6 @@ if __name__ == '__main__':
     # Set up data store
     DATA_STORE.setup()
 
-    # Set up lookup helper
-    lookup_helper.start()
-
     # Set up and start data providers
     DATA_PROVIDERS.setup()
     DATA_PROVIDERS.start()

static/apidocs/openapi.yml

@@ -325,7 +325,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/CallLookup'
+                $ref: '#/components/schemas/CallsignData'
         '422':
           description: Validation error e.g. callsign missing or format incorrect
           content:
@@ -1791,6 +1791,10 @@ components:
             The last time at which this provider received data, UTC seconds since UNIX epoch. If this
             is zero, the provider has never updated.
           example: 1759579508
+        lookup_count:
+          type: number
+          description: The number of callsign lookups performed using this provider since the server was started.
+          example: 1234
 
     SpotList:
       type: array
@@ -1962,12 +1966,16 @@ components:
             on this server.
           example: true
 
-    CallLookup:
+    CallsignData:
       type: object
       properties:
         call:
           type: string
           description: Callsign, as provided to the API
+          example: DL/M0TRT/P
+        home_call:
+          type: string
+          description: The "home" call, without prefixes or suffixes
           example: M0TRT
         name:
           type: string

static/js/status.js

@@ -65,6 +65,7 @@ function loadStatus() {
                     <div class="col"><strong>${p["name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
+                    <div class="col">Lookups: ${(p["enabled"] && p["lookup_count"] > 0) ? p["lookup_count"] : "N/A"}</div>
                 </div>`);
         });
     });