Refactor of caching & data storage part 17 #118

config-example.yml

@@ -251,32 +251,44 @@ sig-ref-data-providers:
 
 
 # Callsign data providers to use. These allow Spothole to provide information about callsigns, either from static
-# lookups or from web-based services such as QRZ.
+# lookups or from web-based services such as QRZ. Note that use of QRZ and HamQTH is *on behalf of the user*, and
+# requires the user to enter their credentials into the website UI or provide them in an API call. Spothole will not
+# look up all calls via QRZ/HamQTH using the server owner's credentials, as this is against their usage policy.
 callsign-data-providers:
-  - class: "CountryFiles"
+  - class: "QRZ"
     enabled: true
+    # Callsign data providers can often provide conflicting data, e.g. a home location from QRZ vs the centre of the
+    # DXCC entity from a country file lookup. The priority flag sets which source "wins" in the event of conflict.
+    # Lower numbers take priority over higher numbers. Generally then, QRZ/HamQTH should have low numbers as they are
+    # likely to have the most accurate data.
+    priority: 1
+    # No server-side credentials for QRZ. Users must provide their own as per QRZ policy.
 
-  - class: "ClublogXML"
+  - class: "HamQTH"
     enabled: true
-    # API key for Clublog to look up information. Required in order to enable this provider. You will need to request
+    priority: 2
-    # one via their helpdesk portal if you want to use callsign lookups from Clublog.
+    # No server-side credentials for HamQTH. Users must provide their own.
-    api-key: ""
 
   - class: "ClublogAPI"
     # Querying the Clublog API directly doesn't provide any more data than the XML version, it just provides slightly
     # more up-to-date information in the rare case that the prefix data changes, at a significant cost of looking up
     # every callsign via an API call. Normally left disabled but it exists as an option.
     enabled: false
-    # API key for Clublog to look up information. Required in order to enable this provider.
+    priority: 3
+    # API key for Clublog to look up information. Required in order to enable this provider. Unlike QRZ and HamQTH,
+    # Clublog uses an API key issued to Spothole, not to the end user.
     api-key: ""
 
-  - class: "QRZ"
+  - class: "ClublogXML"
     enabled: true
-    # No server-side credentials for QRZ. Users must provide their own as per QRZ policy.
+    priority: 4
+    # API key for Clublog to look up information. Required in order to enable this provider. You will need to request
+    # one via their helpdesk portal if you want to use callsign lookups from Clublog.
+    api-key: ""
 
-  - class: "HamQTH"
+  - class: "CountryFiles"
+    priority: 5
     enabled: true
-    # No server-side credentials for HamQTH. Users must provide their own.
 
 
 # Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week

core/call_lookup_helper.py

@@ -1,21 +1,4 @@
-import gzip
-import logging
-import urllib.parse
-from datetime import timedelta
-
-import xmltodict
-from diskcache import Cache
-from pyhamtools import LookupLib, Callinfo, callinfo
-from pyhamtools.exceptions import APIKeyMissingError
-from pyhamtools.locator import latlong_to_locator
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
-from requests_cache import CachedSession
-
-from core.config import config
-from core.constants import HTTP_HEADERS, HAMQTH_PRG
 from core.data_providers import DATA_PROVIDERS
-from core.data_store import DATA_STORE
-from core.url_data_cache import URLDataCache
 from data.callsign import Callsign
 
 
@@ -24,14 +7,24 @@ def get_call_info(callsign, lookup_credentials):
     lookup_credentials is an optional object that carries the user's QRZ.com/HamQTH credentials, if they provided them,
     to enable lookup using those providers."""
 
-    callsign = Callsign(call=callsign)
+    callsign_data = Callsign(call=callsign)
-    for p in DATA_PROVIDERS.callsign_data_providers:
-        # Get new lookup data
-        data = p.lookup(callsign, lookup_credentials)
-        if data:
-            # Merge in turn, replacing any existing content where we have it.
-            for key, value in data.__dict__.items():
-                if value is not None:
-                    callsign.__dict__[key] = value
 
-    return callsign
+    if callsign:
+        # Sort callsign providers by priority order, so we query the highest priority (lowest numbers) first, and only
+        # query other providers for data we are missing as we go along.
+        for p in sorted(DATA_PROVIDERS.callsign_data_providers, key=lambda p: p.priority):
+            if p.enabled:
+                # Get new lookup data
+                data = p.lookup(callsign, lookup_credentials)
+                if data:
+                    # If we have new data for fields that were previously unpopulated, add them in
+                    for key, value in data.__dict__.items():
+                        if value is not None and callsign_data.__dict__.get(key) is None:
+                            callsign_data.__dict__[key] = value
+
+                    # If callsign data is fully populated, avoid looking up using other providers as their data will not
+                    # be used
+                    if callsign_data.fully_populated():
+                        break
+
+    return callsign_data

core/config.py

@@ -30,6 +30,8 @@ LOG_WEB_REQUESTS = config.get("log-web-requests", False)
 # but for consistency we provide this to the front-end in web-ui-options because it has no impact outside of the web UI.
 WEB_UI_OPTIONS["spot-providers-enabled-by-default"] = [p["name"] for p in config["spot-providers"] if p["enabled"] and (
         "enabled-by-default-in-web-ui" not in p or p["enabled-by-default-in-web-ui"])]
+WEB_UI_OPTIONS["qrz-enabled"] = any(p["class"] == "QRZ" and p["enabled"] for p in config["callsign-data-providers"])
+WEB_UI_OPTIONS["hamqth-enabled"] = any(p["class"] == "HamQTH" and p["enabled"] for p in config["callsign-data-providers"])
 # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
 # one of our proviers. We set that to also be enabled by default.
 if ALLOW_SPOTTING:

core/data_providers.py

@@ -29,15 +29,7 @@ class DataProviders:
 
 
     def start(self):
-        for p in self.spot_providers:
+        # Start data providers before spot/alert providers so the lookup data is there already for incoming spots
-            if p.enabled:
-                p.start()
-        for p in self.alert_providers:
-            if p.enabled:
-                p.start()
-        for p in self.solar_condition_providers:
-            if p.enabled:
-                p.start()
         for p in self.static_data_providers:
             if p.enabled:
                 p.start()
@@ -47,6 +39,15 @@ class DataProviders:
         for p in self.callsign_data_providers:
             if p.enabled:
                 p.start()
+        for p in self.spot_providers:
+            if p.enabled:
+                p.start()
+        for p in self.alert_providers:
+            if p.enabled:
+                p.start()
+        for p in self.solar_condition_providers:
+            if p.enabled:
+                p.start()
 
     def stop(self):
         for sp in self.spot_providers:

core/live_data_cache.py

@@ -92,7 +92,7 @@ class LiveDataCache:
                 time.sleep(interval)
                 self.save_snapshot()
 
-        t = threading.Thread(target=loop, daemon=True, name=f"snapshot-{self._snapshot_dir}")
+        t = threading.Thread(target=loop, name=f"LiveDataCache-Snapshot-{self._snapshot_dir}")
         t.start()
 
     def close(self):

core/status_reporter.py

@@ -30,7 +30,7 @@ class StatusReporter:
     def start(self):
         """Start the reporter thread"""
 
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name="StatusReporter")
         self._thread.start()
 
     def stop(self):
@@ -84,7 +84,8 @@ class StatusReporter:
         DATA_STORE.status_data["callsign_data_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
+                           "lookup_count": p.lookup_count},
                 DATA_PROVIDERS.callsign_data_providers))
         DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"],
                                                "last_api_access": WEB_SERVER.web_server_metrics[

core/utils.py

@@ -2,6 +2,7 @@ import logging
 
 import simplejson
 from pyhamtools.frequency import freq_to_band
+from pyhamtools.locator import latlong_to_locator
 
 from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES
 from core.data_store import DATA_STORE
@@ -84,23 +85,33 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
     """Utility function to take the data provided by a PyHamTools CallInfo object and populate our own Callsign data
     object from it"""
 
-    home_call = callinfo.get_homecall(callsign)
+    try:
-    data = callinfo.get_all()
+        home_call = callinfo.get_homecall(callsign)
+        data = callinfo.get_all(callsign)
 
-    country = data["country"] if "country" in data else None
+        country = data["country"] if "country" in data else None
-    dxcc_id = data["adif"] if "adif" in data else None
+        dxcc_id = data["adif"] if "adif" in data else None
-    continent = data["continent"] if "continent" in data else None
+        continent = data["continent"] if "continent" in data else None
-    cq_zone = data["cqz"] if "cqz" in data else None
+        cq_zone = data["cqz"] if "cqz" in data else None
-    itu_zone = data["ituz"] if "ituz" in data else None
+        itu_zone = data["ituz"] if "ituz" in data else None
-    lat = float(data["latitude"]) if "latitude" in data else None
+        lat = float(data["latitude"]) if "latitude" in data else None
-    lon = float(data["longitude"]) if "longitude" in data else None
+        lon = float(data["longitude"]) if "longitude" in data else None
+        grid = None
+        if lat and lon:
+            grid = latlong_to_locator(lat, lon)
 
-    return Callsign(call=callsign,
+        return Callsign(call=callsign,
-                    home_call=home_call,
+                        home_call=home_call,
-                    country=country,
+                        country=country,
-                    dxcc_id=dxcc_id,
+                        dxcc_id=dxcc_id,
-                    continent=continent,
+                        continent=continent,
-                    cq_zone=cq_zone,
+                        cq_zone=cq_zone,
-                    itu_zone=itu_zone,
+                        itu_zone=itu_zone,
-                    latitude=lat,
+                        latitude=lat,
-                    longitude=lon)
+                        longitude=lon,
+                        grid=grid,
+                        location_source="DXCC")
+
+    except (KeyError, ValueError):
+        # Unknown callsign, can't look anything up, return a Callsign object with basic data so that gets cached
+        return Callsign(call=callsign)

data/alert.py

@@ -7,7 +7,7 @@ from datetime import datetime, timedelta
 
 import pytz
 
-from core.call_lookup_helper import lookup_helper
+from core.call_lookup_helper import get_call_info
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.utils import get_flag_for_dxcc
 
@@ -86,16 +86,17 @@ class Alert:
 
             # DX country, continent, zones etc. from callsign. CQ/ITU zone are better looked up with a location but we don't
             # have a real location for alerts.
+            call_info = get_call_info(self.dx_calls[0], credentials)
             if self.dx_calls and self.dx_calls[0] and not self.dx_country:
-                self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_calls[0], credentials)
+                self.dx_country = call_info.country
             if self.dx_calls and self.dx_calls[0] and not self.dx_continent:
-                self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_calls[0], credentials)
+                self.dx_continent = call_info.continent
             if self.dx_calls and self.dx_calls[0] and not self.dx_cq_zone:
-                self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_calls[0], credentials)
+                self.dx_cq_zone = call_info.cq_zone
             if self.dx_calls and self.dx_calls[0] and not self.dx_itu_zone:
-                self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_calls[0], credentials)
+                self.dx_itu_zone = call_info.itu_zone
             if self.dx_calls and self.dx_calls[0] and not self.dx_dxcc_id:
-                self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_calls[0], credentials)
+                self.dx_dxcc_id = call_info.dxcc_id
             if self.dx_dxcc_id and not self.dx_flag:
                 self.dx_flag = get_flag_for_dxcc(self.dx_dxcc_id)
 
@@ -124,12 +125,10 @@ class Alert:
                 self_copy.received_time_iso = ""
                 self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
 
-            # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data
+            # DX operator name lookup, using QRZ.com/HamQTH.
-            # from the actual alerting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon
-            # instead of the one from the park reference they're at.
             if self.dx_calls and not self.dx_names:
                 self.dx_names = list(
-                    map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c, credentials), self.dx_calls))
+                    map(lambda c: get_call_info(c, credentials).name, self.dx_calls))
 
         except Exception as e:
             logging.error("Exception while inferring missing data from spot", e, exc_info=True)

data/callsign.py

@@ -35,4 +35,13 @@ class Callsign:
     cq_zone: int | None = None
     # ITU zone in which the callsign indicates they are operating
     itu_zone: int | None = None
+    # Location source. This can be "HOME QTH" or "DXCC" depending on which provider gave us a location
+    location_source: str | None = None
 
+    def fully_populated(self):
+        """Utility method to indicate that the callsign data is fully populated. Multiple providers can return data for
+        a callsign, and we try them in sequence until we have all the data we can, in which case there's no point
+        querying any other providers."""
+        return self.home_call is not None and self.name is not None and self.qth is not None and self.grid is not None\
+            and self.latitude is not None and self.longitude is not None and self.country is not None and self.continent\
+            is not None and self.dxcc_id is not None and self.cq_zone is not None and self.itu_zone is not None

data/spot.py

@@ -12,7 +12,7 @@ from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 from core.config import MAX_SPOT_AGE
 from core.constants import MODE_ALIASES, PROPAGATION_MODES
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
-from core.call_lookup_helper import lookup_helper
+from core.call_lookup_helper import get_call_info
 from core.sig_utils import ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.utils import infer_band_from_freq, infer_mode_from_comment, \
@@ -173,12 +173,13 @@ class Spot:
                     self.dx_ssid = split[1]
 
             # DX country, continent etc. from callsign
+            dx_call_info = get_call_info(self.dx_call, credentials)
             if self.dx_call and not self.dx_country:
-                self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_call, credentials)
+                self.dx_country = dx_call_info.country
             if self.dx_call and not self.dx_continent:
-                self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_call, credentials)
+                self.dx_continent = dx_call_info.continent
             if self.dx_call and not self.dx_dxcc_id:
-                self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_call, credentials)
+                self.dx_dxcc_id = dx_call_info.dxcc_id
             if self.dx_dxcc_id and not self.dx_flag:
                 self.dx_flag = get_flag_for_dxcc(self.dx_dxcc_id)
 
@@ -205,14 +206,15 @@ class Spot:
 
             # Spotter country, continent, zones etc. from callsign.
             # DE call with no digits, or APRS servers starting "T2" are not things we can look up location for
+            de_call_info = get_call_info(self.de_call, credentials)
             if self.de_call and any(char.isdigit() for char in self.de_call) and not (
                     self.de_call.startswith("T2") and self.source == "APRS-IS"):
                 if not self.de_country:
-                    self.de_country = lookup_helper.infer_country_from_callsign(self.de_call, credentials)
+                    self.de_country = de_call_info.country
                 if not self.de_continent:
-                    self.de_continent = lookup_helper.infer_continent_from_callsign(self.de_call, credentials)
+                    self.de_continent = de_call_info.continent
                 if not self.de_dxcc_id:
-                    self.de_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.de_call, credentials)
+                    self.de_dxcc_id = de_call_info.dxcc_id
                 if self.de_dxcc_id and not self.de_flag:
                     self.de_flag = get_flag_for_dxcc(self.de_dxcc_id)
 
@@ -359,18 +361,16 @@ class Spot:
                 self_copy.received_time_iso = ""
                 self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
 
-            # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data
+            # DX operator details lookup. This should be the last resort compared to taking the data from the actual
-            # from the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon
+            # spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon or DXCC lat/lon
             # instead of the one from the park reference they're at.
             if self.dx_call and not self.dx_name:
-                self.dx_name = lookup_helper.infer_name_from_callsign_online_lookup(self.dx_call, credentials)
+                self.dx_name = dx_call_info.name
             if self.dx_call and not self.dx_latitude:
-                latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.dx_call, credentials)
+                self.dx_latitude = dx_call_info.latitude
-                if latlon:
+                self.dx_longitude = dx_call_info.longitude
-                    self.dx_latitude = latlon[0]
+                self.dx_grid = dx_call_info.grid
-                    self.dx_longitude = latlon[1]
+                self.dx_location_source = dx_call_info.location_source
-                self.dx_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.dx_call, credentials)
-                self.dx_location_source = "HOME QTH"
 
             # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable string,
             # otherwise see what they have set on an online lookup service.
@@ -380,39 +380,19 @@ class Spot:
                     qth += " " + self.sig_refs[0].name
                 self.dx_qth = qth
             else:
-                self.dx_qth = lookup_helper.infer_qth_from_callsign_online_lookup(self.dx_call, credentials)
+                self.dx_qth = dx_call_info.qth
-
-            # Last resort for getting a DX position, use the DXCC entity.
-            if self.dx_call and not self.dx_latitude:
-                latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.dx_call)
-                if latlon:
-                    self.dx_latitude = latlon[0]
-                    self.dx_longitude = latlon[1]
-                    self.dx_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.dx_call)
-                    self.dx_location_source = "DXCC"
-
-            # It looks like we can sometimes get a string into lat/lon, so try to parse as float, reject if not valid
-            if isinstance(self.dx_latitude, str) or isinstance(self.dx_longitude, str):
-                try:
-                    self.dx_latitude = float(str(self.dx_latitude))
-                    self.dx_longitude = float(str(self.dx_longitude))
-                except (TypeError, ValueError):
-                    logging.warning("Received non-numeric strings in lat/lon (" + str(self.dx_latitude) + ", " + str(
-                        self.dx_longitude) + ") for call " + str(self.dx_call) + ", rejecting it")
-                    self.dx_latitude = None
-                    self.dx_longitude = None
 
             # CQ and ITU zone lookup, preferably from location but failing that, from callsign
             if not self.dx_cq_zone:
                 if self.dx_latitude:
                     self.dx_cq_zone = lat_lon_to_cq_zone(self.dx_latitude, self.dx_longitude)
                 elif self.dx_call:
-                    self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_call, credentials)
+                    self.dx_cq_zone = dx_call_info.cq_zone
             if not self.dx_itu_zone:
                 if self.dx_latitude:
                     self.dx_itu_zone = lat_lon_to_itu_zone(self.dx_latitude, self.dx_longitude)
                 elif self.dx_call:
-                    self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_call, credentials)
+                    self.dx_itu_zone = dx_call_info.itu_zone
 
             # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
             # is likely at home.
@@ -424,21 +404,11 @@ class Spot:
             # DE with no digits and APRS servers starting "T2" are not things we can look up location for
             if self.de_call and any(char.isdigit() for char in self.de_call) and not (
                     self.de_call.startswith("T2") and self.source == "APRS-IS"):
-                # DE operator position lookup, using QRZ.com/HamQTH.
+                # DE operator location lookup
                 if not self.de_latitude:
-                    latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.de_call, credentials)
+                    self.de_latitude = de_call_info.latitude
-                    if latlon:
+                    self.de_longitude = de_call_info.longitude
-                        self.de_latitude = latlon[0]
+                    self.de_grid = de_call_info.grid
-                        self.de_longitude = latlon[1]
-                    self.de_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.de_call, credentials)
-
-                # Last resort for getting a DE position, use the DXCC entity.
-                if not self.de_latitude:
-                    latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.de_call)
-                    if latlon:
-                        self.de_latitude = latlon[0]
-                        self.de_longitude = latlon[1]
-                        self.de_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.de_call)
 
         except Exception as e:
             logging.error("Exception while inferring missing data from spot", e, exc_info=True)

providers/alert/http_alert_provider.py

@@ -25,7 +25,7 @@ class HTTPAlertProvider(AlertProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info("Set up query of " + self.name + " alert API every " + str(self._poll_interval) + " seconds.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}")
         self._thread.start()
 
     def stop(self):

providers/callsigndata/api_query_callsign_data_provider.py

@@ -18,6 +18,9 @@ class APIQueryCallsignDataProvider(CallsignDataProvider):
         """ Set up the provider."""
         super().__init__(name, provider_config, storage)
 
+        if self.enabled:
+            self.status = "Ready"
+
     def start(self):
         pass
 

providers/callsigndata/callsign_data_provider.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 
 import pytz
@@ -15,8 +16,10 @@ class CallsignDataProvider:
 
         self.name = name
         self.enabled = provider_config["enabled"]
+        self.priority = int(provider_config["priority"])
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
+        self.lookup_count = 0
         self._storage = storage
 
     def start(self):
@@ -38,13 +41,16 @@ class CallsignDataProvider:
         possible. Data is cached internally for a set period of 30 days to avoid the need to request data from servers
         each time."""
 
-        if callsign in self._storage:
+        if self.enabled:
-            return self._storage[callsign]
+            if callsign in self._storage:
+                return self._storage[callsign]
+            else:
+                c = self._perform_new_lookup(callsign, lookup_credentials)
+                if c:
+                    self._storage.set(callsign, c, expire=DATA_STORE.CALLSIGN_DATA_TTL_SEC)
+                return c
         else:
-            c = self._perform_new_lookup(callsign, lookup_credentials)
+            return None
-            if c:
-                self._storage.set(callsign, c, expire=DATA_STORE.CALLSIGN_DATA_TTL_SEC)
-            return c
 
 
     def _perform_new_lookup(self, callsign, lookup_credentials):

providers/callsigndata/clublogapi.py

@@ -28,16 +28,18 @@ class ClublogAPI(APIQueryCallsignDataProvider):
 
         super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
 
-        self.status = "Ready"
-
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
         callsign_data = Callsign(call=callsign)
 
         try:
-            callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+            if self._callinfo:
-            self.status = "OK"
+                callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
-            self.last_update_time = datetime.now(pytz.UTC)
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                self.lookup_count += 1
+            else:
+                return None
 
         except Exception as e:
             self.status = "Error"

providers/callsigndata/clublogxml.py

@@ -1,10 +1,13 @@
 import gzip
 import logging
+from datetime import datetime
 
+import pytz
 from pyhamtools import LookupLib, Callinfo
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -49,4 +52,18 @@ class ClublogXML(FileDownloadCallsignDataProvider):
             return False
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+        callsign_data = Callsign(call=callsign)
+
+        try:
+            if self._callinfo:
+                callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+                self.status = "OK"
+                self.lookup_count += 1
+            else:
+                return None
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from Clublog XML data", e, exc_info=True)
+
+        return callsign_data

providers/callsigndata/countryfiles.py

@@ -4,6 +4,7 @@ from pyhamtools import LookupLib, Callinfo
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -30,4 +31,18 @@ class CountryFiles(FileDownloadCallsignDataProvider):
             return False
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+        callsign_data = Callsign(call=callsign)
+
+        try:
+            if self._callinfo:
+                callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+                self.status = "OK"
+                self.lookup_count += 1
+            else:
+                return None
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from Country file", e, exc_info=True)
+
+        return callsign_data

providers/callsigndata/file_download_callsign_data_provider.py

@@ -24,12 +24,15 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
         self._stop_event = Event()
         self._url_data_cache = URLDataCache("callsigndata_" + name)
 
+        if self.enabled:
+            self.status = "Ready"
+
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info(
             "Set up query of " + self.name + " callsign reference data every " + str(self._poll_interval) + " days.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}")
         self._thread.start()
 
     def stop(self):

providers/callsigndata/hamqth.py

@@ -1,7 +1,8 @@
 import logging
 import urllib.parse
-from datetime import timedelta
+from datetime import timedelta, datetime
 
+import pytz
 import xmltodict
 from pyhamtools import callinfo
 from requests import ConnectTimeout, ReadTimeout
@@ -27,19 +28,19 @@ class HamQTH(APIQueryCallsignDataProvider):
         # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
         self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/hamqth-creds",
                                                 expire_after=timedelta(minutes=55))
-        self.status = "Ready"
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        # If we don't have HamQTH credentials, skip this lookup
+        # If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because
-        if not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
+        #         # someone might provide credentials next time around.
-                or lookup_credentials.hamqth_session_key):
+        if not lookup_credentials or not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
+                or lookup_credentials.hamqth_session_id):
             return None
 
         try:
             # Obtain session key from credentials, by looking it up from username & password if necessary.
-            session_key = None
+            session_id = None
-            if lookup_credentials.hamqth_session_key:
+            if lookup_credentials.hamqth_session_id:
-                session_key = lookup_credentials.hamqth_session_key
+                session_id = lookup_credentials.hamqth_session_id
             elif lookup_credentials.hamqth_username and lookup_credentials.hamqth_password:
                 try:
                     session_data = self._CREDENTIALS_CACHE.get(
@@ -48,7 +49,7 @@ class HamQTH(APIQueryCallsignDataProvider):
                         headers=HTTP_HEADERS).content
                     dict_data = xmltodict.parse(session_data)
                     if "session_id" in dict_data["HamQTH"]["session"]:
-                        session_key = str(dict_data["HamQTH"]["session"]["session_id"])
+                        session_id = str(dict_data["HamQTH"]["session"]["session_id"])
                     else:
                         # Log this failure at debug level only, not our problem if user entered the wrong password.
                         logging.debug("HamQTH login details incorrect, failed to look up with HamQTH.")
@@ -57,7 +58,7 @@ class HamQTH(APIQueryCallsignDataProvider):
                     logging.error("Exception when getting HamQTH session key")
                     return None
 
-            if not session_key:
+            if not session_id:
                 return None
 
             # Try the call as given, then fall back to the base call (strips /P, /M etc.)
@@ -73,11 +74,14 @@ class HamQTH(APIQueryCallsignDataProvider):
             for lookup_call in calls_to_try:
                 try:
                     response = self._URL_DATA_CACHE.get(
-                        self._HAMQTH_BASE_URL + "?id=" + session_key + "&callsign=" + urllib.parse.quote_plus(
+                        self._HAMQTH_BASE_URL + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                             lookup_call) + "&prg=" + self._PRG, headers=HTTP_HEADERS, timeout=10)
                     if response.ok:
-                        data = xmltodict.parse(response.content)["HamQTH"]["search"]
                         # Found data, convert it to our object and return it
+                        data = xmltodict.parse(response.content)["HamQTH"]["search"]
+                        self.status = "OK"
+                        self.last_update_time = datetime.now(pytz.UTC)
+                        self.lookup_count += 1
                         return self.hamqth_response_to_callsign(callsign, data)
 
                     elif not response.from_cache:
@@ -134,4 +138,5 @@ class HamQTH(APIQueryCallsignDataProvider):
                         grid=grid,
                         dxcc_id=int(data["adif"]) if "adif" in data else None,
                         cq_zone=int(data["cq"]) if "cq" in data else None,
-                        itu_zone=int(data["itu"]) if "itu" in data else None)
+                        itu_zone=int(data["itu"]) if "itu" in data else None,
+                        location_source="HOME QTH")

providers/callsigndata/qrz.py

@@ -1,7 +1,8 @@
 import logging
 import urllib.parse
-from datetime import timedelta
+from datetime import timedelta, datetime
 
+import pytz
 import xmltodict
 from pyhamtools import callinfo
 from requests import ConnectTimeout, ReadTimeout
@@ -25,11 +26,11 @@ class QRZ(APIQueryCallsignDataProvider):
         # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
         self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/qrz-creds",
                                                 expire_after=timedelta(minutes=55))
-        self.status = "Ready"
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        # If we don't have QRZ credentials, skip this lookup
+        # If we don't have QRZ credentials, skip this lookup. Return None so we don't *cache* the lack of data, because
-        if not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
+        # someone might provide credentials next time around.
+        if not lookup_credentials or not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
                 or lookup_credentials.qrz_session_key):
             return None
 
@@ -78,8 +79,12 @@ class QRZ(APIQueryCallsignDataProvider):
                         qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
                         if qrz_response:
                             if "Callsign" in qrz_response:
+                                qrz_data = qrz_response.get("Callsign")
+                                self.status = "OK"
+                                self.last_update_time = datetime.now(pytz.UTC)
+                                self.lookup_count += 1
                                 # Found data, convert it to our object and return it
-                                return self.qrz_response_to_callsign(callsign, qrz_response.get("Callsign"))
+                                return self.qrz_response_to_callsign(callsign, qrz_data)
 
                             elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
                                 # Errors here are normally just "callsign not in database", no need to log that ourselves
@@ -153,4 +158,5 @@ class QRZ(APIQueryCallsignDataProvider):
                         grid=grid,
                         dxcc_id=int(data["adif"]) if "adif" in data else None,
                         cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
-                        itu_zone=int(data["ituzone"]) if "ituzone" in data else None)
+                        itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
+                        location_source="HOME QTH")

providers/sigrefdata/file_download_sig_ref_data_provider.py

@@ -28,7 +28,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info(
             "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " days.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}")
         self._thread.start()
 
     def stop(self):

providers/solarconditions/giroionosonde.py

@@ -58,7 +58,7 @@ class GIROIonosonde(SolarConditionsProvider):
 
     def start(self):
         logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider")
         self._thread.start()
 
     def stop(self):

providers/solarconditions/http_solar_conditions_provider.py

@@ -24,7 +24,7 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
     def start(self):
         logging.info(
             "Set up query of " + self.name + " solar conditions API every " + str(self._poll_interval) + " seconds.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}")
         self._thread.start()
 
     def stop(self):

providers/solarconditions/kc2gprop.py

@@ -30,7 +30,7 @@ class KC2GProp(SolarConditionsProvider):
 
     def start(self):
         logging.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name="KC2GPropProvider")
         self._thread.start()
 
     def stop(self):

providers/spot/aprsis.py

@@ -15,7 +15,7 @@ class APRSIS(SpotProvider):
 
     def __init__(self, provider_config):
         super().__init__("APRS-IS", provider_config)
-        self._thread = Thread(target=self._connect)
+        self._thread = Thread(target=self._connect, name="APRSISSpotProvider")
         self._thread.daemon = True
         self._aprsis = None
 

providers/spot/dxcluster.py

@@ -36,7 +36,7 @@ class DXCluster(SpotProvider):
         self._allow_rbn_spots = provider_config["allow_rbn_spots"] if "allow_rbn_spots" in provider_config else False
         self._spot_line_pattern = self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
         self._telnet = None
-        self._thread = Thread(target=self._handle)
+        self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}")
         self._thread.daemon = True
         self._running = True
 

providers/spot/http_spot_provider.py

@@ -26,7 +26,7 @@ class HTTPSpotProvider(SpotProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info("Set up query of " + self.name + " spot API every " + str(self._poll_interval) + " seconds.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}")
         self._thread.start()
 
     def stop(self):

providers/spot/rbn.py

@@ -27,7 +27,7 @@ class RBN(SpotProvider):
         super().__init__(name, provider_config)
         self._port = provider_config["port"]
         self._telnet = None
-        self._thread = Thread(target=self._handle)
+        self._thread = Thread(target=self._handle, name=f"RBNSpotProvider-{self.name}")
         self._thread.daemon = True
         self._running = True
 

providers/spot/sse_spot_provider.py

@@ -1,7 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Thread
+from threading import Event, Lock, Thread
-from time import sleep
 
 import pytz
 from requests_sse import EventSource
@@ -16,24 +15,35 @@ class SSESpotProvider(SpotProvider):
     def __init__(self, name, provider_config, url):
         super().__init__(name, provider_config)
         self._url = url
-        self._event_source = None
         self._thread = None
-        self._stopped = False
         self._last_event_id = None
+        self._stop_event = Event()
+        self._event_source_lock = Lock()
+        self._event_source = None
 
     def start(self):
         logging.info("Set up SSE connection to " + self.name + " spot API.")
-        self._stopped = False
+        self._stop_event.clear()
-        self._thread = Thread(target=self._run)
+        self._thread = Thread(target=self._run, name=f"SSESpotProvider-{self.name}")
         self._thread.daemon = True
         self._thread.start()
 
     def stop(self):
-        self._stopped = True
+        self._stop_event.set()
-        if self._event_source:
+
-            self._event_source.close()
+        with self._event_source_lock:
+            event_source = self._event_source
+        if event_source:
+            try:
+                event_source.close()
+            except Exception:
+                logging.exception(
+                    "Exception closing SSE connection for " + self.name + " during stop()")
+
         if self._thread:
-            self._thread.join()
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logging.warning(self.name + " SSE worker thread did not exit on time and will be killed.")
 
     def _on_open(self):
         self.status = "Waiting for Data"
@@ -41,36 +51,45 @@ class SSESpotProvider(SpotProvider):
     def _on_error(self):
         self.status = "Connecting"
 
+    def _set_event_source(self, event_source):
+        with self._event_source_lock:
+            self._event_source = event_source
+
     def _run(self):
-        while not self._stopped:
+        while not self._stop_event.is_set():
             try:
                 logging.debug("Connecting to " + self.name + " spot API...")
                 self.status = "Connecting"
-                with EventSource(self._url, headers=HTTP_HEADERS, latest_event_id=self._last_event_id, timeout=30,
+                with EventSource(self._url, headers=HTTP_HEADERS, latest_event_id=self._last_event_id, timeout=10,
                                  on_open=self._on_open, on_error=self._on_error) as event_source:
-                    self._event_source = event_source
+                    self._set_event_source(event_source)
-                    for event in self._event_source:
+                    try:
-                        if event.type == 'message':
+                        for event in event_source:
-                            try:
+                            if self._stop_event.is_set():
-                                self._last_event_id = event.last_event_id
+                                break
-                                new_spot = self._sse_message_to_spot(event.data)
+                            if event.type == 'message':
-                                if new_spot:
+                                try:
-                                    self._submit(new_spot)
+                                    self._last_event_id = event.last_event_id
+                                    new_spot = self._sse_message_to_spot(event.data)
+                                    if new_spot:
+                                        self._submit(new_spot)
 
-                                self.status = "OK"
+                                    self.status = "OK"
-                                self.last_update_time = datetime.now(pytz.UTC)
+                                    self.last_update_time = datetime.now(pytz.UTC)
-                                logging.debug("Received data from " + self.name + " spot API.")
+                                    logging.debug("Received data from " + self.name + " spot API.")
 
-                            except Exception:
+                                except Exception:
-                                logging.exception(
+                                    logging.exception(
-                                    "Exception processing message from SSE Spot Provider (" + self.name + ")")
+                                        "Exception processing message from SSE Spot Provider (" + self.name + ")")
+                    finally:
+                        self._set_event_source(None)
 
             except Exception:
                 self.status = "Error"
                 logging.exception("Exception in SSE Spot Provider (" + self.name + ")")
             else:
                 self.status = "Disconnected"
-            sleep(5)  # Wait before trying to reconnect
+            self._stop_event.wait(timeout=5) # Wait before trying to reconnect
 
     def _sse_message_to_spot(self, message_data):
         """Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass

providers/spot/websocket_spot_provider.py

@@ -24,7 +24,7 @@ class WebsocketSpotProvider(SpotProvider):
     def start(self):
         logging.info("Set up websocket connection to " + self.name + " spot API.")
         self._stopped = False
-        self._thread = Thread(target=self._run)
+        self._thread = Thread(target=self._run, name=f"WebsocketSpotProvider-{self.name}")
         self._thread.daemon = True
         self._thread.start()
 

providers/staticdata/file_download_static_data_provider.py

@@ -29,7 +29,7 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info(
             "Set up query of " + self.name + " static reference data every " + str(self._poll_interval) + " days.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}")
         self._thread.start()
 
     def stop(self):

server/handlers/api/lookups.py

@@ -8,15 +8,15 @@ import tornado
 from tornado import httputil
 from tornado.web import Application
 
+from core.call_lookup_helper import get_call_info
 from core.constants import SIGS
 from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.prometheus_metrics_handler import api_requests_counter
-from core.sig_utils import get_ref_regex_for_sig
 from core.sig_lookup_helper import populate_missing_sig_ref_info
+from core.sig_utils import get_ref_regex_for_sig
 from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 from data.sig_ref import SIGRef
-from data.spot import Spot
 
 
 class APILookupCallHandler(tornado.web.RequestHandler):
@@ -45,27 +45,9 @@ class APILookupCallHandler(tornado.web.RequestHandler):
             if "call" in query_params.keys():
                 call = str(query_params.get("call")).upper()
                 if re.match(r"^[A-Z0-9/\-]*$", call):
-                    # Take the callsign, make a "fake spot" so we can run infer_missing() on it, then repack the
-                    # resulting data in the correct way for the API response.
                     credentials = extract_credentials(query_params)
-                    fake_spot = Spot(dx_call=call)
+                    callsign_data = get_call_info(call, credentials)
-                    fake_spot.infer_missing(credentials)
+                    self.write(safe_json_dumps(callsign_data))
-                    data = {
-                        "call": call,
-                        "name": fake_spot.dx_name,
-                        "qth": fake_spot.dx_qth,
-                        "country": fake_spot.dx_country,
-                        "flag": fake_spot.dx_flag,
-                        "continent": fake_spot.dx_continent,
-                        "dxcc_id": fake_spot.dx_dxcc_id,
-                        "cq_zone": fake_spot.dx_cq_zone,
-                        "itu_zone": fake_spot.dx_itu_zone,
-                        "grid": fake_spot.dx_grid,
-                        "latitude": fake_spot.dx_latitude,
-                        "longitude": fake_spot.dx_longitude,
-                        "location_source": fake_spot.dx_location_source
-                    }
-                    self.write(safe_json_dumps(data))
 
                 else:
                     self.write(safe_json_dumps("Error - '" + call + "' does not look like a valid callsign."))

server/handlers/api/options.py

@@ -38,10 +38,12 @@ class APIOptionsHandler(tornado.web.RequestHandler):
                        "mode_types": MODE_TYPES,
                        "sigs": SIGS,
                        # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle things that aren't even available.
-                       "spot_sources": list(
+                       "spot_providers": list(
                            map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["spot_providers"]))),
-                       "alert_sources": list(
+                       "alert_providers": list(
                            map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"]))),
+                       "callsign_data_providers": list(
+                           map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["callsign_data_providers"]))),
                        "continents": CONTINENTS,
                        "propagation_modes": list(PROPAGATION_MODES.values()),
                        "max_spot_age": MAX_SPOT_AGE,
@@ -49,7 +51,7 @@ class APIOptionsHandler(tornado.web.RequestHandler):
             # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
             # one of our proviers.
             if ALLOW_SPOTTING:
-                options["spot_sources"].append("API")
+                options["spot_providers"].append("API")
 
             self.write(safe_json_dumps(options))
             self.set_status(200)

server/sse_broadcaster.py

@@ -11,6 +11,9 @@ class SSEBroadcaster:
     def __init__(self):
         self._handlers = set()
         self._lock = threading.Lock()
+        self._loop = None
+
+    def bind_to_web_server_loop(self):
         self._loop = IOLoop.current()
 
     def register(self, handler):
@@ -22,9 +25,9 @@ class SSEBroadcaster:
             self._handlers.discard(handler)
 
     def publish(self, value):
-        self._loop.add_callback(self._fan_out, value)
+         self._loop.add_callback(self._broadcast, value)
 
-    def _fan_out(self, value):
+    def _broadcast(self, value):
         with self._lock:
             handlers = list(self._handlers)
         for handler in handlers:

server/webserver.py

@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import os
+import threading
 
 import tornado
 from tornado.web import StaticFileHandler
@@ -61,6 +62,10 @@ class WebServer:
     async def _start_inner(self):
         """Start method (async). Sets up the Tornado application."""
 
+        # Bind the SSE broadcasters to the web server's loop, so they fire correctly
+        self._spot_broadcaster.bind_to_web_server_loop()
+        self._alert_broadcaster.bind_to_web_server_loop()
+
         # Prepare a list of common arguments that are passed in to every API & page handler. This is just a basic thing
         # to avoid copy-pasting the same thing to every route declaration below.
         handler_opts = {"web_server_metrics": self.web_server_metrics}

spothole.py

@@ -4,7 +4,6 @@ import os
 import signal
 import sys
 
-from core.call_lookup_helper import lookup_helper
 from core.config import SERVER_OWNER_CALLSIGN, LOG_LEVEL
 from core.constants import SOFTWARE_VERSION
 from core.data_providers import DATA_PROVIDERS
@@ -49,9 +48,6 @@ if __name__ == '__main__':
     # Set up data store
     DATA_STORE.setup()
 
-    # Set up lookup helper
-    lookup_helper.start()
-
     # Set up and start data providers
     DATA_PROVIDERS.setup()
     DATA_PROVIDERS.start()

static/apidocs/openapi.yml

@@ -17,7 +17,11 @@ info:
     
     ### 2.0
     
-    * Added `sig_ref_data_providers`, `static_data_providers` and `callsign_data_providers` to status and removed `cleanup`
+    * Added `sig_ref_data_providers`, `static_data_providers` and `callsign_data_providers` to `/status` response
+    * Added `callsign_data_providers` to `/options` response
+    * BREAKING: Removed `cleanup` from `/status` response
+    * BREAKING: in the `/options` response, renamed `spot_sources` and `alert_sources` to `spot_providers` and
+      `alert_providers`
     
     ### 1.4
     
@@ -325,7 +329,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/CallLookup'
+                $ref: '#/components/schemas/CallsignData'
         '422':
           description: Validation error e.g. callsign missing or format incorrect
           content:
@@ -1791,6 +1795,10 @@ components:
             The last time at which this provider received data, UTC seconds since UNIX epoch. If this
             is zero, the provider has never updated.
           example: 1759579508
+        lookup_count:
+          type: number
+          description: The number of callsign lookups performed using this provider since the server was started.
+          example: 1234
 
     SpotList:
       type: array
@@ -1930,12 +1938,24 @@ components:
           description: An array of all the supported Special Interest Groups.
           items:
             $ref: '#/components/schemas/SIG'
-        sources:
+        spot_providers:
           type: array
-          description: An array of all the supported data sources.
+          description: An array of all the supported spot data sources.
           items:
             type: string
             example: "Cluster"
+        alert_providers:
+          type: array
+          description: An array of all the supported alert data sources.
+          items:
+            type: string
+            example: "POTA"
+        callsign_data_providers:
+          type: array
+          description: An array of all the supported callsign lookup providers.
+          items:
+            type: string
+            example: "QRZ.com"
         continents:
           type: array
           description: An array of all the supported continents.
@@ -1962,12 +1982,16 @@ components:
             on this server.
           example: true
 
-    CallLookup:
+    CallsignData:
       type: object
       properties:
         call:
           type: string
           description: Callsign, as provided to the API
+          example: DL/M0TRT/P
+        home_call:
+          type: string
+          description: The "home" call, without prefixes or suffixes
           example: M0TRT
         name:
           type: string

static/js/alerts.js

@@ -288,7 +288,7 @@ function loadOptions() {
 
         // Populate the filters panel
         generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
-        generateMultiToggleFilterCard("#source-options", "source", options["alert_sources"]);
+        generateMultiToggleFilterCard("#source-options", "source", options["alert_providers"]);
 
         // Load URL params. These may select things from the various filter & display options, so the function needs
         // to be called after these are set up, but if the URL params ask for "embedded mode", this will suppress

static/js/bands.js

@@ -291,7 +291,7 @@ function loadOptions() {
         generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
         generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
         generateModesMultiToggleFilterCard(options["modes"]);
-        generateSourcesMultiToggleFilterCard(options["spot_sources"], spotProvidersEnabledByDefault);
+        generateSourcesMultiToggleFilterCard(options["spot_providers"], spotProvidersEnabledByDefault);
 
         // Load URL params. These may select things from the various filter & display options, so the function needs
         // to be called after these are set up, but if the URL params ask for "embedded mode", this will suppress

static/js/map.js

@@ -322,7 +322,7 @@ function loadOptions() {
         generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
         generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
         generateModesMultiToggleFilterCard(options["modes"]);
-        generateSourcesMultiToggleFilterCard(options["spot_sources"], spotProvidersEnabledByDefault);
+        generateSourcesMultiToggleFilterCard(options["spot_providers"], spotProvidersEnabledByDefault);
 
         // Load URL params. These may select things from the various filter & display options, so the function needs
         // to be called after these are set up, but if the URL params ask for "embedded mode", this will suppress

static/js/spots.js

@@ -436,7 +436,7 @@ function loadOptions() {
         generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
         generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
         generateModesMultiToggleFilterCard(options["modes"]);
-        generateSourcesMultiToggleFilterCard(options["spot_sources"], spotProvidersEnabledByDefault);
+        generateSourcesMultiToggleFilterCard(options["spot_providers"], spotProvidersEnabledByDefault);
 
         // Load URL params. These may select things from the various filter & display options, so the function needs
         // to be called after these are set up, but if the URL params ask for "embedded mode", this will suppress

static/js/status.js

@@ -14,7 +14,7 @@ function loadStatus() {
 
         jsonData["spot_providers"].forEach(p => {
             $("#spot-providers-status-container").append(`
-                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
                     <div class="col"><strong>${p["name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
@@ -24,7 +24,7 @@ function loadStatus() {
 
         jsonData["alert_providers"].forEach(p => {
             $("#alert-providers-status-container").append(`
-                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
                     <div class="col"><strong>${p["name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
@@ -33,7 +33,7 @@ function loadStatus() {
 
         jsonData["solar_condition_providers"].forEach(p => {
             $("#condition-providers-status-container").append(`
-                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
                     <div class="col"><strong>${p["name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
@@ -42,7 +42,7 @@ function loadStatus() {
 
         jsonData["static_data_providers"].forEach(p => {
             $("#static-data-providers-status-container").append(`
-                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
                     <div class="col"><strong>${p["name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
@@ -51,7 +51,7 @@ function loadStatus() {
 
         jsonData["sig_ref_data_providers"].forEach(p => {
             $("#sig-ref-data-providers-status-container").append(`
-                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
                     <div class="col"><strong>${p["sig_name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
@@ -61,10 +61,11 @@ function loadStatus() {
 
         jsonData["callsign_data_providers"].forEach(p => {
             $("#callsign-data-providers-status-container").append(`
-                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
                     <div class="col"><strong>${p["name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
+                    <div class="col">Lookups: ${(p["enabled"] && p["lookup_count"] > 0) ? p["lookup_count"] : "N/A"}</div>
                 </div>`);
         });
     });

templates/alerts.html

@@ -8,7 +8,8 @@
         </div>
         <div class="col-auto">
             <div class="d-inline-flex gap-1">
-                {% module Template("widgets/filters-display-data-buttons.html", web_ui_options=web_ui_options) %}
+                {% module Template("widgets/filters-display-data-buttons.html", web_ui_options=web_ui_options,
+                show_data_button=web_ui_options["qrz-enabled"] or web_ui_options["hamqth-enabled"]) %}
             </div>
         </div>
     </div>
@@ -50,19 +51,25 @@
         </div>
     </div>
 
+    {% if web_ui_options["qrz-enabled"] or web_ui_options["hamqth-enabled"] %}
     <div id="data-area" class="appearing-panel card mb-3">
         {% module Template("widgets/data-area-header.html", web_ui_options=web_ui_options) %}
         <div class="card-body">
             <div class="row row-cols-1 row-cols-md-4 g-4">
+                {% if web_ui_options["qrz-enabled"] %}
                 <div class="col">
                     {% module Template("cards/qrz.html", web_ui_options=web_ui_options) %}
                 </div>
+                {% end %}
+                {% if web_ui_options["hamqth-enabled"] %}
                 <div class="col">
                     {% module Template("cards/hamqth.html", web_ui_options=web_ui_options) %}
                 </div>
+                {% end %}
             </div>
         </div>
     </div>
+    {% end %}
 
     <div id="table-container">
         <table id="table" class="table">

templates/bands.html

@@ -6,7 +6,8 @@
         <div class="col-auto me-auto pt-3"></div>
         <div class="col-auto">
             <div class="d-inline-flex gap-1">
-                {% module Template("widgets/filters-display-data-buttons.html", web_ui_options=web_ui_options) %}
+                {% module Template("widgets/filters-display-data-buttons.html", web_ui_options=web_ui_options,
+                show_data_button=web_ui_options["qrz-enabled"] or web_ui_options["hamqth-enabled"]) %}
             </div>
         </div>
     </div>
@@ -54,19 +55,25 @@
         </div>
     </div>
 
+    {% if web_ui_options["qrz-enabled"] or web_ui_options["hamqth-enabled"] %}
     <div id="data-area" class="appearing-panel card mb-3">
         {% module Template("widgets/data-area-header.html", web_ui_options=web_ui_options) %}
         <div class="card-body">
             <div class="row row-cols-1 row-cols-md-4 g-4">
+                {% if web_ui_options["qrz-enabled"] %}
                 <div class="col">
                     {% module Template("cards/qrz.html", web_ui_options=web_ui_options) %}
                 </div>
+                {% end %}
+                {% if web_ui_options["hamqth-enabled"] %}
                 <div class="col">
                     {% module Template("cards/hamqth.html", web_ui_options=web_ui_options) %}
                 </div>
+                {% end %}
             </div>
         </div>
     </div>
+    {% end %}
 
     <div id="bands-container"></div>
 

templates/map.html

@@ -20,7 +20,8 @@
             <div class="col-auto me-auto pt-3"></div>
             <div class="col-auto">
                 <div class="d-inline-flex gap-1">
-                    {% module Template("widgets/filters-display-data-buttons.html", web_ui_options=web_ui_options) %}
+                    {% module Template("widgets/filters-display-data-buttons.html", web_ui_options=web_ui_options,
+                show_data_button=web_ui_options["qrz-enabled"] or web_ui_options["hamqth-enabled"]) %}
                 </div>
             </div>
         </div>
@@ -74,19 +75,25 @@
             </div>
         </div>
 
+        {% if web_ui_options["qrz-enabled"] or web_ui_options["hamqth-enabled"] %}
         <div id="data-area" class="appearing-panel card mb-3">
             {% module Template("widgets/data-area-header.html", web_ui_options=web_ui_options) %}
             <div class="card-body">
                 <div class="row row-cols-1 row-cols-md-4 g-4">
+                    {% if web_ui_options["qrz-enabled"] %}
                     <div class="col">
                         {% module Template("cards/qrz.html", web_ui_options=web_ui_options) %}
                     </div>
+                    {% end %}
+                    {% if web_ui_options["hamqth-enabled"] %}
                     <div class="col">
                         {% module Template("cards/hamqth.html", web_ui_options=web_ui_options) %}
                     </div>
+                    {% end %}
                 </div>
             </div>
         </div>
+        {% end %}
     </div>
 </div>
 

templates/spots.html

@@ -25,7 +25,8 @@
         <div class="col-md-8 text-end">
             <div class="d-inline-flex gap-3">
                 {% module Template("widgets/search.html", web_ui_options=web_ui_options) %}
-                {% module Template("widgets/filters-display-data-buttons.html", web_ui_options=web_ui_options) %}
+                {% module Template("widgets/filters-display-data-buttons.html", web_ui_options=web_ui_options,
+                show_data_button=web_ui_options["qrz-enabled"] or web_ui_options["hamqth-enabled"]) %}
             </div>
         </div>
     </div>
@@ -86,12 +87,16 @@
         {% module Template("widgets/data-area-header.html", web_ui_options=web_ui_options) %}
         <div class="card-body">
             <div class="row row-cols-1 row-cols-md-4 g-4">
+                {% if web_ui_options["qrz-enabled"] %}
                 <div class="col">
                     {% module Template("cards/qrz.html", web_ui_options=web_ui_options) %}
                 </div>
+                {% end %}
+                {% if web_ui_options["hamqth-enabled"] %}
                 <div class="col">
                     {% module Template("cards/hamqth.html", web_ui_options=web_ui_options) %}
                 </div>
+                {% end %}
                 <div class="col">
                     {% module Template("cards/location.html", web_ui_options=web_ui_options) %}
                 </div>

templates/status.html

@@ -6,19 +6,19 @@
         Spothole
     </div>
     <div class="card-body">
-        <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+        <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
             <div class="col"><strong>Metadata</strong></div>
             <div class="col">Software Version: <span id="software-version"></span></div>
             <div class="col">Owner Callsign: <span id="server-owner-callsign"></span></div>
             <div class="col">Up since: <span id="up-since"></span></div>
         </div>
-        <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+        <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
             <div class="col"><strong>Performance</strong></div>
             <div class="col">Memory Use: <span id="memory-use"></span></div>
             <div class="col">Total Spots: <span id="total-spots"></span></div>
             <div class="col">Total Alerts: <span id="total-alerts"></span></div>
         </div>
-        <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+        <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
             <div class="col"><strong>Web Server</strong></div>
             <div class="col">Status: <span id="web-server-status"></span></div>
             <div class="col">Last API call: <span id="web-server-last-api"></span></div>

templates/widgets/filters-display-data-buttons.html

@@ -5,7 +5,9 @@
     <button id="display-button" type="button" class="btn btn-outline-secondary" data-bs-toggle="button"
             onclick="toggleDisplayPanel();"><i class="fa-solid fa-desktop"></i><span
             class="hideonmobile">&nbsp;Display</span></button>
+    {% if show_data_button %}
     <button id="data-button" type="button" class="btn btn-outline-secondary" data-bs-toggle="button"
             onclick="toggleDataPanel();"><i class="fa-solid fa-database"></i><span
             class="hideonmobile">&nbsp;Your data</span></button>
+    {% end %}
 </div>