Refactor of caching & data storage part 11 #118

config-example.yml

@@ -166,7 +166,23 @@ alert-providers:
     enabled: true
 
 
-# Static reference data providers to use. This allows Spothole to download data such as mapping between callsign
+# Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band
+# conditions, etc.) and make it available via the /api/v1/solar endpoint.
+solar-condition-providers:
+  - class: "HamQSL"
+    enabled: true
+
+  - class: "NOAA3dayForecast"
+    enabled: true
+
+  - class: "GIROIonosonde"
+    enabled: true
+
+  - class: "KC2GProp"
+    enabled: true
+
+
+# Static reference data providers to use. These allow Spothole to download data such as mapping between callsign
 # prefixes and DXCC entities.
 static-data-providers:
   - class: "K0SWE"
@@ -179,7 +195,7 @@ static-data-providers:
     enabled: true
 
 
-# SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF
+# SIG reference data providers to use. These allow Spothole to download, for example, the WWFF directory that maps WWFF
 # park IDs to their name and location.
 sig-ref-data-providers:
   - class: "POTA"
@@ -233,30 +249,30 @@ sig-ref-data-providers:
   - class: "Toilets"
     enabled: true
 
-# Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band
+
-# conditions, etc.) and make it available via the /api/v1/solar endpoint.
+# Callsign data providers to use. These allow Spothole to provide information about callsigns, either from static
-solar-condition-providers:
+# lookups or from web-based services such as QRZ.
-  - class: "HamQSL"
+callsign-data-providers:
+  - class: "CountryFiles"
     enabled: true
 
-  - class: "NOAA3dayForecast"
+  - class: "ClublogXML"
     enabled: true
+    # API key for Clublog to look up information. Required in order to enable this provider. You will need to request
+    # one via their helpdesk portal if you want to use callsign lookups from Clublog.
+    clublog-api-key: ""
 
-  - class: "GIROIonosonde"
+  - class: "ClublogAPI"
     enabled: true
+    # API key for Clublog to look up information. Required in order to enable this provider.
+    clublog-api-key: ""
 
-  - class: "KC2GProp"
-    enabled: true
 
 # Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week
 # for alerts.
 max-spot-age-sec: 3600
 max-alert-age-sec: 604800
 
-# API key for Clublog to look up information. Optional. You sill need to request one via their helpdesk portal if you
-# want to use callsign lookups from Clublog.
-clublog-api-key: ""
-
 # Allow submitting spots to the Spothole API?
 allow-spotting: true
 

core/config.py

@@ -36,41 +36,11 @@ if ALLOW_SPOTTING:
     WEB_UI_OPTIONS["spot-providers-enabled-by-default"].append("API")
 
 
-def get_spot_provider_from_config(config_providers_entry):
+def create_provider_from_config(package, config_providers_entry):
-    """Utility method to get a spot provider based on the class specified in its config entry."""
+    """Utility method to get a provider based on the class specified in its config entry. You must also provide the
+    package to look for it in, as there are several types of provider. e.g. package "providers.spot", where the config
+    entry is for a POTA spot provider."""
 
-    module = importlib.import_module('providers.spot.' + config_providers_entry["class"].lower())
+    module = importlib.import_module(package + "." + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_alert_provider_from_config(config_providers_entry):
-    """Utility method to get an alert provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('providers.alert.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_solar_conditions_provider_from_config(config_providers_entry):
-    """Utility method to get a solar conditions provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('providers.solarconditions.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_static_data_provider_from_config(config_providers_entry):
-    """Utility method to get a static reference data provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('providers.staticdata.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_sig_ref_data_provider_from_config(config_providers_entry):
-    """Utility method to get a SIG reference data provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('providers.sigrefdata.' + config_providers_entry["class"].lower())
     provider_class = getattr(module, config_providers_entry["class"])
     return provider_class(config_providers_entry)

core/constants.py

@@ -3,7 +3,7 @@ from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "1.4-pre"
+SOFTWARE_VERSION = "2.0-pre"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": "Spothole v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}

core/data_providers.py

@@ -0,0 +1,72 @@
+from core.config import config, create_provider_from_config
+
+
+class DataProviders:
+    """Global object for storing data providers."""
+
+    def __init__(self):
+        self.spot_providers = []
+        self.alert_providers = []
+        self.solar_condition_providers = []
+        self.static_data_providers = []
+        self.sig_ref_data_providers = []
+        self.callsign_data_providers = []
+
+
+    def setup(self):
+        for entry in config["spot-providers"]:
+            self.spot_providers.append(create_provider_from_config("providers.spot", entry))
+        for entry in config["alert-providers"]:
+            self.alert_providers.append(create_provider_from_config("providers.alert", entry))
+        for entry in config.get("solar-condition-providers", []):
+            self.solar_condition_providers.append(create_provider_from_config("providers.solarconditions", entry))
+        for entry in config.get("static-data-providers", []):
+            self.static_data_providers.append(create_provider_from_config("providers.staticdata", entry))
+        for entry in config.get("sig-ref-data-providers", []):
+            self.sig_ref_data_providers.append(create_provider_from_config("providers.sigrefdata", entry))
+        for entry in config.get("callsign-data-providers", []):
+            self.callsign_data_providers.append(create_provider_from_config("providers.callsigndata", entry))
+
+
+    def start(self):
+        for p in self.spot_providers:
+            if p.enabled:
+                p.start()
+        for p in self.alert_providers:
+            if p.enabled:
+                p.start()
+        for p in self.solar_condition_providers:
+            if p.enabled:
+                p.start()
+        for p in self.static_data_providers:
+            if p.enabled:
+                p.start()
+        for p in self.sig_ref_data_providers:
+            if p.enabled:
+                p.start()
+        for p in self.callsign_data_providers:
+            if p.enabled:
+                p.start()
+
+    def stop(self):
+        for sp in self.spot_providers:
+            if sp.enabled:
+                sp.stop()
+        for ap in self.alert_providers:
+            if ap.enabled:
+                ap.stop()
+        for scp in self.solar_condition_providers:
+            if scp.enabled:
+                scp.stop()
+        for srdp in self.sig_ref_data_providers:
+            if srdp.enabled:
+                srdp.stop()
+        for sdp in self.static_data_providers:
+            if sdp.enabled:
+                sdp.stop()
+        for cdp in self.callsign_data_providers:
+            if cdp.enabled:
+                cdp.stop()
+
+# Global object
+DATA_PROVIDERS = DataProviders()

core/data_store.py

@@ -28,9 +28,9 @@ class DataStore:
         self.dxcc_data = None
         self.dxcc_lookup_by_call_regex = []
         self.sigrefs = None
-        self.status_data = None
+        self.status_data = {}
         self._status = None
-        self.solar_conditions = None
+        self.solar_conditions = {}
         self._solar = None
         # ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
         # caches, they can just be straight objects

core/status_reporter.py

@@ -7,24 +7,19 @@ import pytz
 
 from core.config import SERVER_OWNER_CALLSIGN
 from core.constants import SOFTWARE_VERSION
+from core.data_providers import DATA_PROVIDERS
 from core.data_store import DATA_STORE
 from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge
+from server.webserver import WEB_SERVER
 
 
 class StatusReporter:
     """Provides a timed update of the application's status data."""
 
-    def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers,
+    def __init__(self, run_interval):
-                 static_data_providers, sig_ref_data_providers):
         """Constructor"""
 
         self._run_interval = run_interval
-        self._web_server = web_server
-        self._spot_providers = spot_providers
-        self._alert_providers = alert_providers
-        self._solar_condition_providers = solar_condition_providers
-        self._static_data_providers = static_data_providers
-        self._sig_ref_data_providers = sig_ref_data_providers
         self._thread = None
         self._stop_event = Event()
         self._startup_time = datetime.now(pytz.UTC)
@@ -64,39 +59,44 @@ class StatusReporter:
                                tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
                            "last_spot": p.last_spot_time.replace(
                                tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
-            self._spot_providers))
+                DATA_PROVIDERS.spot_providers))
         DATA_STORE.status_data["alert_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
                                tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-            self._alert_providers))
+                DATA_PROVIDERS.alert_providers))
         DATA_STORE.status_data["solar_condition_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
                                tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-            self._solar_condition_providers))
+                DATA_PROVIDERS.solar_condition_providers))
         DATA_STORE.status_data["static_data_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
                                tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-                self._static_data_providers))
+                DATA_PROVIDERS.static_data_providers))
         DATA_STORE.status_data["sig_ref_data_providers"] = list(
             map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
                                tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
                            "reference_count": p.reference_count},
-                self._sig_ref_data_providers))
+                DATA_PROVIDERS.sig_ref_data_providers))
-        DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
+        DATA_STORE.status_data["callsign_data_providers"] = list(
-                                      "last_api_access": self._web_server.web_server_metrics[
+            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+                           "last_updated": p.last_update_time.replace(
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                DATA_PROVIDERS.callsign_data_providers))
+        DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"],
+                                               "last_api_access": WEB_SERVER.web_server_metrics[
                                                    "last_api_access_time"].replace(
-                                          tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
+                                                   tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[
                                                    "last_api_access_time"] else 0,
-                                      "api_access_count": self._web_server.web_server_metrics["api_access_counter"],
+                                               "api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"],
-                                      "last_page_access": self._web_server.web_server_metrics[
+                                               "last_page_access": WEB_SERVER.web_server_metrics[
                                                    "last_page_access_time"].replace(
-                                          tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
+                                                   tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[
                                                    "last_page_access_time"] else 0,
-                                      "page_access_count": self._web_server.web_server_metrics[
+                                               "page_access_count": WEB_SERVER.web_server_metrics[
                                                    "page_access_counter"]}
 
         # Update Prometheus metrics

core/utils.py

@@ -5,6 +5,7 @@ from pyhamtools.frequency import freq_to_band
 
 from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES
 from core.data_store import DATA_STORE
+from data.callsign import Callsign
 
 
 def safe_json_dumps(obj):
@@ -77,3 +78,29 @@ def get_flag_for_dxcc(dxcc):
 
     dxcc_data = DATA_STORE.dxcc_data[dxcc] if dxcc in DATA_STORE.dxcc_data else None
     return dxcc_data["flag"] if dxcc_data else None
+
+
+def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
+    """Utility function to take the data provided by a PyHamTools CallInfo object and populate our own Callsign data
+    object from it"""
+
+    home_call = callinfo.get_homecall(callsign)
+    data = callinfo.get_all()
+
+    country = data["country"] if "country" in data else None
+    dxcc_id = data["adif"] if "adif" in data else None
+    continent = data["continent"] if "continent" in data else None
+    cq_zone = data["cqz"] if "cqz" in data else None
+    itu_zone = data["ituz"] if "ituz" in data else None
+    lat = float(data["latitude"]) if "latitude" in data else None
+    lon = float(data["longitude"]) if "longitude" in data else None
+
+    return Callsign(call=callsign,
+                    home_call=home_call,
+                    country=country,
+                    dxcc_id=dxcc_id,
+                    continent=continent,
+                    cq_zone=cq_zone,
+                    itu_zone=itu_zone,
+                    latitude=lat,
+                    longitude=lon)

data/callsign.py

@@ -11,7 +11,7 @@ class Callsign:
     # Callsign as spotted
     call: str
     # "Home" call, i.e. with any prefixes and suffixes stripped off
-    home_call: str
+    home_call: str | None = None
     # Operator name
     name : str | None = None
     # QTH (location), free text
@@ -26,13 +26,13 @@ class Callsign:
     # the centre of the country they're operating in if no other data is available.
     longitude : float | None = None
     # Country in which the callsign indicates they are operating
-    dx_country: str | None = None
+    country: str | None = None
     # Continent in which the callsign indicates they are operating
-    dx_continent: str | None = None
+    continent: str | None = None
     # DXCC ID in which the callsign indicates they are operating
-    dx_dxcc_id: int | None = None
+    dxcc_id: int | None = None
     # CQ zone in which the callsign indicates they are operating
-    dx_cq_zone: int | None = None
+    cq_zone: int | None = None
     # ITU zone in which the callsign indicates they are operating
-    dx_itu_zone: int | None = None
+    itu_zone: int | None = None
 

providers/callsigndata/api_query_callsign_data_provider.py

@@ -0,0 +1,25 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from providers.callsigndata.callsign_data_provider import CallsignDataProvider
+
+
+class APIQueryCallsignDataProvider(CallsignDataProvider):
+    """Generic callsign data provider class for providers that fetch their data from the web on-demand using an API."""
+
+    def __init__(self, name, provider_config):
+        """ Set up the provider."""
+        super().__init__(name, provider_config)
+
+    def start(self):
+        pass
+
+    def stop(self):
+        pass

providers/callsigndata/callsign_data_provider.py

@@ -0,0 +1,36 @@
+from datetime import datetime
+
+import pytz
+
+
+class CallsignDataProvider:
+    """Generic callsign reference data provider class. Subclasses of this set up the various mechanisms via which
+    Spothole can look up data for callsigns."""
+
+    def __init__(self, name, provider_config):
+        """Constructor"""
+
+        self.name = name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access remote resources, if
+        needed."""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def lookup(self, callsign, lookup_credentials):
+        """Looks up data for the provided callsign. Takes a LookupCredentials object, which provides any credentials
+        that have been provided by the user for this session (QRZ.com/HamQTH) to allow us to look up using those
+        services on the user's behalf. (Clublog is looked up using an API key owned by the server and provided in its
+        config file, so users need not provide their own.) Returns a Callsign object with as much data populated as
+        possible."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/callsigndata/clublogapi.py

@@ -0,0 +1,43 @@
+import logging
+from datetime import datetime
+
+import pytz
+from pyhamtools import LookupLib, Callinfo
+
+from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
+from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
+
+
+class ClublogAPI(APIQueryCallsignDataProvider):
+    """Callsign data provider for Clublog's API."""
+
+    _callinfo = None
+
+    def __init__(self, provider_config):
+        # API key required for this provider
+        self._api_key = provider_config.get("api-key", "")
+        if self._api_key != "":
+            lookuplib = LookupLib(lookuptype="clublogapi", apikey=self._api_key)
+            self._callinfo = Callinfo(lookuplib)
+        else:
+            provider_config["enabled"] = False
+            logging.warning(
+                "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
+
+        super().__init__("Clublog API", provider_config)
+
+
+    def lookup(self, callsign, lookup_credentials):
+        callsign_data = Callsign(call=callsign)
+
+        try:
+            callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+            self.status = "OK"
+            self.last_update_time = datetime.now(pytz.UTC)
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from Clublog API", e, exc_info=True)
+
+        return callsign_data

providers/callsigndata/clublogxml.py

@@ -0,0 +1,52 @@
+import gzip
+import logging
+
+from pyhamtools import LookupLib, Callinfo
+
+from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
+from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
+
+
+class ClublogXML(FileDownloadCallsignDataProvider):
+    """Callsign data provider for ClubLog's Country File, which provides basic callsign to DXCC entity mapping."""
+
+    POLL_INTERVAL_DAYS = 30
+    DATA_URL = "https://cdn.clublog.org/cty.php"
+    CACHE_PATH_ZIPPED = "cache/cty.xml.gz"
+    CACHE_PATH_UNZIPPED = "cache/cty.xml"
+    _callinfo = None
+
+    def __init__(self, provider_config):
+        # API key required for this provider
+        self._api_key = provider_config.get("api-key", "")
+        if self._api_key == "":
+            provider_config["enabled"] = False
+            logging.warning(
+                "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
+
+        super().__init__("Clublog XML", provider_config, self.DATA_URL + "?api=" + self._api_key,
+                         self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS)
+
+    def _handle_file(self, path):
+        try:
+            # The download from Clublog is gzipped so we need to uncompress that and re-save as a separate file that
+            # the LookupLib can actually use.
+            with gzip.open(path, "rb") as uncompressed:
+                file_content = uncompressed.read()
+            assert isinstance(file_content, bytes)
+            with open(self.CACHE_PATH_UNZIPPED, "wb") as f:
+                f.write(file_content)
+                f.flush()
+
+            # Now load the data
+            lookuplib = LookupLib(lookuptype="clublogxml", filename=self.CACHE_PATH_UNZIPPED)
+            self._callinfo = Callinfo(lookuplib)
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading Clublog XML.", e, exc_info=True)
+            return False
+
+    def lookup(self, callsign, lookup_credentials):
+        return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)

providers/callsigndata/countryfiles.py

@@ -0,0 +1,32 @@
+import logging
+
+from pyhamtools import LookupLib, Callinfo
+
+from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
+from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
+
+
+class CountryFiles(FileDownloadCallsignDataProvider):
+    """Callsign data provider for Country-files.com, which provides basic callsign to DXCC entity mapping."""
+
+    POLL_INTERVAL_DAYS = 30
+    DATA_URL = "https://www.country-files.com/cty/cty.plist"
+    CACHE_PATH = "cache/cty.plist"
+    _callinfo = None
+
+    def __init__(self, provider_config):
+        super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS)
+
+    def _handle_file(self, path):
+        try:
+            lookuplib = LookupLib(lookuptype="countryfile", filename=path)
+            self._callinfo = Callinfo(lookuplib)
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading Country Files cty.plist.", e, exc_info=True)
+            return False
+
+    def lookup(self, callsign, lookup_credentials):
+        return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)

providers/callsigndata/file_download_callsign_data_provider.py

@@ -0,0 +1,84 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from providers.callsigndata.callsign_data_provider import CallsignDataProvider
+
+
+class FileDownloadCallsignDataProvider(CallsignDataProvider):
+    """Generic callsign data provider class for providers that fetch their data from the web by downloading a file."""
+
+    def __init__(self, name, provider_config, url, cache_file_path, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
+        super().__init__(name, provider_config)
+        self._url = url
+        self._cache_file_path = cache_file_path
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+        self._url_data_cache = URLDataCache("callsigndata_" + name)
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info(
+            "Set up query of " + self.name + " callsign reference data every " + str(self._poll_interval) + " days.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
+                break
+
+    def _poll(self):
+        try:
+            # Request the file. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
+            logging.debug("Downloading " + self.name + " callsign reference data...")
+            http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
+            # Check response code was good
+            if http_response.ok:
+                # Save the data to a local file
+                with open(self._cache_file_path, "wb") as f:
+                    f.write(http_response.content)
+                    f.flush()
+                    # Pass off to the subclass for processing
+                    ok = self._handle_file(self._cache_file_path)
+                    if ok:
+                        self.status = "OK"
+                        self.last_update_time = datetime.now(pytz.UTC)
+                        logging.info("Updated callsign reference data from " + self.name)
+                    else:
+                        self.status = "Error"
+                        logging.warning(f"Error updating callsign reference data from {self.name}.")
+
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}.")
+
+        except ConnectionError:
+            self.status = "Error"
+            logging.warning(f"Connection error when downloading callsign reference data from {self.name}.")
+        except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
+            logging.warning(f"Timeout when downloading callsign reference data from {self.name}.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in callsign reference data provider (" + self.name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _handle_file(self, path):
+        """Handle an updated file downloaded from the server. Return true if successful, false otherwise."""
+
+        raise NotImplementedError("Subclasses must implement this method")

server/webserver.py

@@ -43,6 +43,7 @@ class WebServer:
             "status": "Starting"
         }
 
+    def setup(self):
         # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
         DATA_STORE.spots.add_listener(self._spot_broadcaster.publish)
         DATA_STORE.alerts.add_listener(self._alert_broadcaster.publish)
@@ -145,3 +146,7 @@ def request_log(handler):
             f'{handler.get_status()} {request.request_time():.2f}ms | '
             f'Ref: {referrer} | UA: {user_agent}'
         )
+
+
+# Global object
+WEB_SERVER = WebServer()

spothole.py

@@ -4,49 +4,25 @@ import os
 import signal
 import sys
 
-from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_data_provider_from_config, \
-    get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config, \
-    get_static_data_provider_from_config
-from core.constants import SOFTWARE_VERSION
-from core.data_store import DATA_STORE
 from core.call_lookup_helper import lookup_helper
+from core.config import SERVER_OWNER_CALLSIGN, LOG_LEVEL
+from core.constants import SOFTWARE_VERSION
+from core.data_providers import DATA_PROVIDERS
+from core.data_store import DATA_STORE
 from core.status_reporter import StatusReporter
-from server.webserver import WebServer
+from server.webserver import WEB_SERVER
 
 # Globals
-web_server = None
-spot_providers = []
-alert_providers = []
-solar_condition_providers = []
-static_data_providers = []
-sig_ref_data_providers = []
-cleanup_timer = None
 run = True
 
-
 def shutdown(_signum=None, _frame=None):
     """Shutdown function"""
 
     global run
 
     logging.info("Stopping program...")
-    if web_server:
+    WEB_SERVER.stop()
-        web_server.stop()
+    DATA_PROVIDERS.stop()
-    for sp in spot_providers:
-        if sp.enabled:
-            sp.stop()
-    for ap in alert_providers:
-        if ap.enabled:
-            ap.stop()
-    for scp in solar_condition_providers:
-        if scp.enabled:
-            scp.stop()
-    for srdp in sig_ref_data_providers:
-        if srdp.enabled:
-            srdp.stop()
-    for srdp in static_data_providers:
-        if srdp.enabled:
-            srdp.stop()
     DATA_STORE.close()
     os._exit(0)
 
@@ -76,54 +52,20 @@ if __name__ == '__main__':
     # Set up lookup helper
     lookup_helper.start()
 
-    # Set up web server
+    # Set up and start data providers
-    web_server = WebServer()
+    DATA_PROVIDERS.setup()
+    DATA_PROVIDERS.start()
 
-    # Fetch, set up and start spot providers
+    # Set up and start status reporter
-    for entry in config["spot-providers"]:
+    status_reporter = StatusReporter(run_interval=5)
-        spot_providers.append(get_spot_provider_from_config(entry))
-    for p in spot_providers:
-        if p.enabled:
-            p.start()
-
-    # Fetch, set up and start alert providers
-    for entry in config["alert-providers"]:
-        alert_providers.append(get_alert_provider_from_config(entry))
-    for p in alert_providers:
-        if p.enabled:
-            p.start()
-
-    # Fetch, set up and start solar conditions providers
-    for entry in config.get("solar-condition-providers", []):
-        solar_condition_providers.append(get_solar_conditions_provider_from_config(entry))
-    for p in solar_condition_providers:
-        if p.enabled:
-            p.start()
-
-    # Fetch, set up and start static reference data providers
-    for entry in config.get("static-data-providers", []):
-        static_data_providers.append(get_static_data_provider_from_config(entry))
-    for p in static_data_providers:
-        if p.enabled:
-            p.start()
-
-    # Fetch, set up and start SIG reference data providers
-    for entry in config.get("sig-ref-data-providers", []):
-        sig_ref_data_providers.append(get_sig_ref_data_provider_from_config(entry))
-    for p in sig_ref_data_providers:
-        if p.enabled:
-            p.start()
-
-    # Set up status reporter
-    status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers,
-                                     alert_providers=alert_providers, static_data_providers=static_data_providers,
-                                     sig_ref_data_providers=sig_ref_data_providers,
-                                     solar_condition_providers=solar_condition_providers, run_interval=5)
     status_reporter.start()
 
+    # Set up the web server
+    WEB_SERVER.setup()
+
     logging.info("Startup complete.")
 
     # Run the web server. This is the blocking call that keeps the application running in the main thread, so this must
     # be the last thing we do. web_server.stop() triggers an await condition in the web server which finishes the main
     # thread.
-    web_server.start()
+    WEB_SERVER.start()

static/apidocs/openapi.yml

@@ -23,7 +23,7 @@ info:
     * Added `comment_names` to SIGs in the `/options`, to reflect how they might be referred to in spot comments where
       it differs from their `name`.
     * Added `propagation_mode` field to spots
-    * Added `sig_ref_data_providers` and `static_data_providers` to status and removed `cleanup`
+    * Added `sig_ref_data_providers`, `static_data_providers` and `callsign_data_providers` to status and removed `cleanup`
 
     ### 1.3
 
@@ -1769,6 +1769,28 @@ components:
           description: The number of references fetched using this provider.
           example: 1234
 
+    CallsignDataProviderStatus:
+      type: object
+      properties:
+        sig_name:
+          type: string
+          description: The name of the provider.
+          example: Country Files
+        enabled:
+          type: boolean
+          description: Whether the provider is enabled or not.
+          example: true
+        status:
+          type: string
+          description: The status of the provider.
+          example: OK
+        last_updated:
+          type: number
+          description: >
+            The last time at which this provider received data, UTC seconds since UNIX epoch. If this
+            is zero, the provider has never updated.
+          example: 1759579508
+
     SpotList:
       type: array
       items:
@@ -1876,6 +1898,11 @@ components:
           description: An array of all the SIG reference data providers.
           items:
             $ref: '#/components/schemas/SIGRefDataProviderStatus'
+        callsign_data_providers:
+          type: array
+          description: An array of all the callsign data providers.
+          items:
+            $ref: '#/components/schemas/CallsignDataProviderStatus'
 
     Options:
       type: object

static/js/status.js

@@ -58,6 +58,15 @@ function loadStatus() {
                     <div class="col">References: ${(p["enabled"] && p["reference_count"] > 0) ? p["reference_count"] : "N/A"}</div>
                 </div>`);
         });
+
+        jsonData["callsign_data_providers"].forEach(p => {
+            $("#callsign-data-providers-status-container").append(`
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                    <div class="col"><strong>${p["name"]}</strong></div>
+                    <div class="col">Status: ${p["status"]}</div>
+                    <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
+                </div>`);
+        });
     });
 }
 

templates/status.html

@@ -72,6 +72,15 @@
     </div>
 </div>
 
+<div class="card mt-3">
+    <div class="card-header">
+        Callsign Data Providers
+    </div>
+    <div class="card-body" id="callsign-data-providers-status-container">
+
+    </div>
+</div>
+
 <script src="/static/js/status.js?v=1785434213"></script>
 <script>
     $(document).ready(function () {