Refactor of caching & data storage part 6 #118

config-example.yml

@@ -166,6 +166,13 @@ alert-providers:
     enabled: true
 
 
+# Static reference data providers to use. This allows Spothole to download data such as mapping between callsign
+# prefixes and DXCC entities.
+static-data-providers:
+  - class: "K0SWE"
+    enabled: true
+
+
 # SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF
 # park IDs to their name and location.
 sig-ref-data-providers:

core/config.py

@@ -60,6 +60,14 @@ def get_solar_conditions_provider_from_config(config_providers_entry):
     return provider_class(config_providers_entry)
 
 
+def get_static_data_provider_from_config(config_providers_entry):
+    """Utility method to get a static reference data provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('staticdataproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
 def get_sig_ref_data_provider_from_config(config_providers_entry):
     """Utility method to get a SIG reference data provider based on the class specified in its config entry."""
 

core/data_store.py

@@ -22,6 +22,7 @@ class DataStore:
         self.alerts = None
         self.spots = None
         self.callsigns = None
+        self.dxcc_data = None
         self.sigrefs = None
         self.status_data = None
         self._status = None
@@ -42,10 +43,13 @@ class DataStore:
             self._status.add("status_data", {})
         self.status_data = self._status.get("status_data")
 
-        # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
-        # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
-        # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
-        # so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
+        # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
+        # a regular basis but there's no need for a TTL since old data is better than no data.
+        self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data")
+
+        # For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers
+        # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
+        # syntax for keys to keep it a single level.
         self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
         logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
 
@@ -73,6 +77,7 @@ class DataStore:
         self.alerts.close()
         self._solar.close()
         self._status.close()
+        self.dxcc_data.close()
         self.sigrefs.close()
         self.callsigns.close()
 

core/lookup_helper.py

@@ -1,7 +1,5 @@
 import gzip
-import json
 import logging
-import re
 import urllib.parse
 from datetime import timedelta
 
@@ -9,14 +7,13 @@ import xmltodict
 from diskcache import Cache
 from pyhamtools import LookupLib, Callinfo, callinfo
 from pyhamtools.exceptions import APIKeyMissingError
-from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
 from core.config import config
-from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
-    HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
+from core.constants import HTTP_HEADERS, HAMQTH_PRG
+from core.data_store import DATA_STORE
 from core.url_data_cache import URLDataCache
 
 # QRZ XML field names differ from pyhamtools' normalised names; map them here.
@@ -84,7 +81,6 @@ class LookupHelper:
         self._lookup_lib_basic = None
         self._country_files_cty_plist_download_location = None
         self._dxcc_json_download_location = None
-        self._dxcc_data = None
 
     def start(self):
         # Lookup helpers from pyhamtools. We use five (!) of these. The simplest is country-files.com, which downloads
@@ -118,22 +114,6 @@ class LookupHelper:
                                                          filename=self._clublog_xml_download_location)
         self._clublog_callsign_data_cache = Cache('cache/clublog_callsign_lookup_cache')
 
-        # We also get a lookup of DXCC data from K0SWE to use for additional lookups of e.g. flags.
-        self._dxcc_json_download_location = "cache/dxcc.json"
-        success = self._download_dxcc_json()
-        if success:
-            with open(self._dxcc_json_download_location) as f:
-                tmp_dxcc_data = json.load(f)["dxcc"]
-                # Reformat as a map for faster lookup
-                self._dxcc_data = {}
-                for dxcc in tmp_dxcc_data:
-                    self._dxcc_data[dxcc["entityCode"]] = dxcc
-        else:
-            logging.error("Could not download DXCC data, flags and similar data may be missing!")
-
-        # Precompile regex matches for DXCCs to improve efficiency when iterating through them
-        for dxcc in (self._dxcc_data.values() if self._dxcc_data else []):
-            dxcc["_prefixRegexCompiled"] = re.compile(dxcc["prefixRegex"])
 
     def _download_country_files_cty_plist(self):
         """Download the cty.plist file from country-files.com on first startup. The pyhamtools lib can actually download and use
@@ -163,31 +143,6 @@ class LookupHelper:
             logging.error("Exception when downloading Clublog cty.xml", e)
             return False
 
-    def _download_dxcc_json(self):
-        """Download the dxcc.json file on first startup."""
-
-        try:
-            logging.info("Downloading dxcc.json...")
-            response = _URL_DATA_CACHE.get(
-                "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
-                headers=HTTP_HEADERS)
-
-            if response.ok:
-                with open(self._dxcc_json_download_location, "w") as f:
-                    f.write(response.text)
-                    f.flush()
-                    return True
-            else:
-                logging.warning(f"HTTP {response.status_code} when downloading dxcc.json.")
-                return False
-
-        except ConnectionError:
-            logging.warning(f"Connection error when downloading dxcc.json.")
-        except (ConnectTimeout, ReadTimeout):
-            logging.warning(f"Timeout when downloading dxcc.json.")
-        except Exception as e:
-            logging.error("Exception when downloading dxcc.json", e)
-            return False
 
     def _download_clublog_ctyxml(self):
         """Download the cty.xml (gzipped) file from Clublog on first startup, so we can use it in preference to querying the
@@ -374,7 +329,8 @@ class LookupHelper:
     def get_flag_for_dxcc(self, dxcc):
         """Get an emoji flag for a given DXCC entity ID"""
 
-        return self._dxcc_data[dxcc]["flag"] if dxcc in self._dxcc_data else None
+        dxcc_data = DATA_STORE.dxcc_data[dxcc]
+        return dxcc_data["flag"] if dxcc_data else None
 
     def infer_name_from_callsign_online_lookup(self, call, credentials=None):
         """Infer an operator name from a callsign (requires QRZ.com/HamQTH)"""
@@ -680,61 +636,3 @@ class LookupHelper:
 
 # Singleton object
 lookup_helper = LookupHelper()
-
-
-def infer_mode_from_comment(comment):
-    """Infer a mode from the comment"""
-
-    for mode in ALL_MODES:
-        if mode in comment.upper():
-            return mode
-    for mode in MODE_ALIASES.keys():
-        if mode in comment.upper():
-            return MODE_ALIASES[mode]
-    return None
-
-
-def infer_mode_type_from_mode(mode):
-    """Infer a "mode family" from a mode."""
-
-    if mode.upper() in CW_MODES:
-        return "CW"
-    elif mode.upper() in PHONE_MODES:
-        return "PHONE"
-    elif mode.upper() in DATA_MODES:
-        return "DATA"
-    else:
-        if mode.upper() != "OTHER":
-            logging.warning("Found an unrecognised mode: " + mode + ". Developer should categorise this.")
-        return None
-
-
-def infer_band_from_freq(freq):
-    """Infer a band from a frequency in Hz"""
-
-    for b in BANDS:
-        if b.start_freq <= freq <= b.end_freq:
-            return b
-    return UNKNOWN_BAND
-
-
-def infer_mode_from_frequency(freq):
-    """Infer a mode from the frequency (in Hz) according to the band plan. Just a guess really."""
-
-    try:
-        khz = freq / 1000.0
-        mode = freq_to_band(khz)["mode"]
-        # Some additional common digimode ranges in addition to what the 3rd-party freq_to_band function returns.
-        # This is mostly here just because freq_to_band is very specific about things like FT8 frequencies, and e.g.
-        # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
-        # might include other common digimode centres of activity here, but this achieves the main goal of keeping
-        # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
-        if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or (
-                21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077):
-            mode = "FT8"
-        if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or (
-                18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183):
-            mode = "FT4"
-        return mode
-    except KeyError:
-        return None

core/status_reporter.py

@@ -15,7 +15,7 @@ class StatusReporter:
     """Provides a timed update of the application's status data."""
 
     def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers,
-                 sig_ref_data_providers):
+                 static_data_providers, sig_ref_data_providers):
         """Constructor"""
 
         self._run_interval = run_interval
@@ -23,6 +23,7 @@ class StatusReporter:
         self._spot_providers = spot_providers
         self._alert_providers = alert_providers
         self._solar_condition_providers = solar_condition_providers
+        self._static_data_providers = static_data_providers
         self._sig_ref_data_providers = sig_ref_data_providers
         self._thread = None
         self._stop_event = Event()
@@ -74,6 +75,11 @@ class StatusReporter:
                        "last_updated": p.last_update_time.replace(
                            tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
             self._solar_condition_providers))
+        DATA_STORE.status_data["static_data_providers"] = list(
+            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+                           "last_updated": p.last_update_time.replace(
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                self._static_data_providers))
         DATA_STORE.status_data["sig_ref_data_providers"] = list(
             map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(

core/utils.py

@@ -1,8 +1,71 @@
+import logging
+
 import simplejson
+from pyhamtools.frequency import freq_to_band
+
+from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES
 
 
 def safe_json_dumps(obj):
     """Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
     which are invalid in JSON."""
 
-    return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
+    return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
+
+
+def infer_mode_from_comment(comment):
+    """Infer a mode from the comment"""
+
+    for mode in ALL_MODES:
+        if mode in comment.upper():
+            return mode
+    for mode in MODE_ALIASES.keys():
+        if mode in comment.upper():
+            return MODE_ALIASES[mode]
+    return None
+
+
+def infer_mode_type_from_mode(mode):
+    """Infer a "mode family" from a mode."""
+
+    if mode.upper() in CW_MODES:
+        return "CW"
+    elif mode.upper() in PHONE_MODES:
+        return "PHONE"
+    elif mode.upper() in DATA_MODES:
+        return "DATA"
+    else:
+        if mode.upper() != "OTHER":
+            logging.warning("Found an unrecognised mode: " + mode + ". Developer should categorise this.")
+        return None
+
+
+def infer_band_from_freq(freq):
+    """Infer a band from a frequency in Hz"""
+
+    for b in BANDS:
+        if b.start_freq <= freq <= b.end_freq:
+            return b
+    return UNKNOWN_BAND
+
+
+def infer_mode_from_frequency(freq):
+    """Infer a mode from the frequency (in Hz) according to the band plan. Just a guess really."""
+
+    try:
+        khz = freq / 1000.0
+        mode = freq_to_band(khz)["mode"]
+        # Some additional common digimode ranges in addition to what the 3rd-party freq_to_band function returns.
+        # This is mostly here just because freq_to_band is very specific about things like FT8 frequencies, and e.g.
+        # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
+        # might include other common digimode centres of activity here, but this achieves the main goal of keeping
+        # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
+        if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or (
+                21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077):
+            mode = "FT8"
+        if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or (
+                18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183):
+            mode = "FT4"
+        return mode
+    except KeyError:
+        return None

data/spot.py

@@ -12,9 +12,10 @@ from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 from core.config import MAX_SPOT_AGE
 from core.constants import MODE_ALIASES, PROPAGATION_MODES
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
-from core.lookup_helper import lookup_helper, infer_band_from_freq, infer_mode_from_comment, \
-    infer_mode_from_frequency, infer_mode_type_from_mode
+from core.lookup_helper import lookup_helper
 from core.sig_utils import populate_sig_ref_info, ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
+from core.utils import infer_band_from_freq, infer_mode_from_comment, \
+    infer_mode_from_frequency, infer_mode_type_from_mode
 from data.sig_ref import SIGRef
 
 

server/handlers/api/addspot.py

@@ -10,7 +10,7 @@ from tornado.web import Application
 
 from core.config import ALLOW_SPOTTING
 from core.constants import UNKNOWN_BAND
-from core.lookup_helper import infer_band_from_freq
+from core.utils import infer_band_from_freq
 from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import safe_json_dumps

sigrefdataproviders/file_download_sig_ref_data_provider.py

@@ -21,7 +21,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         self._poll_interval = poll_interval
         self._thread = None
         self._stop_event = Event()
-        self._url_data_cache = URLDataCache("sigrefdata-" + sig_name)
+        self._url_data_cache = URLDataCache("sigrefdata_" + sig_name)
 
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
@@ -50,7 +50,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
             if http_response.ok:
                 # Pass off to the subclass for processing
                 new_data = self._http_response_to_data(http_response)
-                # Submit the new spots for processing. There might not be any spots for the less popular programs.
+                # Add the new data to the SIG Ref data store
                 if new_data:
                     self._add_data(new_data)
 
@@ -62,8 +62,10 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
                 logging.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.")
 
         except ConnectionError:
+            self.status = "Error"
             logging.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.")
         except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
             logging.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.")
         except Exception:
             self.status = "Error"

sigrefdataproviders/local_file_sig_ref_data_provider.py

@@ -22,10 +22,11 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
             else:
-                logging.info("No new SIG ref data found for " + self.sig_name)
+                self.status = "Error"
+                logging.info("Failed to load SIG ref data for " + self.sig_name)
         except Exception as e:
             self.status = "Error"
-            logging.exception("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")")
+            logging.error("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")", e, exc_info=True)
 
     def stop(self):
         pass

sigrefdataproviders/sig_ref_data_provider.py

@@ -15,7 +15,6 @@ class SIGRefDataProvider:
         self.sig_name = sig_name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
-        self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
         self.reference_count = 0
 

spothole.py

@@ -5,7 +5,8 @@ import signal
 import sys
 
 from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_data_provider_from_config, \
-    get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config
+    get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config, \
+    get_static_data_provider_from_config
 from core.constants import SOFTWARE_VERSION
 from core.data_store import DATA_STORE
 from core.lookup_helper import lookup_helper
@@ -17,6 +18,7 @@ web_server = None
 spot_providers = []
 alert_providers = []
 solar_condition_providers = []
+static_data_providers = []
 sig_ref_data_providers = []
 cleanup_timer = None
 run = True
@@ -42,6 +44,9 @@ def shutdown(_signum=None, _frame=None):
     for srdp in sig_ref_data_providers:
         if srdp.enabled:
             srdp.stop()
+    for srdp in static_data_providers:
+        if srdp.enabled:
+            srdp.stop()
     DATA_STORE.close()
     os._exit(0)
 
@@ -95,6 +100,13 @@ if __name__ == '__main__':
         if p.enabled:
             p.start()
 
+    # Fetch, set up and start static reference data providers
+    for entry in config.get("static-data-providers", []):
+        static_data_providers.append(get_static_data_provider_from_config(entry))
+    for p in static_data_providers:
+        if p.enabled:
+            p.start()
+
     # Fetch, set up and start SIG reference data providers
     for entry in config.get("sig-ref-data-providers", []):
         sig_ref_data_providers.append(get_sig_ref_data_provider_from_config(entry))
@@ -104,7 +116,7 @@ if __name__ == '__main__':
 
     # Set up status reporter
     status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers,
-                                     alert_providers=alert_providers,
+                                     alert_providers=alert_providers, static_data_providers=static_data_providers,
                                      sig_ref_data_providers=sig_ref_data_providers,
                                      solar_condition_providers=solar_condition_providers, run_interval=5)
     status_reporter.start()

static/apidocs/openapi.yml

@@ -23,7 +23,7 @@ info:
     * Added `comment_names` to SIGs in the `/options`, to reflect how they might be referred to in spot comments where
       it differs from their `name`.
     * Added `propagation_mode` field to spots
-    * Added `sig_ref_data_providers` to status and removed `cleanup`
+    * Added `sig_ref_data_providers` and `static_data_providers` to status and removed `cleanup`
 
     ### 1.3
 
@@ -1721,6 +1721,28 @@ components:
             is zero, the provider has never updated.
           example: 1759579508
 
+    StaticDataProviderStatus:
+      type: object
+      properties:
+        sig_name:
+          type: string
+          description: The name of the provider.
+          example: K0SWE
+        enabled:
+          type: boolean
+          description: Whether the provider is enabled or not.
+          example: true
+        status:
+          type: string
+          description: The status of the provider.
+          example: OK
+        last_updated:
+          type: number
+          description: >
+            The last time at which this provider received data, UTC seconds since UNIX epoch. If this
+            is zero, the provider has never updated.
+          example: 1759579508
+
     SIGRefDataProviderStatus:
       type: object
       properties:
@@ -1844,6 +1866,11 @@ components:
           description: An array of all the solar conditions providers.
           items:
             $ref: '#/components/schemas/SolarConditionsProviderStatus'
+        static_data_providers:
+          type: array
+          description: An array of all the static reference data providers.
+          items:
+            $ref: '#/components/schemas/StaticDataProviderStatus'
         sig_ref_data_providers:
           type: array
           description: An array of all the SIG reference data providers.

static/js/status.js

@@ -40,6 +40,15 @@ function loadStatus() {
                 </div>`);
         });
 
+        jsonData["static_data_providers"].forEach(p => {
+            $("#static-data-providers-status-container").append(`
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                    <div class="col"><strong>${p["sig_name"]}</strong></div>
+                    <div class="col">Status: ${p["status"]}</div>
+                    <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
+                </div>`);
+        });
+
         jsonData["sig_ref_data_providers"].forEach(p => {
             $("#sig-ref-data-providers-status-container").append(`
                 <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">

staticdataproviders/file_download_static_data_provider.py

@@ -0,0 +1,77 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from staticdataproviders.static_data_provider import StaticDataProvider
+
+
+class FileDownloadStaticDataProvider(StaticDataProvider):
+    """Generic static reference data provider class for providers that fetch their data from the web by downloading a
+     file."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
+        super().__init__(name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+        self._url_data_cache = URLDataCache("staticdata_" + name)
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info(
+            "Set up query of " + self.name + " static reference data every " + str(self._poll_interval) + " days.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
+                break
+
+    def _poll(self):
+        try:
+            # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
+            logging.debug("Downloading " + self.name + " static reference data...")
+            http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                ok = self._handle_http_response(http_response)
+                if ok:
+                    self.status = "OK"
+                    self.last_update_time = datetime.now(pytz.UTC)
+                    logging.info("Updated static reference data for " + self.name)
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when downloading static reference data for {self.name}.")
+
+        except ConnectionError:
+            self.status = "Error"
+            logging.warning(f"Connection error when downloading static reference data for {self.name}.")
+        except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
+            logging.warning(f"Timeout when downloading static reference data for {self.name}.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP static reference data provider (" + self.name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _handle_http_response(self, http_response):
+        """Handle an HTTP response returned by the server and load the data from it. Return true if successful,
+         false otherwise."""
+
+        raise NotImplementedError("Subclasses must implement this method")

staticdataproviders/k0swe.py

@@ -0,0 +1,40 @@
+import logging
+import re
+
+from core.data_store import DATA_STORE
+from staticdataproviders.file_download_static_data_provider import FileDownloadStaticDataProvider
+
+
+class K0SWE(FileDownloadStaticDataProvider):
+    """Static data provider for K0SWE's dxcc.json, which provides callsign regex to DXCC entity mapping, plus DXCC to
+    continent, flag emoji etc."""
+
+    POLL_INTERVAL_DAYS = 7
+    DATA_URL = "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json"
+
+    def __init__(self, provider_config):
+        super().__init__("K0SWE DXCC JSON", provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _handle_http_response(self, http_response):
+        try:
+            dxcc_list = http_response.json()["dxcc"]
+            # Reformat as a map for to place in the data store
+            dxcc_map = {}
+            for dxcc in dxcc_list:
+                dxcc_map[dxcc["entityCode"]] = dxcc
+
+            # Precompile regex matches for DXCCs to improve efficiency when iterating through them
+            for dxcc in dxcc_map.values():
+                dxcc["_prefixRegexCompiled"] = re.compile(dxcc["prefixRegex"])
+
+            # Add to data store
+            for k, v in dxcc_map.items():
+                DATA_STORE.dxcc_data[k] = v
+
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading K0SWE dxcc.json.", e, exc_info=True)
+            return False
+
+

staticdataproviders/local_file_static_data_provider.py

@@ -0,0 +1,36 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from staticdataproviders.static_data_provider import StaticDataProvider
+
+
+class LocalFileStaticDataProvider(StaticDataProvider):
+    """Generic static reference data provider class for providers that fetch their data from a local file on startup."""
+
+    def __init__(self, name, provider_config, path):
+        super().__init__(name, provider_config)
+        self._path = path
+
+    def start(self):
+        logging.debug("Loading " + self.name + " static reference data from file.")
+        try:
+            ok = self._load_data(self._path)
+            if ok:
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+            else:
+                self.status = "Error"
+                logging.info("Failed to load data for " + self.name)
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception in local file Static Data Provider (" + self.name + ")", e, exc_info=True)
+
+    def stop(self):
+        pass
+
+    def _load_data(self, path):
+        """Load data from the given file path. Return true if successful, false otherwise."""
+
+        raise NotImplementedError("Subclasses must implement this method")

staticdataproviders/static_data_provider.py

@@ -0,0 +1,31 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+
+class StaticDataProvider:
+    """Generic static reference data provider class. Subclasses of this query the individual URLs or files for data."""
+
+    def __init__(self, name, provider_config):
+        """Constructor"""
+
+        self.name = name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+        self.reference_count = 0
+
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")

templates/status.html

@@ -54,6 +54,15 @@
     </div>
 </div>
 
+<div class="card mt-3">
+    <div class="card-header">
+        Static Reference Data Providers
+    </div>
+    <div class="card-body" id="static-data-providers-status-container">
+
+    </div>
+</div>
+
 <div class="card mt-3">
     <div class="card-header">
         SIG Reference Data Providers