Refactor of caching & data storage part 3 #118

config-example.yml

@@ -116,21 +116,30 @@ spot-providers:
   - class: "XOTA"
     # xOTA sources have a "name" property so you can define what programme it is for, since xOTA is generic software
     # for running "something-on-the-air" programmes.
-    name: "39C3 TOTA"
+    name: "C3 TOTA"
     enabled: false
     url: "wss://39c3.totawatch.de/api/spot/live"
-    # Fixed SIG for all spots from a provider & location CSV are currently only a feature for the "XOTA" provider,
-    # the software found at https://github.com/nischu/xOTA/. This is because this is a generic backend for xOTA
-    # programmes and so different URLs provide different programmes.
+    # For the "XOTA" provider, a SIG must be set menually here because xOTA is a generic backend for xOTA
+    # programmes and so different URLs potentially provide different programmes.
     sig: "TOTA"
-    locations-csv: "datafiles/39c3-tota.csv"
+    # For Toilets on the Air, we prefix the SIG references (T-01 etc) with some characters that define the conference:
+    # C3, EH or HOPE - so we can look up the correct locations in our database, because each conference starts from T-01
+    # but refers to a toilet in a different building (or continent!)
+    sig-ref-prefix: "C3"
 
   - class: "XOTA"
     name: "EH23 TOTA"
     enabled: false
     url: "wss://eh23.totawatch.de/api/spot/live"
     sig: "TOTA"
-    locations-csv: "datafiles/eh23-tota.csv"
+    sig-ref-prefix: "EH"
+
+  - class: "XOTA"
+    name: "HOPE26 TOTA"
+    enabled: false
+    url: "wss://hope-26.totawatch.de/api/spot/live"
+    sig: "TOTA"
+    sig-ref-prefix: "HOPE"
 
 
 # Alert providers to use. Same setup as the spot providers list above.
@@ -157,6 +166,30 @@ alert-providers:
     enabled: true
 
 
+# SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF
+# park IDs to their name and location.
+sig-ref-data-providers:
+  - class: "WWFF"
+    enabled: true
+
+  - class: "WOTA"
+    enabled: true
+
+  - class: "SIOTA"
+    enabled: true
+
+  - class: "ZLOTA"
+    enabled: true
+
+  - class: "LLOTA"
+    enabled: true
+
+  - class: "DME"
+    enabled: true
+
+  - class: "TOTA"
+    enabled: true
+
 # Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band
 # conditions, etc.) and make it available via the /api/v1/solar endpoint.
 solar-condition-providers:

core/config.py

@@ -1,3 +1,4 @@
+import importlib
 import logging
 import os
 
@@ -33,3 +34,35 @@ WEB_UI_OPTIONS["spot-providers-enabled-by-default"] = [p["name"] for p in config
 # one of our proviers. We set that to also be enabled by default.
 if ALLOW_SPOTTING:
     WEB_UI_OPTIONS["spot-providers-enabled-by-default"].append("API")
+
+
+def get_spot_provider_from_config(config_providers_entry):
+    """Utility method to get a spot provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('spotproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
+def get_alert_provider_from_config(config_providers_entry):
+    """Utility method to get an alert provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('alertproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
+def get_solar_conditions_provider_from_config(config_providers_entry):
+    """Utility method to get a solar conditions provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('solarconditionsproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
+def get_sig_ref_data_provider_from_config(config_providers_entry):
+    """Utility method to get a SIG reference data provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('sigrefdataproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)

core/data_store.py

@@ -12,57 +12,52 @@ class DataStore:
     lookup data using different caching strategies for each."""
 
     def __init__(self):
-        cache_dir = "./cache"
+        self.CACHE_DIR = "./cache"
         self.MAX_SPOT_COUNT = 10000
         self.MAX_ALERT_COUNT = 10000
         self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
         self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
 
-        Path(cache_dir).mkdir(parents=True, exist_ok=True)
-
-        # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval
-        self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
-                                   snapshot_dir=cache_dir + "/spots",
-                                   snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
-        self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
-                                   snapshot_dir=cache_dir + "/alerts",
-                                    snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+    def setup(self):
+        Path(self.CACHE_DIR).mkdir(parents=True, exist_ok=True)
 
         # Standard disk cache for solar data and status data, but each cache contains only a single object which we
         # expose to the wider application
-        self.solar = diskcache.Cache(cache_dir + "/solar")
+        self.solar = diskcache.Cache(self.CACHE_DIR + "/solar")
         if "solar_conditions" not in self.solar:
             self.solar.add("solar_conditions", SolarConditions())
         self.solar_conditions = self.solar.get("solar_conditions")
-        self.status = diskcache.Cache(cache_dir + "/status")
+        self.status = diskcache.Cache(self.CACHE_DIR + "/status")
         if "status_data" not in self.status:
             self.status.add("status_data", {})
         self.status_data = self.status.get("status_data")
 
-        # Standard disk caches for SIG ref data. Separate provider threads will repopulate these on a regular basis
-        # but there's no need for a TTL since old data is better than no data.
-        self.sigrefs_wwff = diskcache.Cache(cache_dir + "/sigrefs/wwff")
-        self.sigrefs_siota = diskcache.Cache(cache_dir + "/sigrefs/siota")
-        self.sigrefs_wota = diskcache.Cache(cache_dir + "/sigrefs/wota")
-        self.sigrefs_zlota = diskcache.Cache(cache_dir + "/sigrefs/zlota")
-        self.sigrefs_llota = diskcache.Cache(cache_dir + "/sigrefs/llota")
-        self.sigrefs_dme = diskcache.Cache(cache_dir + "/sigrefs/dme")
+        # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
+        # but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG
+        # name and then reference ID, with the final value being a SIGRef object.
+        self.sigrefs = diskcache.Cache(self.CACHE_DIR + "/sigrefs")
 
         # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
         # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
         # we've seen, so we rely on them timing out and this triggering another lookup.
-        self.callsigns = diskcache.Cache(cache_dir + "/callsigns")
+        self.callsigns = diskcache.Cache(self.CACHE_DIR + "/callsigns")
+
+        # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
+        # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
+        # before the spots and alerts are live in the system.
+        self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
+                                   snapshot_dir=self.CACHE_DIR + "/spots",
+                                   snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+        self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
+                                   snapshot_dir=self.CACHE_DIR + "/alerts",
+                                    snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
 
     def close(self):
         self.spots.close()
         self.alerts.close()
         self.solar.close()
         self.status.close()
-        self.sigrefs_wwff.close()
-        self.sigrefs_siota.close()
-        self.sigrefs_wota.close()
-        self.sigrefs_zlota.close()
-        self.sigrefs_llota.close()
+        self.sigrefs.close()
         self.callsigns.close()
 
 # Global object

core/sig_utils.py

@@ -1,4 +1,3 @@
-import csv
 import logging
 
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
@@ -9,11 +8,6 @@ from core.data_store import DATA_STORE
 from core.geo_utils import wab_wai_square_to_lat_lon
 from core.url_data_cache import URL_DATA_CACHE
 
-# Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data
-# file in Spothole and load it on startup.
-with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
-    for row in csv.DictReader(_f, delimiter=";"):
-        DATA_STORE.sigrefs_dme.add(row["COD_INE"][:5], row)
 
 def get_ref_regex_for_sig(sig):
     """Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
@@ -35,12 +29,15 @@ def get_sig_name_from_comment_name(sig):
 
 
 def populate_sig_ref_info(sig_ref):
-    """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object which
-    must at minimum have a "sig" and an "id". The rest of the object will be populated and returned.
+    """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
+    which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
+    use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
+    SIG we are getting data for.
     Note there is currently no support for KRMNPA location lookup, see issue #61."""
 
-    if sig_ref.sig is None or sig_ref.id is None:
-        logging.warning("Failed to look up sig_ref info, sig or id were not set.")
+    if sig_ref.sig is None or sig_ref.id is None or sig_ref.id == "":
+        logging.debug("Failed to look up sig_ref info, sig or id were not set.")
+        return sig_ref
 
     sig = sig_ref.sig or ""
     ref_id = sig_ref.id
@@ -123,95 +120,36 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "WWFF":
-            response = URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                if not bool(DATA_STORE.sigrefs_wwff) or not response.from_cache:
-                    # New data from WWFF, update our internal map
-                    for row in csv.DictReader(response.content.decode().splitlines()):
-                        DATA_STORE.sigrefs_wwff.add(row["reference"], row)
-                row = DATA_STORE.sigrefs_wwff.get(ref_id)
-                if row:
-                    sig_ref.name = row["name"] if "name" in row else None
-                    sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
-                    sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
-                    sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
-                    sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[
-                        "longitude"] != "-" else None
-                elif not response.from_cache:
-                    logging.warning("WWFF database did not contain data for ref %s", ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            lookup_data = DATA_STORE.sigrefs["WWFF"][ref_id] if ref_id in DATA_STORE.sigrefs["WWFF"] else None
+            if lookup_data:
+                # Copy new sig ref data into existing object
+                sig_ref.__dict__.update(lookup_data.__dict__)
+            else:
+                logging.warning("WWFF database did not contain data for ref %s", ref_id)
 
         elif sig.upper() == "SIOTA":
-            response = URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                if not bool(DATA_STORE.sigrefs_siota) or not response.from_cache:
-                    # New data from SIOTA, update our internal map
-                    for row in csv.DictReader(response.content.decode().splitlines()):
-                        DATA_STORE.sigrefs_siota.add(row["SILO_CODE"], row)
-                row = DATA_STORE.sigrefs_siota.get(ref_id)
-                if row:
-                    sig_ref.name = row["NAME"] if "NAME" in row else None
-                    sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
-                    sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
-                    sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
-                elif not response.from_cache:
-                    logging.warning("SIOTA database did not contain data for ref %s", ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            lookup_data = DATA_STORE.sigrefs["SIOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["SIOTA"] else None
+            if lookup_data:
+                # Copy new sig ref data into existing object
+                sig_ref.__dict__.update(lookup_data.__dict__)
+            else:
+                logging.warning("SIOTA database did not contain data for ref %s", ref_id)
 
         elif sig.upper() == "WOTA":
-            response = URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if data:
-                    if not bool(DATA_STORE.sigrefs_wota) or not response.from_cache:
-                        # New data from WOTA, update our internal map
-                        for feature in data.get("features", []):
-                            DATA_STORE.sigrefs_wota.add(feature["properties"]["wotaId"], feature)
-                    feature = DATA_STORE.sigrefs_wota.get(ref_id)
-                    if feature:
-                        sig_ref.name = feature["properties"]["title"]
-                        # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
-                        # added to them
-                        sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id
-                        if ref_id.upper().startswith("LDO-"):
-                            number = int(ref_id.upper().replace("LDO-", ""))
-                            sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
-                        sig_ref.grid = feature["properties"]["qthLocator"]
-                        sig_ref.latitude = feature["geometry"]["coordinates"][1]
-                        sig_ref.longitude = feature["geometry"]["coordinates"][0]
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            lookup_data = DATA_STORE.sigrefs["WOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["WOTA"] else None
+            if lookup_data:
+                # Copy new sig ref data into existing object
+                sig_ref.__dict__.update(lookup_data.__dict__)
+            else:
+                logging.warning("WOTA database did not contain data for ref %s", ref_id)
 
         elif sig.upper() == "ZLOTA":
-            response = URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if isinstance(data, list):
-                    if not bool(DATA_STORE.sigrefs_zlota) or not response.from_cache:
-                        # New data from ZLOTA, update our internal map
-                        for ref in data:
-                            DATA_STORE.sigrefs_zlota.add(ref["code"], ref)
-                    ref = DATA_STORE.sigrefs_zlota.get(ref_id)
-                    if ref:
-                        sig_ref.name = ref["name"]
-                        sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
-                        try:
-                            sig_ref.grid = latlong_to_locator(ref["y"], ref["x"], 6)
-                        except:
-                            logging.debug("Invalid lat/lon received for reference")
-                            sig_ref.latitude = ref["y"]
-                            sig_ref.longitude = ref["x"]
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            lookup_data = DATA_STORE.sigrefs["ZLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["ZLOTA"] else None
+            if lookup_data:
+                # Copy new sig ref data into existing object
+                sig_ref.__dict__.update(lookup_data.__dict__)
+            else:
+                logging.warning("ZLOTA database did not contain data for ref %s", ref_id)
 
         elif sig.upper() == "BOTA":
             if not sig_ref.name:
@@ -219,27 +157,29 @@ def populate_sig_ref_info(sig_ref):
             sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
 
         elif sig.upper() == "LLOTA":
-            response = URL_DATA_CACHE.get("https://llota.app/api/public/references",
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if isinstance(data, list):
-                    if not bool(DATA_STORE.sigrefs_llota) or not response.from_cache:
-                        # New data from LLOTA, update our internal map
-                        for ref in data:
-                            DATA_STORE.sigrefs_llota.add(ref["reference_code"], ref)
-                    ref = DATA_STORE.sigrefs_llota.get(ref_id)
-                    if ref:
-                        sig_ref.name = str(ref["name"])
-                        sig_ref.url = "https://llota.app/list/ref/" + ref_id
-                        sig_ref.grid = str(ref["grid_locator"])
-                        ll = locator_to_latlong(sig_ref.grid)
-                        sig_ref.latitude = ll[0]
-                        sig_ref.longitude = ll[1]
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            lookup_data = DATA_STORE.sigrefs["LLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["LLOTA"] else None
+            if lookup_data:
+                # Copy new sig ref data into existing object
+                sig_ref.__dict__.update(lookup_data.__dict__)
+            else:
+                logging.warning("LLOTA database did not contain data for ref %s", ref_id)
+
+        elif sig.upper() == "DME":
+            # Zero-pad to 5 digits to match our source data
+            lookup_data = DATA_STORE.sigrefs["DME"][ref_id.zfill(5)] if ref_id.zfill(5) in DATA_STORE.sigrefs["DME"] else None
+            if lookup_data:
+                # Copy new sig ref data into existing object
+                sig_ref.__dict__.update(lookup_data.__dict__)
+            else:
+                logging.warning("DME database did not contain data for ref %s", ref_id)
+
+        elif sig.upper() == "TOTA":
+            lookup_data = DATA_STORE.sigrefs["TOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["TOTA"] else None
+            if lookup_data:
+                # Copy new sig ref data into existing object
+                sig_ref.__dict__.update(lookup_data.__dict__)
+            else:
+                logging.warning("TOTA database did not contain data for ref %s", ref_id)
 
         elif sig.upper() == "WWTOTA":
             if not sig_ref.name:
@@ -268,23 +208,6 @@ def populate_sig_ref_info(sig_ref):
                 except:
                     logging.warning("Invalid lat/lon received for WAB/WAI reference")
 
-        elif sig.upper() == "DME":
-            # Zero-pad to 5 digits to match our source data
-            row = DATA_STORE.sigrefs_dme.get(ref_id.zfill(5))
-            if row:
-                sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
-                sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
-                    "LATITUD_ETRS89_REGCAN95") else None
-                sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
-                    "LONGITUD_ETRS89_REGCAN95") else None
-                if sig_ref.latitude and sig_ref.longitude:
-                    try:
-                        sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6)
-                    except Exception:
-                        logging.warning("Invalid lat/lon received for DME reference")
-            else:
-                logging.warning("DME database did not contain data for ref %s", ref_id)
-
     except ConnectionError:
         logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
     except (ConnectTimeout, ReadTimeout):

core/status_reporter.py

@@ -14,7 +14,8 @@ from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alert
 class StatusReporter:
     """Provides a timed update of the application's status data."""
 
-    def __init__(self, run_interval, web_server,spot_providers, alert_providers, solar_condition_providers):
+    def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers,
+                 sig_ref_data_providers):
         """Constructor"""
 
         self._run_interval = run_interval
@@ -22,6 +23,7 @@ class StatusReporter:
         self._spot_providers = spot_providers
         self._alert_providers = alert_providers
         self._solar_condition_providers = solar_condition_providers
+        self._sig_ref_data_providers = sig_ref_data_providers
         self._thread = None
         self._stop_event = Event()
         self._startup_time = datetime.now(pytz.UTC)
@@ -72,6 +74,11 @@ class StatusReporter:
                        "last_updated": p.last_update_time.replace(
                            tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
             self._solar_condition_providers))
+        DATA_STORE.status_data["sig_ref_data_providers"] = list(
+            map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
+                           "last_updated": p.last_update_time.replace(
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                self._sig_ref_data_providers))
         DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
                                       "last_api_access": self._web_server.web_server_metrics[
                                           "last_api_access_time"].replace(

datafiles/39c3-tota.csv

@@ -1,18 +0,0 @@
-ref,lat,lon
-T-01,53.56278090617755,9.984341869295505
-T-02,53.562383404176416,9.98551893027115
-T-03,53.56170184391514,9.985416035619778
-T-04,53.562026534393176,9.986372919078974
-T-11,53.56284641242506,9.98475590239655
-T-12,53.562431705517035,9.98551675702443
-T-13,53.56223704898424,9.985774520335664
-T-14,53.5617893512591,9.986344302837976
-T-21,53.56284641242506,9.98475590239655
-T-22,53.56245816412497,9.985456089490567
-T-23,53.56199560857136,9.985636761412673
-T-24,53.5617893512591,9.986344302837976
-T-31,53.56247470064887,9.985611427551902
-T-32,53.5617893512591,9.986344302837976
-T-41,53.56245039134992,9.985486136112701
-T-91,53.56147934973529,9.984626806439744
-T-92,53.561396810300735,9.987553052152899

datafiles/eh23-tota.csv

@@ -1,13 +0,0 @@
-ref,lat,lon
-T-01,50.3636495,7.5584857
-T-02,50.3636495,7.5584857
-T-03,50.3636495,7.5584857
-T-11,50.3636495,7.5584857
-T-13,50.3636495,7.5584857
-T-14,50.3636495,7.5584857
-T-21,50.3636495,7.5584857
-T-31,50.3636495,7.5584857
-T-33,50.3636495,7.5584857
-T-34,50.3636495,7.5584857
-T-41,50.3636495,7.5584857
-T-51,50.3636495,7.5584857

datafiles/tota.csv

@@ -0,0 +1,30 @@
+ref,lat,lon
+C3 T-01,53.56278090617755,9.984341869295505
+C3 T-02,53.562383404176416,9.98551893027115
+C3 T-03,53.56170184391514,9.985416035619778
+C3 T-04,53.562026534393176,9.986372919078974
+C3 T-11,53.56284641242506,9.98475590239655
+C3 T-12,53.562431705517035,9.98551675702443
+C3 T-13,53.56223704898424,9.985774520335664
+C3 T-14,53.5617893512591,9.986344302837976
+C3 T-21,53.56284641242506,9.98475590239655
+C3 T-22,53.56245816412497,9.985456089490567
+C3 T-23,53.56199560857136,9.985636761412673
+C3 T-24,53.5617893512591,9.986344302837976
+C3 T-31,53.56247470064887,9.985611427551902
+C3 T-32,53.5617893512591,9.986344302837976
+C3 T-41,53.56245039134992,9.985486136112701
+C3 T-91,53.56147934973529,9.984626806439744
+C3 T-92,53.561396810300735,9.987553052152899
+EH T-01,50.3636495,7.5584857
+EH T-02,50.3636495,7.5584857
+EH T-03,50.3636495,7.5584857
+EH T-11,50.3636495,7.5584857
+EH T-13,50.3636495,7.5584857
+EH T-14,50.3636495,7.5584857
+EH T-21,50.3636495,7.5584857
+EH T-31,50.3636495,7.5584857
+EH T-33,50.3636495,7.5584857
+EH T-34,50.3636495,7.5584857
+EH T-41,50.3636495,7.5584857
+EH T-51,50.3636495,7.5584857

sigrefdataproviders/dme.py

@@ -0,0 +1,34 @@
+import csv
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
+
+
+class DME(LocalFileSIGRefDataProvider):
+    """SIG ref data provider for Diploma Municipios de Espana"""
+
+    SIG = "DME"
+    PATH = "datafiles/MUNICIPIOS.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.PATH)
+
+    def _file_to_data(self, path):
+        new_data = {}
+        with open(path, encoding="latin-1") as _f:
+            for row in csv.DictReader(_f, delimiter=";"):
+                ref_id = row["COD_INE"][:5]
+                latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LATITUD_ETRS89_REGCAN95") else None
+                longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LONGITUD_ETRS89_REGCAN95") else None
+
+                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id,
+                                          name=row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"],
+                                          latitude=latitude,
+                                          longitude=longitude)
+                if latitude and longitude:
+                    new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
+        return new_data

sigrefdataproviders/file_download_sig_ref_data_provider.py

@@ -0,0 +1,75 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+import requests
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
+
+
+class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
+    """Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
+
+    def __init__(self, sig_name, provider_config, url, poll_interval):
+        super().__init__(sig_name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info(
+            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " seconds.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval):
+                break
+
+    def _poll(self):
+        try:
+            # Request data from API
+            logging.debug("Downloading " + self.sig_name + " SIG ref data...")
+            http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                new_data = self._http_response_to_data(http_response)
+                # Submit the new spots for processing. There might not be any spots for the less popular programs.
+                if new_data:
+                    self._replace_data(new_data)
+
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.debug("Received SIG ref data for " + self.sig_name)
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.")
+
+        except ConnectionError:
+            logging.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP SIG Ref Data Provider (" + self.sig_name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _http_response_to_data(self, http_response):
+        """Convert an HTTP response returned by the server into SIG Ref data. The whole response is provided here so the
+        subclass implementations can check for HTTP status codes if necessary, and handle the response as JSON, CSV,
+        whatever the remote file actually is."""
+
+        raise NotImplementedError("Subclasses must implement this method")

sigrefdataproviders/llota.py

@@ -0,0 +1,32 @@
+from pyhamtools.locator import locator_to_latlong
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class LLOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Lagos y Lagunas on the Air"""
+
+    POLL_INTERVAL_SEC = 7 * 24 * 60 * 60  # 7 days
+    SIG = "LLOTA"
+    DATA_URL = "https://llota.app/api/public/references"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data:
+                ref_id = ref["reference_code"]
+                grid = str(ref["grid_locator"])
+                ll = locator_to_latlong(grid)
+
+                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
+                                          url="https://llota.app/list/ref/" + ref_id,
+                                          grid=grid,
+                                          latitude=ll[0],
+                                          longitude=ll[1])
+
+        return new_data

sigrefdataproviders/local_file_sig_ref_data_provider.py

@@ -0,0 +1,36 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
+
+
+class LocalFileSIGRefDataProvider(SIGRefDataProvider):
+    """Generic SIG ref data provider class for providers that fetch their data from a local file on startup."""
+
+    def __init__(self, sig, provider_config, path):
+        super().__init__(sig, provider_config)
+        self._path = path
+
+    def start(self):
+        logging.info("Loading " + self.sig_name + " SIG ref data from file.")
+        try:
+            new_data = self._file_to_data(self._path)
+            if new_data:
+                self._replace_data(new_data)
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+            else:
+                logging.info("No new SIG ref data found for " + self.sig_name)
+        except Exception as e:
+            self.status = "Error"
+            logging.exception("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")")
+
+    def stop(self):
+        pass
+
+    def _file_to_data(self, path):
+        """Load a file on the given path and turn it into SIG Ref data."""
+
+        raise NotImplementedError("Subclasses must implement this method")

sigrefdataproviders/sig_ref_data_provider.py

@@ -2,19 +2,25 @@ from datetime import datetime
 
 import pytz
 
+from core.data_store import DATA_STORE
+
 
 class SIGRefDataProvider:
     """Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data."""
 
-    def __init__(self, name, provider_config):
+    def __init__(self, sig_name, provider_config):
         """Constructor"""
 
-        self.name = name
+        self.sig_name = sig_name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
 
+        # Create an empty dict to store data if one doesn't already exist
+        if not sig_name in DATA_STORE.sigrefs:
+            DATA_STORE.sigrefs[sig_name] = {}
+
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -26,3 +32,9 @@ class SIGRefDataProvider:
         """Stop any threads and prepare for application shutdown"""
 
         raise NotImplementedError("Subclasses must implement this method")
+
+    def _replace_data(self, new_data):
+        """Replace all data for the named sig with the new data. new_data should be a map of reference ID to SIGRef
+        objects."""
+
+        DATA_STORE.sigrefs[self.sig_name] = new_data

sigrefdataproviders/siota.py

@@ -0,0 +1,26 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class SIOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Silos on the Air"""
+
+    POLL_INTERVAL_SEC = 30 * 24 * 60 * 60  # 30 days
+    SIG = "SIOTA"
+    DATA_URL = "https://www.silosontheair.com/data/silos.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode().splitlines()):
+            ref_id = row["SILO_CODE"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
+                                      grid=row["LOCATOR"] if "LOCATOR" in row else None,
+                                      latitude=float(row["LAT"]) if "LAT" in row else None,
+                                      longitude=float(row["LNG"]) if "LNG" in row else None)
+
+        return new_data

sigrefdataproviders/tota.py

@@ -0,0 +1,24 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
+
+
+class TOTA(LocalFileSIGRefDataProvider):
+    """SIG ref data provider for Toilets on the Air"""
+
+    SIG = "TOTA"
+    PATH = "datafiles/tota.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.PATH)
+
+    def _file_to_data(self, path):
+        new_data = {}
+        f = open(path)
+        csv_data = f.read()
+        dr = csv.DictReader(csv_data.splitlines())
+        for row in dr:
+            new_data[row["ref"]] = SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
+                                          longitude=float(row["lon"]))
+        return new_data

sigrefdataproviders/wota.py

@@ -0,0 +1,31 @@
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Wainwrights on the Air"""
+
+    POLL_INTERVAL_SEC = 365 * 24 * 60 * 60  # 365 days
+    SIG = "WOTA"
+    DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for feature in http_response.json().get("features", []):
+            ref_id = feature["properties"]["wotaId"]
+            # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
+            # added to them
+            url = "https://www.wota.org.uk/MM_" + ref_id
+            if ref_id.upper().startswith("LDO-"):
+                number = int(ref_id.upper().replace("LDO-", ""))
+                url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
+
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
+                                      grid=feature["properties"]["qthLocator"],
+                                      latitude=feature["geometry"]["coordinates"][1],
+                                      longitude=feature["geometry"]["coordinates"][0])
+
+        return new_data

sigrefdataproviders/wwff.py

@@ -0,0 +1,30 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WWFF(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Worldwide Flora & Fauna"""
+
+    POLL_INTERVAL_SEC = 30 * 24 * 60 * 60  # 30 days
+    SIG = "WWFF"
+    DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode().splitlines()):
+            ref_id = row["reference"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
+                                      url="https://wwff.co/directory/?showRef=" + ref_id,
+                                      grid=row["iaruLocator"] if "iaruLocator" in row and row[
+                                          "iaruLocator"] != "-" else None,
+                                      latitude=float(row["latitude"]) if "latitude" in row and row[
+                                          "latitude"] != "" and row["latitude"] != "-" else None,
+                                      longitude=float(row["longitude"]) if "longitude" in row and row[
+                                          "longitude"] != "" and row["longitude"] != "-" else None)
+
+        return new_data

sigrefdataproviders/zlota.py

@@ -0,0 +1,33 @@
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ZLOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for New Zealand on the Air"""
+
+    POLL_INTERVAL_SEC = 30 * 24 * 60 * 60  # 30 days
+    SIG = "ZLOTA"
+    DATA_URL = "https://ontheair.nz/assets/assets.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data:
+                ref_id = ref["code"]
+                latitude = ref["y"]
+                longitude = ref["x"]
+
+                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
+                                          url="https://ontheair.nz/assets/" + ref_id.replace("/", "_"),
+                                          latitude=latitude,
+                                          longitude=longitude)
+                if latitude and longitude:
+                    new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
+
+        return new_data

spothole.py

@@ -1,11 +1,11 @@
 # Main script
-import importlib
 import logging
 import os
 import signal
 import sys
 
-from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL
+from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_data_provider_from_config, \
+    get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config
 from core.constants import SOFTWARE_VERSION
 from core.data_store import DATA_STORE
 from core.lookup_helper import lookup_helper
@@ -17,6 +17,7 @@ web_server = None
 spot_providers = []
 alert_providers = []
 solar_condition_providers = []
+sig_ref_data_providers = []
 cleanup_timer = None
 run = True
 
@@ -38,34 +39,13 @@ def shutdown(_signum=None, _frame=None):
     for scp in solar_condition_providers:
         if scp.enabled:
             scp.stop()
+    for srdp in sig_ref_data_providers:
+        if srdp.enabled:
+            srdp.stop()
     DATA_STORE.close()
     os._exit(0)
 
 
-def get_spot_provider_from_config(config_providers_entry):
-    """Utility method to get a spot provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('spotproviders.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_alert_provider_from_config(config_providers_entry):
-    """Utility method to get an alert provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('alertproviders.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_solar_conditions_provider_from_config(config_providers_entry):
-    """Utility method to get a solar conditions provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('solarconditionsproviders.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
 # Main function
 if __name__ == '__main__':
     # Set up logging
@@ -85,6 +65,9 @@ if __name__ == '__main__':
     # Shut down gracefully on SIGINT
     signal.signal(signal.SIGINT, shutdown)
 
+    # Set up data store
+    DATA_STORE.setup()
+
     # Set up lookup helper
     lookup_helper.start()
 
@@ -112,9 +95,17 @@ if __name__ == '__main__':
         if p.enabled:
             p.start()
 
+    # Fetch, set up and start SIG reference data providers
+    for entry in config.get("sig-ref-data-providers", []):
+        sig_ref_data_providers.append(get_sig_ref_data_provider_from_config(entry))
+    for p in sig_ref_data_providers:
+        if p.enabled:
+            p.start()
+
     # Set up status reporter
     status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers,
                                      alert_providers=alert_providers,
+                                     sig_ref_data_providers=sig_ref_data_providers,
                                      solar_condition_providers=solar_condition_providers, run_interval=5)
     status_reporter.start()
 

spotproviders/http_spot_provider.py

@@ -64,7 +64,7 @@ class HTTPSpotProvider(SpotProvider):
             logging.warning(f"Timeout when accessing {self.name} spots API.")
         except Exception:
             self.status = "Error"
-            logging.exception("Exception in HTTP JSON Spot Provider (" + self.name + ")")
+            logging.exception("Exception in HTTP Spot Provider (" + self.name + ")")
             self._stop_event.wait(timeout=1)
 
     def _http_response_to_spots(self, http_response):

spotproviders/xota.py

@@ -1,6 +1,4 @@
-import csv
 import json
-import logging
 from datetime import datetime
 
 import pytz
@@ -12,10 +10,11 @@ from spotproviders.websocket_spot_provider import WebsocketSpotProvider
 
 class XOTA(WebsocketSpotProvider):
     """Spot provider for servers based on the "xOTA" software at https://github.com/nischu/xOTA/
-    The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG and a reference
-    to a local CSV file with location information. This functionality is implemented for TOTA events, of which there are
-    several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA and hence which server
-    supplied the data, which is why the CSV location lookup is here and not in sig_utils."""
+    The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG which we can
+    then use for lookups. This functionality is implemented for TOTA events, of which there are
+    several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA, which is why we also
+    provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3 might become
+    "C3 T-01". This allows us to provide location lookups for TOTA at several conferences."""
 
     LOCATION_DATA = {}
     SIG = None
@@ -23,26 +22,13 @@ class XOTA(WebsocketSpotProvider):
     def __init__(self, provider_config):
         name = provider_config["name"] if "name" in provider_config else "xOTA"
         super().__init__(name, provider_config, provider_config["url"])
-        locations_csv = str(provider_config["locations-csv"]) if "locations-csv" in provider_config else None
         self.SIG = str(provider_config["sig"]) if "sig" in provider_config else None
-
-        # Load location data
-        if locations_csv:
-            try:
-                f = open(locations_csv)
-                csv_data = f.read()
-                dr = csv.DictReader(csv_data.splitlines())
-                for row in dr:
-                    self.LOCATION_DATA[row["ref"]] = {"lat": row["lat"], "lon": row["lon"]}
-            except:
-                logging.exception("Could not look up location data for XOTA source.")
+        self._sig_ref_prefix = str(provider_config["sig-ref-prefix"]) if "sig-ref-prefix" in provider_config else ""
 
     def _ws_message_to_spot(self, b):
         string = b.decode("utf-8")
         source_spot = json.loads(string)
-        ref_id = source_spot["reference"]["title"]
-        lat = float(self.LOCATION_DATA[ref_id]["lat"]) if ref_id in self.LOCATION_DATA else None
-        lon = float(self.LOCATION_DATA[ref_id]["lon"]) if ref_id in self.LOCATION_DATA else None
+        ref_id = self._sig_ref_prefix + " " + source_spot["reference"]["title"]
         spot = Spot(source=self.name,
                     source_id=source_spot["id"],
                     dx_call=source_spot["stationCallSign"].upper(),
@@ -50,10 +36,7 @@ class XOTA(WebsocketSpotProvider):
                     mode=source_spot["mode"].upper(),
                     sig=self.SIG,
                     sig_refs=[
-                        SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"], latitude=lat,
-                               longitude=lon)],
+                        SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"])],
                     time=datetime.now(pytz.UTC).timestamp(),
-                    dx_latitude=lat,
-                    dx_longitude=lon,
                     qrt=source_spot["state"] != "active")
         return spot

static/apidocs/openapi.yml

@@ -23,6 +23,7 @@ info:
     * Added `comment_names` to SIGs in the `/options`, to reflect how they might be referred to in spot comments where
       it differs from their `name`.
     * Added `propagation_mode` field to spots
+    * Added `sig_ref_data_providers` to status and removed `cleanup`
 
     ### 1.3
 
@@ -1720,6 +1721,28 @@ components:
             is zero, the provider has never updated.
           example: 1759579508
 
+    SIGRefDataProviderStatus:
+      type: object
+      properties:
+        sig_name:
+          type: string
+          description: The name of the SIG.
+          example: WWFF
+        enabled:
+          type: boolean
+          description: Whether the provider is enabled or not.
+          example: true
+        status:
+          type: string
+          description: The status of the provider.
+          example: OK
+        last_updated:
+          type: number
+          description: >
+            The last time at which this provider received data, UTC seconds since UNIX epoch. If this
+            is zero, the provider has never updated.
+          example: 1759579508
+
     SpotList:
       type: array
       items:
@@ -1787,17 +1810,6 @@ components:
           type: integer
           description: Number of alerts currently in the system.
           example: 123
-        "cleanup":
-          type: object
-          properties:
-            status:
-              type: string
-              description: The status of the cleanup thread
-              example: OK
-            last_ran:
-              type: number
-              description: The last time the cleanup operation ran, UTC seconds since UNIX epoch.
-              example: 1759579508
         "webserver":
           type: object
           properties:
@@ -1828,6 +1840,11 @@ components:
           description: An array of all the solar conditions providers.
           items:
             $ref: '#/components/schemas/SolarConditionsProviderStatus'
+        sig_ref_data_providers:
+          type: array
+          description: An array of all the SIG reference data providers.
+          items:
+            $ref: '#/components/schemas/SIGRefDataProviderStatus'
 
     Options:
       type: object

static/js/status.js

@@ -12,9 +12,6 @@ function loadStatus() {
         $("#web-server-last-api").text(moment.unix(jsonData["webserver"]["last_api_access"]).utc().fromNow());
         $("#web-server-last-page").text(moment.unix(jsonData["webserver"]["last_page_access"]).utc().fromNow());
 
-        $("#cleanup-status").text(jsonData["cleanup"]["status"]);
-        $("#cleanup-last-ran").text(moment.unix(jsonData["cleanup"]["last_ran"]).utc().fromNow());
-
         jsonData["spot_providers"].forEach(p => {
             $("#spot-providers-status-container").append(`
                 <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
@@ -42,6 +39,15 @@ function loadStatus() {
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
                 </div>`);
         });
+
+        jsonData["sig_ref_data_providers"].forEach(p => {
+            $("#sig-ref-data-providers-status-container").append(`
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                    <div class="col"><strong>${p["sig_name"]}</strong></div>
+                    <div class="col">Status: ${p["status"]}</div>
+                    <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
+                </div>`);
+        });
     });
 }
 

templates/status.html

@@ -24,11 +24,6 @@
             <div class="col">Last API call: <span id="web-server-last-api"></span></div>
             <div class="col">Last page req: <span id="web-server-last-page"></span></div>
         </div>
-        <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
-            <div class="col"><strong>Cleanup Service</strong></div>
-            <div class="col">Status: <span id="cleanup-status"></span></div>
-            <div class="col">Last ran: <span id="cleanup-last-ran"></span></div>
-        </div>
     </div>
 </div>
 
@@ -59,6 +54,15 @@
     </div>
 </div>
 
+<div class="card mt-3">
+    <div class="card-header">
+        SIG Reference Data Providers
+    </div>
+    <div class="card-body" id="sig-ref-data-providers-status-container">
+
+    </div>
+</div>
+
 <script src="/static/js/status.js?v=1785434213"></script>
 <script>
     $(document).ready(function () {