Release 1.4

alertproviders/alert_provider.py

@@ -2,7 +2,7 @@ from datetime import datetime
 
 import pytz
 
-from core.data_store import DATA_STORE
+from core.config import MAX_ALERT_AGE
 
 
 class AlertProvider:
@@ -15,7 +15,14 @@ class AlertProvider:
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._alerts = DATA_STORE.alerts
+        self._alerts = None
+        self._web_server = None
+
+    def setup(self, alerts, web_server):
+        """Set up the provider, e.g. giving it the alert list to work from"""
+
+        self._alerts = alerts
+        self._web_server = web_server
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -37,7 +44,10 @@ class AlertProvider:
 
     def _add_alert(self, alert):
         if not alert.expired():
-            self._alerts.set(alert.id, alert)
+            self._alerts.add(alert.id, alert, expire=MAX_ALERT_AGE)
+            # Ping the web server in case we have any SSE connections that need to see this immediately
+            if self._web_server:
+                self._web_server.notify_new_alert(alert)
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""

alertproviders/bota.py

@@ -20,7 +20,7 @@ class BOTA(HTTPAlertProvider):
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
         # Find the table of upcoming alerts
-        bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
+        bs = BeautifulSoup(http_response.content.decode(), features="lxml")
         if not bs.body:
             return new_alerts
         div = bs.body.find('div', attrs={'class': 'view-activations-public'})

alertproviders/ng3k.py

@@ -22,7 +22,7 @@ class NG3K(HTTPAlertProvider):
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
-        rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
+        rss = cast(RSS, Parser.parse(http_response.content.decode()))
         # Iterate through source data
         for source_alert in rss.channel.items:
             # Deal with "the format"...

alertproviders/parksnpeaks.py

@@ -22,7 +22,7 @@ class ParksNPeaks(HTTPAlertProvider):
         # Iterate through source data
         for source_alert in http_response.json():
             # Calculate some things
-            sig = source_alert["Class"].upper()
+            sig = source_alert["Class"]
             if " - " in source_alert["Location"]:
                 split = source_alert["Location"].split(" - ")
                 sig_ref = split[0]
@@ -50,7 +50,7 @@ class ParksNPeaks(HTTPAlertProvider):
                           is_dxpedition=False)
 
             # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-            if sig and sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
+            if sig and sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
                 logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!")
 
             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to

alertproviders/wota.py

@@ -22,7 +22,7 @@ class WOTA(HTTPAlertProvider):
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
-        rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig")))
+        rss = cast(RSS, RSSParser.parse(http_response.content.decode()))
         # Iterate through source data
         for source_alert in rss.channel.items:
 

config-example.yml

@@ -116,30 +116,21 @@ spot-providers:
   - class: "XOTA"
     # xOTA sources have a "name" property so you can define what programme it is for, since xOTA is generic software
     # for running "something-on-the-air" programmes.
-    name: "C3 TOTA"
+    name: "39C3 TOTA"
     enabled: false
     url: "wss://39c3.totawatch.de/api/spot/live"
-    # For the "XOTA" provider, a SIG must be set menually here because xOTA is a generic backend for xOTA
+    # Fixed SIG for all spots from a provider & location CSV are currently only a feature for the "XOTA" provider,
-    # programmes and so different URLs potentially provide different programmes.
+    # the software found at https://github.com/nischu/xOTA/. This is because this is a generic backend for xOTA
-    sig: "Toilets"
+    # programmes and so different URLs provide different programmes.
-    # For Toilets on the Air, we prefix the SIG references (T-01 etc) with some characters that define the conference:
+    sig: "TOTA"
-    # C3, EH or HOPE - so we can look up the correct locations in our database, because each conference starts from T-01
+    locations-csv: "datafiles/39c3-tota.csv"
-    # but refers to a toilet in a different building (or continent!)
-    sig-ref-prefix: "C3"
 
   - class: "XOTA"
     name: "EH23 TOTA"
     enabled: false
     url: "wss://eh23.totawatch.de/api/spot/live"
-    sig: "Toilets"
+    sig: "TOTA"
-    sig-ref-prefix: "EH"
+    locations-csv: "datafiles/eh23-tota.csv"
-
-  - class: "XOTA"
-    name: "HOPE26 TOTA"
-    enabled: false
-    url: "wss://hope-26.totawatch.de/api/spot/live"
-    sig: "Toilets"
-    sig-ref-prefix: "HOPE"
 
 
 # Alert providers to use. Same setup as the spot providers list above.
@@ -166,60 +157,6 @@ alert-providers:
     enabled: true
 
 
-# SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF
-# park IDs to their name and location.
-sig-ref-data-providers:
-  - class: "POTA"
-    enabled: true
-
-  - class: "SOTA"
-    enabled: true
-
-  - class: "WWFF"
-    enabled: true
-
-  - class: "WWBOTA"
-    enabled: true
-
-  - class: "GMA"
-    enabled: true
-
-  - class: "MOTA"
-    enabled: true
-
-  - class: "ILLW"
-    enabled: true
-
-  - class: "ARLHS"
-    enabled: true
-
-  - class: "WCA"
-    enabled: true
-
-  - class: "IOTA"
-    enabled: true
-
-  - class: "WOTA"
-    enabled: true
-
-  - class: "SIOTA"
-    enabled: true
-
-  - class: "ZLOTA"
-    enabled: true
-
-  - class: "LLOTA"
-    enabled: true
-
-  - class: "Towers"
-    enabled: true
-
-  - class: "DME"
-    enabled: true
-
-  - class: "Toilets"
-    enabled: true
-
 # Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band
 # conditions, etc.) and make it available via the /api/v1/solar endpoint.
 solar-condition-providers:

core/cache_utils.py

@@ -0,0 +1,27 @@
+import threading
+from datetime import timedelta
+
+from requests_cache import CachedSession
+
+# Cache for "semi-static" data such as the locations of parks, CSVs of reference lists, etc.
+# This has an expiry time of 30 days, so will re-request from the source after that amount
+# of time has passed. This is used throughout Spothole to cache data that does not change
+# rapidly. The ThreadSafeSession construct here protects it against some multithreading
+# contention weirdness we sometimes used to see on startup where the cache was hammered
+# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type
+# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote
+# servers for data they've told us they can't provide.
+_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30),
+                         allowable_codes=(200, 400, 401, 403, 404))
+_lock = threading.Lock()
+
+
+class _ThreadSafeSession:
+    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads."""
+
+    def get(self, *args, **kwargs):
+        with _lock:
+            return _session.get(*args, **kwargs)
+
+
+SEMI_STATIC_URL_DATA_CACHE = _ThreadSafeSession()

core/cleanup.py

@@ -0,0 +1,73 @@
+import logging
+from datetime import datetime
+from threading import Event, Thread
+
+import pytz
+
+
+class CleanupTimer:
+    """Provides a timed cleanup of the spot list."""
+
+    def __init__(self, spots, alerts, web_server, cleanup_interval):
+        """Constructor"""
+
+        self._spots = spots
+        self._alerts = alerts
+        self._web_server = web_server
+        self._cleanup_interval = cleanup_interval
+        self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Starting"
+        self._thread = None
+        self._stop_event = Event()
+
+    def start(self):
+        """Start the cleanup timer"""
+
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        self._stop_event.set()
+
+    def _run(self):
+        while not self._stop_event.wait(timeout=self._cleanup_interval):
+            self._cleanup()
+
+    def _cleanup(self):
+        """Perform cleanup and reschedule next timer"""
+
+        try:
+            # Perform cleanup via letting the data expire
+            self._spots.expire()
+            self._alerts.expire()
+
+            # Explicitly clean up any spots and alerts that have expired
+            for i in list(self._spots.iterkeys()):
+                try:
+                    spot = self._spots[i]
+                    if spot.expired():
+                        self._spots.delete(i)
+                except KeyError:
+                    # Must have already been deleted, OK with that
+                    pass
+            for i in list(self._alerts.iterkeys()):
+                try:
+                    alert = self._alerts[i]
+                    if alert.expired():
+                        self._alerts.delete(i)
+                except KeyError:
+                    # Must have already been deleted, OK with that
+                    pass
+
+            # Clean up web server SSE spot/alert queues
+            self._web_server.clean_up_sse_queues()
+
+            self.status = "OK"
+            self.last_cleanup_time = datetime.now(pytz.UTC)
+
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in Cleanup thread")
+            self._stop_event.wait(timeout=1)

core/config.py

@@ -1,4 +1,3 @@
-import importlib
 import logging
 import os
 
@@ -34,35 +33,3 @@ WEB_UI_OPTIONS["spot-providers-enabled-by-default"] = [p["name"] for p in config
 # one of our proviers. We set that to also be enabled by default.
 if ALLOW_SPOTTING:
     WEB_UI_OPTIONS["spot-providers-enabled-by-default"].append("API")
-
-
-def get_spot_provider_from_config(config_providers_entry):
-    """Utility method to get a spot provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('spotproviders.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_alert_provider_from_config(config_providers_entry):
-    """Utility method to get an alert provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('alertproviders.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_solar_conditions_provider_from_config(config_providers_entry):
-    """Utility method to get a solar conditions provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('solarconditionsproviders.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_sig_ref_data_provider_from_config(config_providers_entry):
-    """Utility method to get a SIG reference data provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('sigrefdataproviders.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)

core/constants.py

@@ -3,7 +3,7 @@ from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "1.4-pre"
+SOFTWARE_VERSION = "1.4"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": "Spothole v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
@@ -19,9 +19,9 @@ SIGS = [
     SIG(name="HEMA", comment_names=["HEMA"], description="HuMPs Excluding Marilyns Award", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}"),
     SIG(name="IOTA", comment_names=["IOTA"], description="Islands on the Air", ref_regex=r"[A-Z]{2}\-\d{3}"),
     SIG(name="MOTA", comment_names=["MOTA"], description="Mills on the Air", ref_regex=r"X\d{4,6}"),
-    SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}[\- ]\d{3,4}"),
+    SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}\-\d{3,4}"),
     SIG(name="ILLW", comment_names=["ILLW"], description="International Lighthouse & Lightship Weekend", ref_regex=r"[A-Z]{2}\d{4}"),
-    SIG(name="SIOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
+    SIG(name="SiOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
     SIG(name="WCA", comment_names=["WCA"], description="World Castles Award", ref_regex=r"[A-Z0-9]{1,3}\-\d{5}"),
     SIG(name="ZLOTA", comment_names=["ZLOTA"], description="New Zealand on the Air", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}"),
     SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"),

core/data_store.py

@@ -1,80 +0,0 @@
-import logging
-from pathlib import Path
-
-import diskcache
-
-from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
-from core.live_data_cache import LiveDataCache
-from data.solar_conditions import SolarConditions
-
-CACHE_DIR = "./cache/"
-
-
-class DataStore:
-    """Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
-    lookup data using different caching strategies for each."""
-
-    def __init__(self):
-        self._MAX_SPOT_COUNT = 100000
-        self._MAX_ALERT_COUNT = 100000
-        self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
-        self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
-        self.alerts = None
-        self.spots = None
-        self.callsigns = None
-        self.sigrefs = None
-        self.status_data = None
-        self._status = None
-        self.solar_conditions = None
-        self._solar = None
-
-    def setup(self):
-        Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
-
-        # Standard disk cache for solar data and status data, but each cache contains only a single object which we
-        # expose to the wider application
-        self._solar = diskcache.Cache(CACHE_DIR + "solar")
-        if "solar_conditions" not in self._solar:
-            self._solar.add("solar_conditions", SolarConditions())
-        self.solar_conditions = self._solar.get("solar_conditions")
-        self._status = diskcache.Cache(CACHE_DIR + "status")
-        if "status_data" not in self._status:
-            self._status.add("status_data", {})
-        self.status_data = self._status.get("status_data")
-
-        # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
-        # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
-        # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
-        # so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
-        self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
-        logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
-
-        # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
-        # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
-        # we've seen, so we rely on them timing out and this triggering another lookup.
-        self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns")
-        logging.info(f"Loaded data for %d callsigns.", len(self.callsigns))
-
-        # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
-        # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
-        # before the spots and alerts are live in the system.
-        self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
-                                   snapshot_dir=CACHE_DIR + "spots",
-                                   snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
-        logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys()))
-
-        self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
-                                    snapshot_dir=CACHE_DIR + "alerts",
-                                    snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
-        logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
-
-    def close(self):
-        self.spots.close()
-        self.alerts.close()
-        self._solar.close()
-        self._status.close()
-        self.sigrefs.close()
-        self.callsigns.close()
-
-# Global object
-DATA_STORE = DataStore()

core/live_data_cache.py

@@ -1,100 +0,0 @@
-import logging
-import threading
-import time
-
-import diskcache
-from cachetools import TTLCache
-
-
-class LiveDataCache:
-    """Cache for spots and alerts. Uses the fast in-memory TTLCache for normal data I/O, including the TTL to enforce
-    maximum lifetime, and adds a separate diskcache to which we can save and load the TTLCache to provide persistence.
-    Also adds thread safety so spots and alerts can come from any thread, and a listener mechanism so the web server
-    can get a callback when new spots/alerts are added, and send them to any SSE clients."""
-
-    def __init__(self, maxsize, ttl, snapshot_dir, snapshot_interval_sec):
-        self._cache = TTLCache(maxsize=maxsize, ttl=ttl)
-        self._lock = threading.Lock()
-        self._ttl = ttl
-        self._listeners = []
-        self._listeners_lock = threading.Lock()
-        self._snapshot_dir = snapshot_dir
-        self._disk_cache = diskcache.Cache(str(snapshot_dir))
-        self._load_snapshot()
-        self._start_periodic_snapshot(snapshot_interval_sec)
-
-    def set(self, key, value):
-        with self._lock:
-            self._cache[key] = value
-
-        # Notify listeners
-        with self._listeners_lock:
-            listeners = list(self._listeners)
-        for callback in listeners:
-            try:
-                callback(value)
-            except Exception:
-                logging.error("Listener raised an exception for key %s", key, exc_info=True)
-
-
-    def get(self, key, default=None):
-        with self._lock:
-            return self._cache.get(key, default)
-
-    def delete(self, key):
-        with self._lock:
-            self._cache.pop(key, None)
-
-    def keys(self):
-        with self._lock:
-            return list(self._cache.keys())
-
-    def values(self):
-        with self._lock:
-            return list(self._cache.values())
-
-    def add_listener(self, callback):
-        """Register callback(value) which will be called whenever a new spot/alert item is added via set(). Used by the
-        web server (via SSEBroadcaster) to send SSE clients an update on every new spot."""
-
-        with self._listeners_lock:
-            self._listeners.append(callback)
-
-    def remove_listener(self, callback):
-        with self._listeners_lock:
-            self._listeners.remove(callback)
-
-    def save_snapshot(self):
-        with self._lock:
-            # Store the time with the data so we can avoid loading anything nxt time that's older than TTL
-            data = [(k, v, time.time()) for k, v in self._cache.items()]
-        try:
-            self._disk_cache.set("snapshot", data)
-        except Exception as e:
-            logging.error("Failed to write snapshot to %s", self._snapshot_dir, e, exc_info=True)
-
-    def _load_snapshot(self):
-        data = self._disk_cache.get("snapshot")
-        if not data:
-            return
-
-        now = time.time()
-        with self._lock:
-            for key, value, saved_at in data:
-                # Only restore entries that would still be within TTL
-                if now - saved_at < self._ttl:
-                    self._cache[key] = value
-        logging.info("Loaded snapshot from %s", self._snapshot_dir)
-
-    def _start_periodic_snapshot(self, interval):
-        def loop():
-            while True:
-                time.sleep(interval)
-                self.save_snapshot()
-
-        t = threading.Thread(target=loop, daemon=True, name=f"snapshot-{self._snapshot_dir}")
-        t.start()
-
-    def close(self):
-        self.save_snapshot()
-        self._disk_cache.close()

core/lookup_helper.py

@@ -14,10 +14,10 @@ from pyhamtools.locator import latlong_to_locator
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
+from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.config import config
 from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
     HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
-from core.url_data_cache import URLDataCache
 
 # QRZ XML field names differ from pyhamtools' normalised names; map them here.
 _QRZ_FIELD_MAP = {
@@ -29,7 +29,6 @@ _QRZ_FIELD_MAP = {
 }
 _QRZ_INT_FIELDS = {"adif", "cqz", "ituz"}
 _QRZ_FLOAT_FIELDS = {"latitude", "longitude"}
-_URL_DATA_CACHE = URLDataCache("callsign_lookup")
 
 
 def _normalize_qrz_data(raw):
@@ -143,7 +142,7 @@ class LookupHelper:
 
         try:
             logging.info("Downloading Country-files.com cty.plist...")
-            response = _URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
                                                       headers=HTTP_HEADERS)
 
             if response.ok:
@@ -168,7 +167,7 @@ class LookupHelper:
 
         try:
             logging.info("Downloading dxcc.json...")
-            response = _URL_DATA_CACHE.get(
+            response = SEMI_STATIC_URL_DATA_CACHE.get(
                 "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
                 headers=HTTP_HEADERS)
 
@@ -516,7 +515,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = _URL_DATA_CACHE.get(
+                response = SEMI_STATIC_URL_DATA_CACHE.get(
                     self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
                     headers=HTTP_HEADERS, timeout=10)
                 if response.ok:
@@ -594,7 +593,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = _URL_DATA_CACHE.get(
+                response = SEMI_STATIC_URL_DATA_CACHE.get(
                     self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                         lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
                 if response.ok:

core/sig_utils.py

@@ -1,11 +1,26 @@
+import csv
 import logging
 
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
-from core.constants import SIGS
+from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
-from core.data_store import DATA_STORE
+from core.constants import SIGS, HTTP_HEADERS
 from core.geo_utils import wab_wai_square_to_lat_lon
 
+# Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data
+# file in Spothole and load it on startup.
+with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
+    _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
+# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
+# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
+# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
+_WWFF_INDEX_CACHE = {}
+_SIOTA_INDEX_CACHE = {}
+_WOTA_INDEX_CACHE = {}
+_ZLOTA_INDEX_CACHE = {}
+_LLOTA_INDEX_CACHE = {}
+
 
 def get_ref_regex_for_sig(sig):
     """Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
@@ -27,31 +42,220 @@ def get_sig_name_from_comment_name(sig):
 
 
 def populate_sig_ref_info(sig_ref):
-    """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
+    """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object which
-    which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
+    must at minimum have a "sig" and an "id". The rest of the object will be populated and returned.
-    use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
-    SIG we are getting data for.
     Note there is currently no support for KRMNPA location lookup, see issue #61."""
 
-    if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "":
+    if sig_ref.sig is None or sig_ref.id is None:
-        logging.debug("Failed to look up sig_ref info, sig or id were not set.")
+        logging.warning("Failed to look up sig_ref info, sig or id were not set.")
-        return sig_ref
 
-    sig = sig_ref.sig
+    sig = sig_ref.sig or ""
     ref_id = sig_ref.id
-
-    # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all activators
-    # add leading zeros.
-    if sig.upper() == "DME":
-        ref_id = ref_id.zfill(5)
-
     try:
-        # If the SIG is HEMA or KRMNPA, we have no current lookup for this so just skip it.
+        if sig.upper() == "POTA":
-        if sig.upper() == "HEMA" or sig.upper() == "KRMNPA":
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
-            return sig_ref
+            if response.ok:
+                data = response.json()
+                if data:
+                    fullname = str(data["name"]) if "name" in data else None
+                    if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
+                        fullname = fullname + " " + data["parktypeDesc"]
+                    sig_ref.name = fullname
+                    sig_ref.url = "https://pota.app/#/park/" + ref_id
+                    sig_ref.grid = data["grid6"] if "grid6" in data else None
+                    sig_ref.latitude = data["latitude"] if "latitude" in data else None
+                    sig_ref.longitude = data["longitude"] if "longitude" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
+        elif sig.upper() == "SOTA":
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
+                                                      headers=HTTP_HEADERS)
+            if response.ok:
+                data = response.json()
+                if data:
+                    sig_ref.name = data["name"] if "name" in data else None
+                    sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
+                    sig_ref.grid = data["locator"] if "locator" in data else None
+                    sig_ref.latitude = data["latitude"] if "latitude" in data else None
+                    sig_ref.longitude = data["longitude"] if "longitude" in data else None
+                    sig_ref.activation_score = data["points"] if "points" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
+        elif sig.upper() == "WWBOTA":
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
+                                                      headers=HTTP_HEADERS)
+            if response.ok:
+                data = response.json()
+                if data:
+                    sig_ref.name = data["name"] if "name" in data else None
+                    sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
+                    sig_ref.grid = data["locator"] if "locator" in data else None
+                    sig_ref.latitude = data["lat"] if "lat" in data else None
+                    sig_ref.longitude = data["long"] if "long" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
+        elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
+                                                      headers=HTTP_HEADERS)
+            if response.ok:
+                data = response.json()
+                if data:
+                    sig_ref.name = data["name"] if "name" in data else None
+                    sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
+                    sig_ref.grid = data["locator"] if "locator" in data else None
+
+                    # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
+                    # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
+                    # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
+                    # just use the single point we have.
+                    if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
+                            "lat2") is not None and data.get("lng2") is not None:
+                        sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
+                        sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
+                    else:
+                        sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
+                        sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
+        elif sig.upper() == "WWFF":
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
+                                                      headers=HTTP_HEADERS)
+            if response.ok:
+                global _WWFF_INDEX_CACHE
+                if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
+                    # New data from WWFF, update our internal map
+                    _WWFF_INDEX_CACHE = {row["reference"]: row for row in
+                                         csv.DictReader(response.content.decode().splitlines())}
+                row = _WWFF_INDEX_CACHE.get(ref_id)
+                if row:
+                    sig_ref.name = row["name"] if "name" in row else None
+                    sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
+                    sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
+                    sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
+                    sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[
+                        "longitude"] != "-" else None
+                elif not response.from_cache:
+                    logging.warning("WWFF database did not contain data for ref %s", ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
+        elif sig.upper() == "SIOTA":
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
+                                                      headers=HTTP_HEADERS)
+            if response.ok:
+                global _SIOTA_INDEX_CACHE
+                if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
+                    # New data from SIOTA, update our internal map
+                    _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
+                                          csv.DictReader(response.content.decode().splitlines())}
+                row = _SIOTA_INDEX_CACHE.get(ref_id)
+                if row:
+                    sig_ref.name = row["NAME"] if "NAME" in row else None
+                    sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
+                    sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
+                    sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
+                elif not response.from_cache:
+                    logging.warning("SIOTA database did not contain data for ref %s", ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
+        elif sig.upper() == "WOTA":
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
+                                                      headers=HTTP_HEADERS)
+            if response.ok:
+                data = response.json()
+                if data:
+                    global _WOTA_INDEX_CACHE
+                    if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from WOTA, update our internal map
+                        _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
+                                             data.get("features", [])}
+                    feature = _WOTA_INDEX_CACHE.get(ref_id)
+                    if feature:
+                        sig_ref.name = feature["properties"]["title"]
+                        # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
+                        # added to them
+                        sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id
+                        if ref_id.upper().startswith("LDO-"):
+                            number = int(ref_id.upper().replace("LDO-", ""))
+                            sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
+                        sig_ref.grid = feature["properties"]["qthLocator"]
+                        sig_ref.latitude = feature["geometry"]["coordinates"][1]
+                        sig_ref.longitude = feature["geometry"]["coordinates"][0]
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
+        elif sig.upper() == "ZLOTA":
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
+            if response.ok:
+                data = response.json()
+                if isinstance(data, list):
+                    global _ZLOTA_INDEX_CACHE
+                    if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from ZLOTA, update our internal map
+                        _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
+                    asset = _ZLOTA_INDEX_CACHE.get(ref_id)
+                    if asset:
+                        sig_ref.name = asset["name"]
+                        sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
+                        try:
+                            sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
+                        except:
+                            logging.debug("Invalid lat/lon received for reference")
+                            sig_ref.latitude = asset["y"]
+                            sig_ref.longitude = asset["x"]
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
+        elif sig.upper() == "BOTA":
+            if not sig_ref.name:
+                sig_ref.name = sig_ref.id
+            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
+
+        elif sig.upper() == "LLOTA":
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
+                                                      headers=HTTP_HEADERS)
+            if response.ok:
+                data = response.json()
+                if isinstance(data, list):
+                    global _LLOTA_INDEX_CACHE
+                    if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from LLOTA, update our internal map
+                        _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
+                    ref = _LLOTA_INDEX_CACHE.get(ref_id)
+                    if ref:
+                        sig_ref.name = str(ref["name"])
+                        sig_ref.url = "https://llota.app/list/ref/" + ref_id
+                        sig_ref.grid = str(ref["grid_locator"])
+                        ll = locator_to_latlong(sig_ref.grid)
+                        sig_ref.latitude = ll[0]
+                        sig_ref.longitude = ll[1]
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
+        elif sig.upper() == "WWTOTA":
+            if not sig_ref.name:
+                sig_ref.name = sig_ref.id
+            sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + str(sig_ref.name)
 
-        # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
-        # calculate all the information we are going to get directly. So handle those cases first
         elif sig.upper() == "TILES":
             # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
             if not sig_ref.name:
@@ -74,26 +278,27 @@ def populate_sig_ref_info(sig_ref):
                 except:
                     logging.warning("Invalid lat/lon received for WAB/WAI reference")
 
-        elif sig.upper() == "BOTA":
+        elif sig.upper() == "DME":
-            # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
+            # Zero-pad to 5 digits to match our source data
-            if not sig_ref.name:
+            row = _DME_INDEX.get(ref_id.zfill(5))
-                sig_ref.name = sig_ref.id
+            if row:
-            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
+                sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
-
+                sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
-        # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
+                    "LATITUD_ETRS89_REGCAN95") else None
-        # this SIG. If so, check for the reference data and use that.
+                sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
-        elif sig in DATA_STORE.sigrefs:
+                    "LONGITUD_ETRS89_REGCAN95") else None
-            key = sig + ":" + ref_id
+                if sig_ref.latitude and sig_ref.longitude:
-            lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
+                    try:
-            if lookup_data:
+                        sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6)
-                # Copy new sig ref data into existing object
+                    except Exception:
-                sig_ref.__dict__.update(lookup_data.__dict__)
+                        logging.warning("Invalid lat/lon received for DME reference")
             else:
-                logging.warning("%s database did not contain data for ref %s", sig, ref_id)
+                logging.warning("DME database did not contain data for ref %s", ref_id)
-
-        else:
-            logging.warning(f"Tried to look up a SIG called %s but Spothole does not know what that is.", sig)
 
+    except ConnectionError:
+        logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
+    except (ConnectTimeout, ReadTimeout):
+        logging.warning(f"Timeout when looking up sig_ref info for " + sig + " ref " + ref_id)
     except Exception:
         logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
     return sig_ref

core/status_reporter.py

@@ -7,29 +7,31 @@ import pytz
 
 from core.config import SERVER_OWNER_CALLSIGN
 from core.constants import SOFTWARE_VERSION
-from core.data_store import DATA_STORE
 from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge
 
 
 class StatusReporter:
     """Provides a timed update of the application's status data."""
 
-    def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers,
+    def __init__(self, status_data, run_interval, web_server, cleanup_timer, spots, spot_providers, alerts,
-                 sig_ref_data_providers):
+                 alert_providers, solar_condition_providers):
         """Constructor"""
 
+        self._status_data = status_data
         self._run_interval = run_interval
         self._web_server = web_server
+        self._cleanup_timer = cleanup_timer
+        self._spots = spots
         self._spot_providers = spot_providers
+        self._alerts = alerts
         self._alert_providers = alert_providers
         self._solar_condition_providers = solar_condition_providers
-        self._sig_ref_data_providers = sig_ref_data_providers
         self._thread = None
         self._stop_event = Event()
         self._startup_time = datetime.now(pytz.UTC)
 
-        DATA_STORE.status_data["software-version"] = SOFTWARE_VERSION
+        self._status_data["software-version"] = SOFTWARE_VERSION
-        DATA_STORE.status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
+        self._status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
 
     def start(self):
         """Start the reporter thread"""
@@ -53,34 +55,31 @@ class StatusReporter:
     def _report(self):
         """Write status information"""
 
-        DATA_STORE.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
+        self._status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
-        DATA_STORE.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
+        self._status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
-        DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
+        self._status_data["num_spots"] = len(self._spots)
-        DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
+        self._status_data["num_alerts"] = len(self._alerts)
-        DATA_STORE.status_data["spot_providers"] = list(
+        self._status_data["spot_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
                                tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
                            "last_spot": p.last_spot_time.replace(
                                tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
                 self._spot_providers))
-        DATA_STORE.status_data["alert_providers"] = list(
+        self._status_data["alert_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
                                tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
                 self._alert_providers))
-        DATA_STORE.status_data["solar_condition_providers"] = list(
+        self._status_data["solar_condition_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
                                tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
                 self._solar_condition_providers))
-        DATA_STORE.status_data["sig_ref_data_providers"] = list(
+        self._status_data["cleanup"] = {"status": self._cleanup_timer.status,
-            map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
+                                        "last_ran": self._cleanup_timer.last_cleanup_time.replace(
-                           "last_updated": p.last_update_time.replace(
+                                            tzinfo=pytz.UTC).timestamp() if self._cleanup_timer.last_cleanup_time else 0}
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
+        self._status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
-                           "reference_count": p.reference_count},
-                self._sig_ref_data_providers))
-        DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
                                           "last_api_access": self._web_server.web_server_metrics[
                                               "last_api_access_time"].replace(
                                               tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
@@ -95,5 +94,5 @@ class StatusReporter:
 
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)
-        spots_gauge.set(len(DATA_STORE.spots.values()))
+        spots_gauge.set(len(self._spots))
-        alerts_gauge.set(len(DATA_STORE.alerts.values()))
+        alerts_gauge.set(len(self._alerts))

core/url_data_cache.py

@@ -1,25 +0,0 @@
-import threading
-from datetime import timedelta
-
-from requests_cache import CachedSession
-
-from core.data_store import CACHE_DIR
-
-
-class URLDataCache(CachedSession):
-    """Cache for "semi-static" data retrieved from a URL. This is an extra layer of caching in addition to the normal
-    caching of spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints
-    frequently when e.g. restarting Spothole many times during testing. It should only be used for data that isn't
-    expected to change on a sub-daily basis, so never for spots & alerts. It also implements a lock to allow it to be
-    used across multiple threads, though note that URL lookups will block each other this way, so it is still better to
-    create one of these objects per thread if possible."""
-
-    _lock = threading.Lock()
-
-    def __init__(self, name):
-        super().__init__(CACHE_DIR + "urls/" + name, expire_after=timedelta(days=1),
-                         allowable_codes=(200, 400, 401, 403, 404))
-
-    def get(self, *args, **kwargs):
-        with self._lock:
-            return super().get(*args, **kwargs)

core/utils.py

@@ -6,3 +6,13 @@ def safe_json_dumps(obj):
     which are invalid in JSON."""
 
     return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
+
+
+def empty_queue(q):
+    """Empty a queue"""
+
+    while not q.empty():
+        try:
+            q.get_nowait()
+        except:
+            break

data/alert.py

@@ -1,7 +1,6 @@
 import copy
 import hashlib
 import json
-import logging
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 
@@ -65,7 +64,6 @@ class Alert:
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
 
-        try:
         # If we somehow don't have a start time, set it to zero so it sorts off the bottom of any list but
         # clients can still reliably parse it as a number.
         if not self.start_time:
@@ -130,9 +128,6 @@ class Alert:
             self.dx_names = list(
                 map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c, credentials), self.dx_calls))
 
-        except Exception as e:
-            logging.error("Exception while inferring missing data from spot", e, exc_info=True)
-
     def to_json(self):
         """JSON serialise"""
 

data/spot.py

@@ -147,7 +147,6 @@ class Spot:
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
 
-        try:
         # If we somehow don't have a spot time, set it to zero so it sorts off the bottom of any list but
         # clients can still reliably parse it as a number.
         if not self.time:
@@ -438,9 +437,6 @@ class Spot:
                     self.de_longitude = latlon[1]
                     self.de_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.de_call)
 
-        except Exception as e:
-            logging.error("Exception while inferring missing data from spot", e, exc_info=True)
-
     def to_json(self):
         """JSON serialise"""
 

datafiles/39c3-tota.csv

@@ -0,0 +1,18 @@
+ref,lat,lon
+T-01,53.56278090617755,9.984341869295505
+T-02,53.562383404176416,9.98551893027115
+T-03,53.56170184391514,9.985416035619778
+T-04,53.562026534393176,9.986372919078974
+T-11,53.56284641242506,9.98475590239655
+T-12,53.562431705517035,9.98551675702443
+T-13,53.56223704898424,9.985774520335664
+T-14,53.5617893512591,9.986344302837976
+T-21,53.56284641242506,9.98475590239655
+T-22,53.56245816412497,9.985456089490567
+T-23,53.56199560857136,9.985636761412673
+T-24,53.5617893512591,9.986344302837976
+T-31,53.56247470064887,9.985611427551902
+T-32,53.5617893512591,9.986344302837976
+T-41,53.56245039134992,9.985486136112701
+T-91,53.56147934973529,9.984626806439744
+T-92,53.561396810300735,9.987553052152899

datafiles/eh23-tota.csv

@@ -0,0 +1,13 @@
+ref,lat,lon
+T-01,50.3636495,7.5584857
+T-02,50.3636495,7.5584857
+T-03,50.3636495,7.5584857
+T-11,50.3636495,7.5584857
+T-13,50.3636495,7.5584857
+T-14,50.3636495,7.5584857
+T-21,50.3636495,7.5584857
+T-31,50.3636495,7.5584857
+T-33,50.3636495,7.5584857
+T-34,50.3636495,7.5584857
+T-41,50.3636495,7.5584857
+T-51,50.3636495,7.5584857

datafiles/toilets.csv

@@ -1,30 +0,0 @@
-ref,lat,lon
-C3 T-01,53.56278090617755,9.984341869295505
-C3 T-02,53.562383404176416,9.98551893027115
-C3 T-03,53.56170184391514,9.985416035619778
-C3 T-04,53.562026534393176,9.986372919078974
-C3 T-11,53.56284641242506,9.98475590239655
-C3 T-12,53.562431705517035,9.98551675702443
-C3 T-13,53.56223704898424,9.985774520335664
-C3 T-14,53.5617893512591,9.986344302837976
-C3 T-21,53.56284641242506,9.98475590239655
-C3 T-22,53.56245816412497,9.985456089490567
-C3 T-23,53.56199560857136,9.985636761412673
-C3 T-24,53.5617893512591,9.986344302837976
-C3 T-31,53.56247470064887,9.985611427551902
-C3 T-32,53.5617893512591,9.986344302837976
-C3 T-41,53.56245039134992,9.985486136112701
-C3 T-91,53.56147934973529,9.984626806439744
-C3 T-92,53.561396810300735,9.987553052152899
-EH T-01,50.3636495,7.5584857
-EH T-02,50.3636495,7.5584857
-EH T-03,50.3636495,7.5584857
-EH T-11,50.3636495,7.5584857
-EH T-13,50.3636495,7.5584857
-EH T-14,50.3636495,7.5584857
-EH T-21,50.3636495,7.5584857
-EH T-31,50.3636495,7.5584857
-EH T-33,50.3636495,7.5584857
-EH T-34,50.3636495,7.5584857
-EH T-41,50.3636495,7.5584857
-EH T-51,50.3636495,7.5584857

requirements.txt

@@ -18,4 +18,3 @@ tornado~=6.4.2
 tornado_eventsource~=3.0.0
 geopandas~=0.13.2
 simplejson~=4.1.1
-cachetools~=7.1.6

server/handlers/api/addspot.py

@@ -8,7 +8,7 @@ import tornado
 from tornado import httputil
 from tornado.web import Application
 
-from core.config import ALLOW_SPOTTING
+from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
 from core.constants import UNKNOWN_BAND
 from core.lookup_helper import infer_band_from_freq
 from core.prometheus_metrics_handler import api_requests_counter
@@ -119,7 +119,7 @@ class APISpotHandler(tornado.web.RequestHandler):
             # infer missing data, and add it to our database.
             spot.source = "API"
             spot.infer_missing()
-            self._spots.set(spot.id, spot)
+            self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
 
             self.write(safe_json_dumps("OK"))
             self.set_status(201)

server/handlers/api/alerts.py

@@ -1,6 +1,7 @@
 import copy
 import logging
 from datetime import datetime
+from queue import Queue
 from typing import Any
 
 import pytz
@@ -10,9 +11,12 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import safe_json_dumps
+from core.utils import safe_json_dumps, empty_queue
 from data.lookup_credentials import extract_credentials
 
+SSE_HANDLER_MAX_QUEUE_SIZE = 100
+SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
+
 
 class APIAlertsHandler(tornado.web.RequestHandler):
     """API request handler for /api/v1/alerts"""
@@ -69,14 +73,16 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
     """API request handler for /api/v1/alerts/stream"""
 
     def __init__(self, application, request, **kwargs: Any):
-        self._sse_alert_broadcaster = None
+        self._sse_alert_queues = None
         self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
+        self._alert_queue = None
+        self._heartbeat = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, _sse_alert_broadcaster, web_server_metrics):
+    def initialize(self, sse_alert_queues, web_server_metrics):
-        self._sse_alert_broadcaster = _sse_alert_broadcaster
+        self._sse_alert_queues = sse_alert_queues
         self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
@@ -98,32 +104,59 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
             self._credentials = extract_credentials(self._query_params)
 
+            # Create a alert queue and add it to the web server's list. The web server will fill this when alerts arrive
+            self._alert_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
+            self._sse_alert_queues.append(self._alert_queue)
+
+            # Set up a timed callback to check if anything is in the queue
+            self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
+            self._heartbeat.start()
+
             # Flush headers immediately so nginx doesn't time out waiting for a response
             self.write_message("keepalive", "")
 
-            # Register to handle new alerts arriving. The callback() method will get called with the new alert as an
-            # argument.
-            self._sse_alert_broadcaster.register(self)
-
         except Exception as e:
             logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
             self.close()
 
     def close(self):
-        """When the user closes the socket, deregister ourselves from the alert broadcaster"""
+        """When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
-
-        self._sse_alert_broadcaster.unregister(self)
-        super().close()
-
-    def callback(self, alert):
-        """Callback when a new alert arrives"""
 
         try:
+            if self._alert_queue in self._sse_alert_queues:
+                self._sse_alert_queues.remove(self._alert_queue)
+            empty_queue(self._alert_queue)
+        except:
+            pass
+        try:
+            self._heartbeat.stop()
+        except:
+            pass
+        self._alert_queue = None
+        super().close()
+
+    def _callback(self):
+        """Callback to check if anything has arrived in the queue, and if so send it to the client"""
+
+        try:
+            if self._alert_queue:
+                if not self._alert_queue.empty():
+                    while not self._alert_queue.empty():
+                        alert = self._alert_queue.get()
+                        # If the new alert matches our param filters, send it to the client. If not, ignore it.
                         if alert_allowed_by_query(alert, self._query_params):
                             if self._credentials:
                                 alert = copy.deepcopy(alert)
                                 alert.infer_missing(self._credentials)
                             self.write_message(msg=safe_json_dumps(alert))
+
+                else:
+                    # Send a keepalive comment if the queue was empty
+                    self.write_message("keepalive", "")
+
+                if self._alert_queue not in self._sse_alert_queues:
+                    logging.error("Web server cleared up a queue of an active connection!")
+                    self.close()
         except Exception as e:
             logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
             self.close()
@@ -136,7 +169,7 @@ def get_alert_list_with_filters(all_alerts, query):
     # Create a shallow copy of the alert list ordered by start time, then filter the list to reduce it only to alerts
     # that match the filter parameters in the query string. Finally, apply a limit to the number of alerts returned.
     # The list of query string filters is defined in the API docs.
-    alert_ids = all_alerts.keys()
+    alert_ids = list(all_alerts.iterkeys())
     alerts = []
     for k in alert_ids:
         a = all_alerts.get(k)

server/handlers/api/dxstats.py

@@ -40,7 +40,7 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
             one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
             counts = Counter()
 
-            for key in self._spots.keys():
+            for key in self._spots.iterkeys():
                 spot = self._spots.get(key)
                 if spot is None:
                     continue

server/handlers/api/spots.py

@@ -1,6 +1,7 @@
 import copy
 import logging
 from datetime import datetime, timedelta
+from queue import Queue
 from typing import Any
 
 import pytz
@@ -10,9 +11,12 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import safe_json_dumps
+from core.utils import safe_json_dumps, empty_queue
 from data.lookup_credentials import extract_credentials
 
+SSE_HANDLER_MAX_QUEUE_SIZE = 1000
+SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
+
 
 class APISpotsHandler(tornado.web.RequestHandler):
     """API request handler for /api/v1/spots"""
@@ -69,14 +73,16 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
     """API request handler for /api/v1/spots/stream"""
 
     def __init__(self, application, request, **kwargs: Any):
-        self._sse_spot_broadcaster = None
+        self._sse_spot_queues = None
         self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
+        self._spot_queue = None
+        self._heartbeat = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, sse_spot_broadcaster, web_server_metrics):
+    def initialize(self, sse_spot_queues, web_server_metrics):
-        self._sse_spot_broadcaster = sse_spot_broadcaster
+        self._sse_spot_queues = sse_spot_queues
         self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
@@ -100,33 +106,59 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
             self._credentials = extract_credentials(self._query_params)
 
+            # Create a spot queue and add it to the web server's list. The web server will fill this when spots arrive
+            self._spot_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
+            self._sse_spot_queues.append(self._spot_queue)
+
+            # Set up a timed callback to check if anything is in the queue
+            self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
+            self._heartbeat.start()
+
             # Flush headers immediately so nginx doesn't time out waiting for a response
             self.write_message("keepalive", "")
 
-            # Register to handle new spots arriving. The callback() method will get called with the new spot as an
-            # argument.
-            self._sse_spot_broadcaster.register(self)
-
         except Exception as e:
             logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
             self.close()
 
     def close(self):
-        """When the user closes the socket, deregister ourselves from the spot broadcaster"""
+        """When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
-
-        self._sse_spot_broadcaster.unregister(self)
-        super().close()
-
-    def callback(self, spot):
-        """Callback when a new spot arrives"""
 
         try:
+            if self._spot_queue in self._sse_spot_queues:
+                self._sse_spot_queues.remove(self._spot_queue)
+            empty_queue(self._spot_queue)
+        except:
+            pass
+        try:
+            self._heartbeat.stop()
+        except:
+            pass
+        self._spot_queue = None
+        super().close()
+
+    def _callback(self):
+        """Callback to check if anything has arrived in the queue, and if so send it to the client"""
+
+        try:
+            if self._spot_queue:
+                if not self._spot_queue.empty():
+                    while not self._spot_queue.empty():
+                        spot = self._spot_queue.get()
                         # If the new spot matches our param filters, send it to the client. If not, ignore it.
                         if spot_allowed_by_query(spot, self._query_params):
                             if self._credentials:
                                 spot = copy.deepcopy(spot)
                                 spot.infer_missing(self._credentials)
                             self.write_message(msg=safe_json_dumps(spot))
+
+                else:
+                    # Send a keepalive comment if the queue was empty
+                    self.write_message("keepalive", "")
+
+                if self._spot_queue not in self._sse_spot_queues:
+                    logging.error("Web server cleared up a queue of an active connection!")
+                    self.close()
         except Exception as e:
             logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
             self.close()
@@ -139,7 +171,7 @@ def get_spot_list_with_filters(all_spots, query):
     # Create a shallow copy of the spot list, ordered by spot time, then filter the list to reduce it only to spots
     # that match the filter parameters in the query string. Finally, apply a limit to the number of spots returned.
     # The list of query string filters is defined in the API docs.
-    spot_ids = all_spots.keys()
+    spot_ids = list(all_spots.iterkeys())
     spots = []
     for k in spot_ids:
         s = all_spots.get(k)

server/sse_broadcaster.py

@@ -1,36 +0,0 @@
-import logging
-import threading
-
-from tornado.ioloop import IOLoop
-
-
-class SSEBroadcaster:
-    """Bridge between DataStore listener callbacks (which fire on provider threads) to Tornado's async SSE handlers
-    (which live on the IOLoop thread) to avoid any interdependency between them."""
-
-    def __init__(self):
-        self._handlers = set()
-        self._lock = threading.Lock()
-        self._loop = IOLoop.current()
-
-    def register(self, handler):
-        with self._lock:
-            self._handlers.add(handler)
-
-    def unregister(self, handler):
-        with self._lock:
-            self._handlers.discard(handler)
-
-    def publish(self, value):
-        self._loop.add_callback(self._fan_out, value)
-
-    def _fan_out(self, value):
-        with self._lock:
-            handlers = list(self._handlers)
-        for handler in handlers:
-            try:
-                handler.callback(value)
-            except Exception:
-                # Connection probably dropped, ignore and de-register the handler to stop getting future items.
-                logging.debug("Failed to push to an SSE client; dropping it")
-                self.unregister(handler)

server/webserver.py

@@ -6,7 +6,7 @@ import tornado
 from tornado.web import StaticFileHandler
 
 from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL
-from core.data_store import DATA_STORE
+from core.utils import empty_queue
 from server.handlers.api.addspot import APISpotHandler
 from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
 from server.handlers.api.dxstats import APIDxStatsHandler
@@ -18,7 +18,6 @@ from server.handlers.api.status import APIStatusHandler
 from server.handlers.manifesthandler import ManifestHandler
 from server.handlers.metrics import PrometheusMetricsHandler
 from server.handlers.pagetemplate import PageTemplateHandler
-from server.sse_broadcaster import SSEBroadcaster
 
 _HERE = os.path.dirname(__file__ or "")
 
@@ -26,12 +25,15 @@ _HERE = os.path.dirname(__file__ or "")
 class WebServer:
     """Provides the public-facing web server."""
 
-    def __init__(self):
+    def __init__(self, spots, alerts, solar_conditions, status_data):
         """Constructor"""
 
-        self._data_store = DATA_STORE
+        self._spots = spots
-        self._spot_broadcaster = SSEBroadcaster()
+        self._alerts = alerts
-        self._alert_broadcaster = SSEBroadcaster()
+        self._solar_conditions = solar_conditions
+        self._sse_spot_queues = []
+        self._sse_alert_queues = []
+        self._status_data = status_data
         self._port = WEB_SERVER_PORT
         self._api_only_mode = API_ONLY_MODE
         self._shutdown_event = asyncio.Event()
@@ -43,10 +45,6 @@ class WebServer:
             "status": "Starting"
         }
 
-        # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
-        DATA_STORE.spots.add_listener(self._spot_broadcaster.publish)
-        DATA_STORE.alerts.add_listener(self._alert_broadcaster.publish)
-
     def start(self):
         """Start the web server"""
 
@@ -66,21 +64,20 @@ class WebServer:
 
         # API endpoints are always enabled
         api_routes = [
-            (r"/api/v1/spots", APISpotsHandler, {"spots": self._data_store.spots, **handler_opts}),
+            (r"/api/v1/spots", APISpotsHandler, {"spots": self._spots, **handler_opts}),
-            (r"/api/v1/alerts", APIAlertsHandler, {"alerts": self._data_store.alerts, **handler_opts}),
+            (r"/api/v1/alerts", APIAlertsHandler, {"alerts": self._alerts, **handler_opts}),
             (r"/api/v1/spots/stream", APISpotsStreamHandler,
-             {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts}),
+             {"sse_spot_queues": self._sse_spot_queues, **handler_opts}),
             (r"/api/v1/alerts/stream", APIAlertsStreamHandler,
-             {"sse_alert_broadcaster": self._alert_broadcaster, **handler_opts}),
+             {"sse_alert_queues": self._sse_alert_queues, **handler_opts}),
-            (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._data_store.solar_conditions,
+            (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._solar_conditions, **handler_opts}),
-                                                           **handler_opts}),
+            (r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._spots, **handler_opts}),
-            (r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._data_store.spots, **handler_opts}),
+            (r"/api/v1/options", APIOptionsHandler, {"status_data": self._status_data, **handler_opts}),
-            (r"/api/v1/options", APIOptionsHandler, {"status_data": self._data_store.status_data, **handler_opts}),
+            (r"/api/v1/status", APIStatusHandler, {"status_data": self._status_data, **handler_opts}),
-            (r"/api/v1/status", APIStatusHandler, {"status_data": self._data_store.status_data, **handler_opts}),
             (r"/api/v1/lookup/call", APILookupCallHandler, {**handler_opts}),
             (r"/api/v1/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
             (r"/api/v1/lookup/grid", APILookupGridHandler, {**handler_opts}),
-            (r"/api/v1/spot", APISpotHandler, {"spots": self._data_store.spots, **handler_opts}),
+            (r"/api/v1/spot", APISpotHandler, {"spots": self._spots, **handler_opts}),
         ]
 
         # If in API-only mode, serve a basic homepage; in normal mode, serve the usual UI routes
@@ -124,6 +121,55 @@ class WebServer:
         logging.info("You can access your copy of Spothole at " + BASE_URL)
         await self._shutdown_event.wait()
 
+    def notify_new_spot(self, spot):
+        """Internal method called when a new spot is added to the system. This is used to ping any SSE clients that are
+        awaiting a server-sent message with new spots."""
+
+        for queue in self._sse_spot_queues:
+            try:
+                queue.put(spot)
+            except:
+                # Cleanup thread was probably deleting the queue, that's fine
+                pass
+        pass
+
+    def notify_new_alert(self, alert):
+        """Internal method called when a new alert is added to the system. This is used to ping any SSE clients that are
+        awaiting a server-sent message with new spots."""
+
+        for queue in self._sse_alert_queues:
+            try:
+                queue.put(alert)
+            except:
+                # Cleanup thread was probably deleting the queue, that's fine
+                pass
+        pass
+
+    def clean_up_sse_queues(self):
+        """Clean up any SSE queues that are growing too large; probably their client disconnected and we didn't catch it
+        properly for some reason."""
+
+        for q in self._sse_spot_queues:
+            try:
+                if q.full():
+                    logging.warning(
+                        "A full SSE spot queue was found, presumably because the client disconnected strangely. It has been removed.")
+                    self._sse_spot_queues.remove(q)
+                    empty_queue(q)
+            except:
+                # Probably got deleted already on another thread
+                pass
+        for q in self._sse_alert_queues:
+            try:
+                if q.full():
+                    logging.warning(
+                        "A full SSE alert queue was found, presumably because the client disconnected strangely. It has been removed.")
+                    self._sse_alert_queues.remove(q)
+                    empty_queue(q)
+            except:
+                # Probably got deleted already on another thread
+                pass
+        pass
 
 def request_log(handler):
     """Custom log function to provide more data about requests when enabled, and to provide the ability to turn off

sigrefdataproviders/arlhs.py

@@ -1,30 +0,0 @@
-import csv
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class ARLHS(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Amateur Radio Light House Society"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "ARLHS"
-    DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
-            if "ARLHS" in row and row["ARLHS"] != "":
-                ref_id = row["ARLHS"]
-                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
-                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
-                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
-                                              "Latitude"] != "" else None,
-                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
-                                              "Longitude"] != "" else None,
-                                          grid=row["Maidenhead Locator"]))
-
-        return new_data

sigrefdataproviders/dme.py

@@ -1,35 +0,0 @@
-import csv
-
-from pyhamtools.locator import latlong_to_locator
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
-
-
-class DME(LocalFileSIGRefDataProvider):
-    """SIG ref data provider for Diploma Municipios de Espana"""
-
-    SIG = "DME"
-    PATH = "datafiles/MUNICIPIOS.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.PATH)
-
-    def _file_to_data(self, path):
-        new_data = []
-        with open(path, encoding="latin-1") as _f:
-            for row in csv.DictReader(_f, delimiter=";"):
-                ref_id = row["COD_INE"][:5]
-                latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
-                    "LATITUD_ETRS89_REGCAN95") else None
-                longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
-                    "LONGITUD_ETRS89_REGCAN95") else None
-
-                ref = SIGRef(sig=self.SIG, id=ref_id,
-                                          name=row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"],
-                                          latitude=latitude,
-                                          longitude=longitude)
-                if latitude and longitude:
-                    ref.grid = latlong_to_locator(latitude, longitude, 6)
-                new_data.append(ref)
-        return new_data

sigrefdataproviders/file_download_sig_ref_data_provider.py

@@ -1,78 +0,0 @@
-import logging
-from datetime import datetime
-from threading import Thread, Event
-
-import pytz
-from requests import ReadTimeout
-from requests.exceptions import ConnectionError, ConnectTimeout
-
-from core.constants import HTTP_HEADERS
-from core.url_data_cache import URLDataCache
-from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
-
-
-class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
-    """Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
-
-    def __init__(self, sig_name, provider_config, url, poll_interval):
-        """ Set up the provider, note poll_interval is in *days*."""
-        super().__init__(sig_name, provider_config)
-        self._url = url
-        self._poll_interval = poll_interval
-        self._thread = None
-        self._stop_event = Event()
-        self._url_data_cache = URLDataCache("sigrefdata-" + sig_name)
-
-    def start(self):
-        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
-        # subsequent polls, so start() returns immediately and the application can continue starting.
-        logging.info(
-            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " days.")
-        self._thread = Thread(target=self._run, daemon=True)
-        self._thread.start()
-
-    def stop(self):
-        self._stop_event.set()
-
-    def _run(self):
-        while True:
-            self._poll()
-            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
-                break
-
-    def _poll(self):
-        try:
-            # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
-            # caching, but just so continual restarts of the software during testing don't hammer the servers.
-            logging.debug("Downloading " + self.sig_name + " SIG ref data...")
-            http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
-            # Check response code was good
-            if http_response.ok:
-                # Pass off to the subclass for processing
-                new_data = self._http_response_to_data(http_response)
-                # Submit the new spots for processing. There might not be any spots for the less popular programs.
-                if new_data:
-                    self._add_data(new_data)
-
-                self.status = "OK"
-                self.last_update_time = datetime.now(pytz.UTC)
-                logging.debug("Received SIG ref data for " + self.sig_name)
-            else:
-                self.status = "Error"
-                logging.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.")
-
-        except ConnectionError:
-            logging.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.")
-        except (ConnectTimeout, ReadTimeout):
-            logging.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.")
-        except Exception:
-            self.status = "Error"
-            logging.exception("Exception in HTTP SIG Ref Data Provider (" + self.sig_name + ")")
-            self._stop_event.wait(timeout=1)
-
-    def _http_response_to_data(self, http_response):
-        """Convert an HTTP response returned by the server into SIG Ref data. The whole response is provided here so the
-        subclass implementations can check for HTTP status codes if necessary, and handle the response as JSON, CSV,
-        whatever the remote file actually is."""
-
-        raise NotImplementedError("Subclasses must implement this method")

sigrefdataproviders/gma.py

@@ -1,29 +0,0 @@
-import csv
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class GMA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Global Mountain Activity"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "GMA"
-    DATA_URL = "https://www.gma.rocks/download/summits.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
-            ref_id = row["Reference"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
-                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
-                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
-                                          "Latitude"] != "" else None,
-                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
-                                          "Longitude"] != "" else None,
-                                      grid=row["Maidenhead Locator"]))
-
-        return new_data

sigrefdataproviders/illw.py

@@ -1,30 +0,0 @@
-import csv
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class ILLW(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for International Lighthouse & Lightship Weekend"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "ILLW"
-    DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
-            if "ILLW" in row and row["ILLW"] != "":
-                ref_id = row["ILLW"]
-                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
-                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
-                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
-                                              "Latitude"] != "" else None,
-                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
-                                              "Longitude"] != "" else None,
-                                          grid=row["Maidenhead Locator"]))
-
-        return new_data

sigrefdataproviders/iota.py

@@ -1,36 +0,0 @@
-import logging
-
-from pyhamtools.locator import latlong_to_locator
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class IOTA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Islands on the Air"""
-
-    POLL_INTERVAL_DAYS = 365
-    SIG = "IOTA"
-    DATA_URL = "https://www.iota-world.org/islands-on-the-air/downloads/download-file.html?path=groups.json"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        data = http_response.json()
-        if isinstance(data, list):
-            for ref in data:
-                ref_id = ref["refno"]
-                latitude = float(ref["latitude_min"]) + float(ref["latitude_max"]) / 2.0
-                longitude = float(ref["longitude_min"]) + float(ref["longitude_max"]) / 2.0
-                grid = None
-                try:
-                    grid = latlong_to_locator(latitude, longitude, 6)
-                except ValueError:
-                    logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
-
-                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
-                                          grid=grid, latitude=latitude, longitude=longitude))
-
-        return new_data

sigrefdataproviders/llota.py

@@ -1,32 +0,0 @@
-from pyhamtools.locator import locator_to_latlong
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class LLOTA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Lagos y Lagunas on the Air"""
-
-    POLL_INTERVAL_DAYS = 7
-    SIG = "LLOTA"
-    DATA_URL = "https://llota.app/api/public/references"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        data = http_response.json()
-        if isinstance(data, list):
-            for ref in data:
-                ref_id = ref["reference_code"]
-                grid = str(ref["grid_locator"])
-                ll = locator_to_latlong(grid)
-
-                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
-                                          url="https://llota.app/list/ref/" + ref_id,
-                                          grid=grid,
-                                          latitude=ll[0],
-                                          longitude=ll[1]))
-
-        return new_data

sigrefdataproviders/local_file_sig_ref_data_provider.py

@@ -1,36 +0,0 @@
-import logging
-from datetime import datetime
-
-import pytz
-
-from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
-
-
-class LocalFileSIGRefDataProvider(SIGRefDataProvider):
-    """Generic SIG ref data provider class for providers that fetch their data from a local file on startup."""
-
-    def __init__(self, sig, provider_config, path):
-        super().__init__(sig, provider_config)
-        self._path = path
-
-    def start(self):
-        logging.debug("Loading " + self.sig_name + " SIG ref data from file.")
-        try:
-            new_data = self._file_to_data(self._path)
-            if new_data:
-                self._add_data(new_data)
-                self.status = "OK"
-                self.last_update_time = datetime.now(pytz.UTC)
-            else:
-                logging.info("No new SIG ref data found for " + self.sig_name)
-        except Exception as e:
-            self.status = "Error"
-            logging.exception("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")")
-
-    def stop(self):
-        pass
-
-    def _file_to_data(self, path):
-        """Load a file on the given path and turn it into SIG Ref data."""
-
-        raise NotImplementedError("Subclasses must implement this method")

sigrefdataproviders/mota.py

@@ -1,29 +0,0 @@
-import csv
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class MOTA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Mills on the Air"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "MOTA"
-    DATA_URL = "https://www.gma.rocks/download/mills.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
-            ref_id = row["Reference"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
-                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
-                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
-                                          "Latitude"] != "" else None,
-                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
-                                          "Longitude"] != "" else None,
-                                      grid=row["Maidenhead Locator"]))
-
-        return new_data

sigrefdataproviders/pota.py

@@ -1,29 +0,0 @@
-import csv
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class POTA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Parks on the Air"""
-
-    POLL_INTERVAL_DAYS = 7
-    SIG = "POTA"
-    DATA_URL = "https://pota.app/all_parks_ext.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
-            ref_id = row["reference"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
-                                      url="https://pota.app/#/park/" + ref_id,
-                                      grid=row["grid"] if "grid" in row else None,
-                                      latitude=float(row["latitude"]) if "latitude" in row and row[
-                                          "latitude"] != "" else None,
-                                      longitude=float(row["longitude"]) if "longitude" in row and row[
-                                          "longitude"] != "" else None))
-
-        return new_data

sigrefdataproviders/sig_ref_data_provider.py

@@ -1,40 +0,0 @@
-import logging
-from datetime import datetime
-
-import pytz
-
-from core.data_store import DATA_STORE
-
-
-class SIGRefDataProvider:
-    """Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data."""
-
-    def __init__(self, sig_name, provider_config):
-        """Constructor"""
-
-        self.sig_name = sig_name
-        self.enabled = provider_config["enabled"]
-        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
-        self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
-        self.status = "Not Started" if self.enabled else "Disabled"
-        self.reference_count = 0
-
-
-    def start(self):
-        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
-
-        raise NotImplementedError("Subclasses must implement this method")
-
-
-    def stop(self):
-        """Stop any threads and prepare for application shutdown"""
-
-        raise NotImplementedError("Subclasses must implement this method")
-
-    def _add_data(self, new_data):
-        """Add all the provided reference data objects to the data store."""
-
-        for d in new_data:
-            DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d
-        self.reference_count = len(new_data)
-        logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)

sigrefdataproviders/siota.py

@@ -1,26 +0,0 @@
-import csv
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class SIOTA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Silos on the Air"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "SIOTA"
-    DATA_URL = "https://www.silosontheair.com/data/silos.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
-            ref_id = row["SILO_CODE"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
-                                      grid=row["LOCATOR"] if "LOCATOR" in row else None,
-                                      latitude=float(row["LAT"]) if "LAT" in row else None,
-                                      longitude=float(row["LNG"]) if "LNG" in row else None))
-
-        return new_data

sigrefdataproviders/sota.py

@@ -1,34 +0,0 @@
-import csv
-
-from pyhamtools.locator import latlong_to_locator
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class SOTA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Summits on the Air"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "SOTA"
-    DATA_URL = "https://storage.sota.org.uk/summitslist.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
-            ref_id = row["SummitCode"]
-            latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
-            longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
-            ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
-                                      url="https://www.sotadata.org.uk/en/summit/" + ref_id,
-                                      latitude=latitude,
-                                      longitude=longitude,
-                                      activation_score=int(row["Points"]) if "Points" in row else None)
-            if latitude and longitude:
-                ref.grid = latlong_to_locator(latitude, longitude, 6)
-            new_data.append(ref)
-
-        return new_data

sigrefdataproviders/toilets.py

@@ -1,24 +0,0 @@
-import csv
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
-
-
-class Toilets(LocalFileSIGRefDataProvider):
-    """SIG ref data provider for Toilets on the Air"""
-
-    SIG = "Toilets"
-    PATH = "datafiles/toilets.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.PATH)
-
-    def _file_to_data(self, path):
-        new_data = []
-        f = open(path)
-        csv_data = f.read()
-        dr = csv.DictReader(csv_data.splitlines())
-        for row in dr:
-            new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
-                                          longitude=float(row["lon"])))
-        return new_data

sigrefdataproviders/towers.py

@@ -1,27 +0,0 @@
-import csv
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class Towers(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Towers on the Air"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "Towers"
-    DATA_URL = "https://wwtota.com/servis/generate_csv.php?ref=&filter=all"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
-            ref_id = row["Ref"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
-                                      url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id,
-                                      grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
-                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
-                                      longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None))
-
-        return new_data

sigrefdataproviders/wca.py

@@ -1,44 +0,0 @@
-import csv
-import logging
-
-from pyhamtools.locator import latlong_to_locator
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class WCA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for World Castles Award"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "WCA"
-    DATA_URL = "https://polo.ham2k.com/data/activities/wca/all-castles.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
-            ref_id = row["REF"]
-
-            coordsStr = row["COORDINATES"]
-            latitude = None
-            longitude = None
-            grid = None
-            try:
-                if coordsStr:
-                    split = coordsStr.split(", ")
-                    latitude = float(split[0])
-                    longitude = float(split[1])
-                    grid = latlong_to_locator(latitude, longitude)
-            except ValueError:
-                logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
-
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
-                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
-                                      latitude=latitude,
-                                      longitude=longitude,
-                                      grid=grid))
-
-        return new_data

sigrefdataproviders/wota.py

@@ -1,31 +0,0 @@
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class WOTA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Wainwrights on the Air"""
-
-    POLL_INTERVAL_DAYS = 365
-    SIG = "WOTA"
-    DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for feature in http_response.json().get("features", []):
-            ref_id = feature["properties"]["wotaId"]
-            # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
-            # added to them
-            url = "https://www.wota.org.uk/MM_" + ref_id
-            if ref_id.upper().startswith("LDO-"):
-                number = int(ref_id.upper().replace("LDO-", ""))
-                url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
-
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
-                                      grid=feature["properties"]["qthLocator"],
-                                      latitude=feature["geometry"]["coordinates"][1],
-                                      longitude=feature["geometry"]["coordinates"][0]))
-
-        return new_data

sigrefdataproviders/wwbota.py

@@ -1,27 +0,0 @@
-import csv
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class WWBOTA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Worldwide Bunkers on the Air"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "WWBOTA"
-    DATA_URL = "https://api.wwbota.org/bunkers/?format=CSV"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
-            ref_id = row["Reference"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
-                                      url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None,
-                                      grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
-                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
-                                      longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None))
-
-        return new_data

sigrefdataproviders/wwff.py

@@ -1,30 +0,0 @@
-import csv
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class WWFF(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for Worldwide Flora & Fauna"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "WWFF"
-    DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
-            ref_id = row["reference"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
-                                      url="https://wwff.co/directory/?showRef=" + ref_id,
-                                      grid=row["iaruLocator"] if "iaruLocator" in row and row[
-                                          "iaruLocator"] != "-" else None,
-                                      latitude=float(row["latitude"]) if "latitude" in row and row[
-                                          "latitude"] != "" and row["latitude"] != "-" else None,
-                                      longitude=float(row["longitude"]) if "longitude" in row and row[
-                                          "longitude"] != "" and row["longitude"] != "-" else None))
-
-        return new_data

sigrefdataproviders/zlota.py

@@ -1,34 +0,0 @@
-from pyhamtools.locator import latlong_to_locator
-
-from data.sig_ref import SIGRef
-from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
-
-
-class ZLOTA(FileDownloadSIGRefDataProvider):
-    """SIG ref data provider for New Zealand on the Air"""
-
-    POLL_INTERVAL_DAYS = 30
-    SIG = "ZLOTA"
-    DATA_URL = "https://ontheair.nz/assets/assets.json"
-
-    def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
-
-    def _http_response_to_data(self, http_response):
-        new_data = []
-        data = http_response.json()
-        if isinstance(data, list):
-            for ref in data:
-                ref_id = ref["code"]
-                latitude = ref["y"]
-                longitude = ref["x"]
-
-                ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
-                                          url="https://ontheair.nz/assets/" + ref_id.replace("/", "_"),
-                                          latitude=latitude,
-                                          longitude=longitude)
-                if latitude and longitude:
-                    ref.grid = latlong_to_locator(latitude, longitude, 6)
-                new_data.append(ref)
-
-        return new_data

solarconditionsproviders/giroionosonde.py

@@ -35,18 +35,6 @@ class GIROIonosonde(SolarConditionsProvider):
         self._thread = None
         self._stop_event = Event()
 
-        # Pre-populate ionosonde_data with known station names for stations not already present,
-        # so the station dropdown is available before the first poll. Does not overwrite existing
-        # entries so KC2G cache data is preserved.
-        existing = self._solar_conditions.ionosonde_data or {}
-        new_entries = {
-            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
-                        "luf": None, "band_states": None}
-            for s in self._stations if s["ursi"] not in existing
-        }
-        if new_entries:
-            self.update_data({"ionosonde_data": {**existing, **new_entries}})
-
     @staticmethod
     def _load_stations():
         stations = []
@@ -56,6 +44,21 @@ class GIROIonosonde(SolarConditionsProvider):
                     stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
         return stations
 
+    def setup(self, solar_conditions, solar_conditions_cache):
+        """Pre-populate ionosonde_data with known station names for stations not already present,
+        so the station dropdown is available before the first poll. Does not overwrite existing
+        entries so KC2G cache data is preserved."""
+
+        super().setup(solar_conditions, solar_conditions_cache)
+        existing = solar_conditions.ionosonde_data or {}
+        new_entries = {
+            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
+                        "luf": None, "band_states": None}
+            for s in self._stations if s["ursi"] not in existing
+        }
+        if new_entries:
+            self.update_data({"ionosonde_data": {**existing, **new_entries}})
+
     def start(self):
         logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
         self._thread = Thread(target=self._run, daemon=True)

solarconditionsproviders/solar_conditions_provider.py

@@ -2,8 +2,6 @@ from datetime import datetime
 
 import pytz
 
-from core.data_store import DATA_STORE
-
 
 class SolarConditionsProvider:
     """Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
@@ -12,11 +10,18 @@ class SolarConditionsProvider:
     def __init__(self, name, provider_config):
         """Constructor"""
 
+        self._solar_conditions_cache = None
         self.name = name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._solar_conditions = DATA_STORE.solar_conditions
+        self._solar_conditions = None
+
+    def setup(self, solar_conditions, solar_conditions_cache):
+        """Set up the provider, giving it the solar conditions object and its backing cache"""
+
+        self._solar_conditions = solar_conditions
+        self._solar_conditions_cache = solar_conditions_cache
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -36,3 +41,4 @@ class SolarConditionsProvider:
                 if hasattr(self._solar_conditions, key):
                     setattr(self._solar_conditions, key, value)
             self._solar_conditions.infer_descriptions()
+            self._solar_conditions_cache['solar_conditions'] = self._solar_conditions

spothole.py

@@ -1,23 +1,30 @@
 # Main script
+import importlib
 import logging
 import os
 import signal
 import sys
 
-from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_data_provider_from_config, \
+from diskcache import Cache
-    get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config
+
+from core.cleanup import CleanupTimer
+from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL
 from core.constants import SOFTWARE_VERSION
-from core.data_store import DATA_STORE
 from core.lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
+from data.solar_conditions import SolarConditions
 from server.webserver import WebServer
 
 # Globals
+spots = Cache('cache/spots_cache')
+alerts = Cache('cache/alerts_cache')
+solar_conditions_cache = Cache('cache/solar_conditions_cache')
+solar_conditions = solar_conditions_cache.get('solar_conditions', SolarConditions())
 web_server = None
+status_data = {}
 spot_providers = []
 alert_providers = []
 solar_condition_providers = []
-sig_ref_data_providers = []
 cleanup_timer = None
 run = True
 
@@ -39,13 +46,40 @@ def shutdown(_signum=None, _frame=None):
     for scp in solar_condition_providers:
         if scp.enabled:
             scp.stop()
-    for srdp in sig_ref_data_providers:
+    if cleanup_timer:
-        if srdp.enabled:
+        cleanup_timer.stop()
-            srdp.stop()
+    if lookup_helper:
-    DATA_STORE.close()
+        lookup_helper.stop()
+    spots.close()
+    alerts.close()
+    solar_conditions_cache.close()
     os._exit(0)
 
 
+def get_spot_provider_from_config(config_providers_entry):
+    """Utility method to get a spot provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('spotproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
+def get_alert_provider_from_config(config_providers_entry):
+    """Utility method to get an alert provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('alertproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
+def get_solar_conditions_provider_from_config(config_providers_entry):
+    """Utility method to get a solar conditions provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('solarconditionsproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
 # Main function
 if __name__ == '__main__':
     # Set up logging
@@ -65,19 +99,17 @@ if __name__ == '__main__':
     # Shut down gracefully on SIGINT
     signal.signal(signal.SIGINT, shutdown)
 
-    # Set up data store
-    DATA_STORE.setup()
-
     # Set up lookup helper
     lookup_helper.start()
 
     # Set up web server
-    web_server = WebServer()
+    web_server = WebServer(spots=spots, alerts=alerts, solar_conditions=solar_conditions, status_data=status_data)
 
     # Fetch, set up and start spot providers
     for entry in config["spot-providers"]:
         spot_providers.append(get_spot_provider_from_config(entry))
     for p in spot_providers:
+        p.setup(spots=spots, web_server=web_server)
         if p.enabled:
             p.start()
 
@@ -85,6 +117,7 @@ if __name__ == '__main__':
     for entry in config["alert-providers"]:
         alert_providers.append(get_alert_provider_from_config(entry))
     for p in alert_providers:
+        p.setup(alerts=alerts, web_server=web_server)
         if p.enabled:
             p.start()
 
@@ -92,20 +125,18 @@ if __name__ == '__main__':
     for entry in config.get("solar-condition-providers", []):
         solar_condition_providers.append(get_solar_conditions_provider_from_config(entry))
     for p in solar_condition_providers:
+        p.setup(solar_conditions=solar_conditions, solar_conditions_cache=solar_conditions_cache)
         if p.enabled:
             p.start()
 
-    # Fetch, set up and start SIG reference data providers
+    # Set up timer to clear spot list of old data
-    for entry in config.get("sig-ref-data-providers", []):
+    cleanup_timer = CleanupTimer(spots=spots, alerts=alerts, web_server=web_server, cleanup_interval=60)
-        sig_ref_data_providers.append(get_sig_ref_data_provider_from_config(entry))
+    cleanup_timer.start()
-    for p in sig_ref_data_providers:
-        if p.enabled:
-            p.start()
 
     # Set up status reporter
-    status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers,
+    status_reporter = StatusReporter(status_data=status_data, spots=spots, alerts=alerts, web_server=web_server,
+                                     cleanup_timer=cleanup_timer, spot_providers=spot_providers,
                                      alert_providers=alert_providers,
-                                     sig_ref_data_providers=sig_ref_data_providers,
                                      solar_condition_providers=solar_condition_providers, run_interval=5)
     status_reporter.start()
 

spotproviders/gma.py

@@ -3,8 +3,8 @@ from datetime import datetime
 
 import pytz
 
+from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import HTTP_HEADERS
-from core.url_data_cache import URLDataCache
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -21,13 +21,12 @@ class GMA(HTTPSpotProvider):
     def __init__(self, provider_config):
         # Ensure there is an API key in our config, and set up the query URL using it. If no key is provided,
         # disable this spot provider.
-        self._api_key = provider_config.get("api-key", "")
+        self.api_key = provider_config.get("api-key", "")
-        if self._api_key == "":
+        if self.api_key == "":
             provider_config["enabled"] = False
             logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
-        self._url_data_cache = URLDataCache("GMA")
 
-        super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self._api_key, self.POLL_INTERVAL_SEC)
+        super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self.api_key, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -57,7 +56,7 @@ class GMA(HTTPSpotProvider):
                 # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
                 if "REF" in source_spot:
                     try:
-                        ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
+                        ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
                                                                       headers=HTTP_HEADERS)
                         # Sometimes this is blank even if it's a 200 response, so handle that
                         if ref_response.ok and ref_response.text is not None and ref_response.text != "":

spotproviders/http_spot_provider.py

@@ -64,7 +64,7 @@ class HTTPSpotProvider(SpotProvider):
             logging.warning(f"Timeout when accessing {self.name} spots API.")
         except Exception:
             self.status = "Error"
-            logging.exception("Exception in HTTP Spot Provider (" + self.name + ")")
+            logging.exception("Exception in HTTP JSON Spot Provider (" + self.name + ")")
             self._stop_event.wait(timeout=1)
 
     def _http_response_to_spots(self, http_response):

spotproviders/sota.py

@@ -19,6 +19,8 @@ class SOTA(HTTPSpotProvider):
     # The actual data lookup all happens after parsing and checking the epoch.
     EPOCH_URL = "https://api-db2.sota.org.uk/api/spots/epoch"
     SPOTS_URL = "https://api-db2.sota.org.uk/api/spots/60/all/all"
+    # SOTA spots don't contain lat/lon, we need a separate lookup for that
+    SUMMIT_URL_ROOT = "https://api-db2.sota.org.uk/api/summits/"
 
     def __init__(self, provider_config):
         super().__init__("SOTA", provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)

spotproviders/spot_provider.py

@@ -2,7 +2,7 @@ from datetime import datetime
 
 import pytz
 
-from core.data_store import DATA_STORE
+from core.config import MAX_SPOT_AGE
 
 
 class SpotProvider:
@@ -16,7 +16,14 @@ class SpotProvider:
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._spots = DATA_STORE.spots
+        self._spots = None
+        self._web_server = None
+
+    def setup(self, spots, web_server):
+        """Set up the provider, e.g. giving it the spot list to work from"""
+
+        self._spots = spots
+        self._web_server = web_server
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -52,7 +59,10 @@ class SpotProvider:
 
     def _add_spot(self, spot):
         if not spot.expired():
-            self._spots.set(spot.id, spot)
+            self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
+            # Ping the web server in case we have any SSE connections that need to see this immediately
+            if self._web_server:
+                self._web_server.notify_new_spot(spot)
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""

spotproviders/towers.py

@@ -30,8 +30,8 @@ class Towers(HTTPSpotProvider):
                         dx_call=source_spot["call"].upper(),
                         freq=likely_freq,
                         comment=source_spot["comment"],
-                        sig="Towers",
+                        sig="WWTOTA",
-                        sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
+                        sig_refs=[SIGRef(id=source_spot["ref"], sig="WWTOTA")],
                         time=datetime.strptime(response_json["updated"][:10] + source_spot["time"],
                                                "%Y-%m-%d%H:%M").timestamp())
 

spotproviders/wota.py

@@ -25,7 +25,7 @@ class WOTA(HTTPSpotProvider):
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
-        rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
+        rss = cast(RSS, Parser.parse(http_response.content.decode()))
         # Iterate through source data
         for source_spot in rss.channel.items:
 

spotproviders/xota.py

@@ -1,4 +1,6 @@
+import csv
 import json
+import logging
 from datetime import datetime
 
 import pytz
@@ -10,11 +12,10 @@ from spotproviders.websocket_spot_provider import WebsocketSpotProvider
 
 class XOTA(WebsocketSpotProvider):
     """Spot provider for servers based on the "xOTA" software at https://github.com/nischu/xOTA/
-    The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG which we can
+    The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG and a reference
-    then use for lookups. This functionality is implemented for Toilets on the Air events, of which there are
+    to a local CSV file with location information. This functionality is implemented for TOTA events, of which there are
-    several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA, which is why we also
+    several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA and hence which server
-    provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3 might become
+    supplied the data, which is why the CSV location lookup is here and not in sig_utils."""
-    "C3 T-01". This allows us to provide location lookups for TOTA at several conferences."""
 
     LOCATION_DATA = {}
     SIG = None
@@ -22,13 +23,26 @@ class XOTA(WebsocketSpotProvider):
     def __init__(self, provider_config):
         name = provider_config["name"] if "name" in provider_config else "xOTA"
         super().__init__(name, provider_config, provider_config["url"])
+        locations_csv = str(provider_config["locations-csv"]) if "locations-csv" in provider_config else None
         self.SIG = str(provider_config["sig"]) if "sig" in provider_config else None
-        self._sig_ref_prefix = str(provider_config["sig-ref-prefix"]) if "sig-ref-prefix" in provider_config else ""
+
+        # Load location data
+        if locations_csv:
+            try:
+                f = open(locations_csv)
+                csv_data = f.read()
+                dr = csv.DictReader(csv_data.splitlines())
+                for row in dr:
+                    self.LOCATION_DATA[row["ref"]] = {"lat": row["lat"], "lon": row["lon"]}
+            except:
+                logging.exception("Could not look up location data for XOTA source.")
 
     def _ws_message_to_spot(self, b):
         string = b.decode("utf-8")
         source_spot = json.loads(string)
-        ref_id = self._sig_ref_prefix + " " + source_spot["reference"]["title"]
+        ref_id = source_spot["reference"]["title"]
+        lat = float(self.LOCATION_DATA[ref_id]["lat"]) if ref_id in self.LOCATION_DATA else None
+        lon = float(self.LOCATION_DATA[ref_id]["lon"]) if ref_id in self.LOCATION_DATA else None
         spot = Spot(source=self.name,
                     source_id=source_spot["id"],
                     dx_call=source_spot["stationCallSign"].upper(),
@@ -36,7 +50,10 @@ class XOTA(WebsocketSpotProvider):
                     mode=source_spot["mode"].upper(),
                     sig=self.SIG,
                     sig_refs=[
-                        SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"])],
+                        SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"], latitude=lat,
+                               longitude=lon)],
                     time=datetime.now(pytz.UTC).timestamp(),
+                    dx_latitude=lat,
+                    dx_longitude=lon,
                     qrt=source_spot["state"] != "active")
         return spot

static/apidocs/openapi.yml

@@ -23,7 +23,6 @@ info:
     * Added `comment_names` to SIGs in the `/options`, to reflect how they might be referred to in spot comments where
       it differs from their `name`.
     * Added `propagation_mode` field to spots
-    * Added `sig_ref_data_providers` to status and removed `cleanup`
 
     ### 1.3
 
@@ -796,7 +795,7 @@ components:
         - HEMA
         - WCA
         - MOTA
-        - SIOTA
+        - SiOTA
         - ARLHS
         - ILLW
         - ZLOTA
@@ -1721,32 +1720,6 @@ components:
             is zero, the provider has never updated.
           example: 1759579508
 
-    SIGRefDataProviderStatus:
-      type: object
-      properties:
-        sig_name:
-          type: string
-          description: The name of the SIG.
-          example: WWFF
-        enabled:
-          type: boolean
-          description: Whether the provider is enabled or not.
-          example: true
-        status:
-          type: string
-          description: The status of the provider.
-          example: OK
-        last_updated:
-          type: number
-          description: >
-            The last time at which this provider received data, UTC seconds since UNIX epoch. If this
-            is zero, the provider has never updated.
-          example: 1759579508
-        reference_count:
-          type: number
-          description: The number of references fetched using this provider.
-          example: 1234
-
     SpotList:
       type: array
       items:
@@ -1814,6 +1787,17 @@ components:
           type: integer
           description: Number of alerts currently in the system.
           example: 123
+        "cleanup":
+          type: object
+          properties:
+            status:
+              type: string
+              description: The status of the cleanup thread
+              example: OK
+            last_ran:
+              type: number
+              description: The last time the cleanup operation ran, UTC seconds since UNIX epoch.
+              example: 1759579508
         "webserver":
           type: object
           properties:
@@ -1844,11 +1828,6 @@ components:
           description: An array of all the solar conditions providers.
           items:
             $ref: '#/components/schemas/SolarConditionsProviderStatus'
-        sig_ref_data_providers:
-          type: array
-          description: An array of all the SIG reference data providers.
-          items:
-            $ref: '#/components/schemas/SIGRefDataProviderStatus'
 
     Options:
       type: object

static/js/status.js

@@ -12,6 +12,9 @@ function loadStatus() {
         $("#web-server-last-api").text(moment.unix(jsonData["webserver"]["last_api_access"]).utc().fromNow());
         $("#web-server-last-page").text(moment.unix(jsonData["webserver"]["last_page_access"]).utc().fromNow());
 
+        $("#cleanup-status").text(jsonData["cleanup"]["status"]);
+        $("#cleanup-last-ran").text(moment.unix(jsonData["cleanup"]["last_ran"]).utc().fromNow());
+
         jsonData["spot_providers"].forEach(p => {
             $("#spot-providers-status-container").append(`
                 <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
@@ -39,16 +42,6 @@ function loadStatus() {
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
                 </div>`);
         });
-
-        jsonData["sig_ref_data_providers"].forEach(p => {
-            $("#sig-ref-data-providers-status-container").append(`
-                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
-                    <div class="col"><strong>${p["sig_name"]}</strong></div>
-                    <div class="col">Status: ${p["status"]}</div>
-                    <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
-                    <div class="col">References: ${(p["enabled"] && p["reference_count"] > 0) ? p["reference_count"] : "N/A"}</div>
-                </div>`);
-        });
     });
 }
 

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1785434214"></script>
+<script src="/static/js/add-spot.js?v=1785667229"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -75,7 +75,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1785434213"></script>
+<script src="/static/js/alerts.js?v=1785667229"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -75,8 +75,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/static/js/spotsbandsandmap.js?v=1785434214"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1785667229"></script>
-<script src="/static/js/bands.js?v=1785434214"></script>
+<script src="/static/js/bands.js?v=1785667229"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1785434213" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1785667228" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
 <script src="/static/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
 <script src="/static/vendor/js/tinycolor2-1.6.0.min.js"></script>
 
-<script src="/static/js/utils.js?v=1785434213"></script>
+<script src="/static/js/utils.js?v=1785667228"></script>
-<script src="/static/js/ui-ham.js?v=1785434213"></script>
+<script src="/static/js/ui-ham.js?v=1785667228"></script>
-<script src="/static/js/geo.js?v=1785434213"></script>
+<script src="/static/js/geo.js?v=1785667228"></script>
-<script src="/static/js/common.js?v=1785434213"></script>
+<script src="/static/js/common.js?v=1785667228"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1785434213"></script>
+<script src="/static/js/conditions.js?v=1785667229"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -108,8 +108,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/static/js/spotsbandsandmap.js?v=1785434213"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1785667228"></script>
-<script src="/static/js/map.js?v=1785434213"></script>
+<script src="/static/js/map.js?v=1785667228"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -116,8 +116,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/static/js/spotsbandsandmap.js?v=1785434213"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1785667228"></script>
-<script src="/static/js/spots.js?v=1785434213"></script>
+<script src="/static/js/spots.js?v=1785667228"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -24,6 +24,11 @@
             <div class="col">Last API call: <span id="web-server-last-api"></span></div>
             <div class="col">Last page req: <span id="web-server-last-page"></span></div>
         </div>
+        <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+            <div class="col"><strong>Cleanup Service</strong></div>
+            <div class="col">Status: <span id="cleanup-status"></span></div>
+            <div class="col">Last ran: <span id="cleanup-last-ran"></span></div>
+        </div>
     </div>
 </div>
 
@@ -54,16 +59,7 @@
     </div>
 </div>
 
-<div class="card mt-3">
+<script src="/static/js/status.js?v=1785667229"></script>
-    <div class="card-header">
-        SIG Reference Data Providers
-    </div>
-    <div class="card-body" id="sig-ref-data-providers-status-container">
-
-    </div>
-</div>
-
-<script src="/static/js/status.js?v=1785434213"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");