Release 1.4

alertproviders/alert_provider.py

@@ -2,7 +2,7 @@ from datetime import datetime
 
 import pytz
 
-from core.data_store import DATA_STORE
+from core.config import MAX_ALERT_AGE
 
 
 class AlertProvider:
@@ -15,7 +15,14 @@ class AlertProvider:
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._alerts = DATA_STORE.alerts
+        self._alerts = None
+        self._web_server = None
+
+    def setup(self, alerts, web_server):
+        """Set up the provider, e.g. giving it the alert list to work from"""
+
+        self._alerts = alerts
+        self._web_server = web_server
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -37,7 +44,10 @@ class AlertProvider:
 
     def _add_alert(self, alert):
         if not alert.expired():
-            self._alerts.set(alert.id, alert)
+            self._alerts.add(alert.id, alert, expire=MAX_ALERT_AGE)
+            # Ping the web server in case we have any SSE connections that need to see this immediately
+            if self._web_server:
+                self._web_server.notify_new_alert(alert)
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""

core/cache_utils.py

@@ -0,0 +1,27 @@
+import threading
+from datetime import timedelta
+
+from requests_cache import CachedSession
+
+# Cache for "semi-static" data such as the locations of parks, CSVs of reference lists, etc.
+# This has an expiry time of 30 days, so will re-request from the source after that amount
+# of time has passed. This is used throughout Spothole to cache data that does not change
+# rapidly. The ThreadSafeSession construct here protects it against some multithreading
+# contention weirdness we sometimes used to see on startup where the cache was hammered
+# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type
+# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote
+# servers for data they've told us they can't provide.
+_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30),
+                         allowable_codes=(200, 400, 401, 403, 404))
+_lock = threading.Lock()
+
+
+class _ThreadSafeSession:
+    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads."""
+
+    def get(self, *args, **kwargs):
+        with _lock:
+            return _session.get(*args, **kwargs)
+
+
+SEMI_STATIC_URL_DATA_CACHE = _ThreadSafeSession()

core/cleanup.py

@@ -0,0 +1,73 @@
+import logging
+from datetime import datetime
+from threading import Event, Thread
+
+import pytz
+
+
+class CleanupTimer:
+    """Provides a timed cleanup of the spot list."""
+
+    def __init__(self, spots, alerts, web_server, cleanup_interval):
+        """Constructor"""
+
+        self._spots = spots
+        self._alerts = alerts
+        self._web_server = web_server
+        self._cleanup_interval = cleanup_interval
+        self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Starting"
+        self._thread = None
+        self._stop_event = Event()
+
+    def start(self):
+        """Start the cleanup timer"""
+
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        self._stop_event.set()
+
+    def _run(self):
+        while not self._stop_event.wait(timeout=self._cleanup_interval):
+            self._cleanup()
+
+    def _cleanup(self):
+        """Perform cleanup and reschedule next timer"""
+
+        try:
+            # Perform cleanup via letting the data expire
+            self._spots.expire()
+            self._alerts.expire()
+
+            # Explicitly clean up any spots and alerts that have expired
+            for i in list(self._spots.iterkeys()):
+                try:
+                    spot = self._spots[i]
+                    if spot.expired():
+                        self._spots.delete(i)
+                except KeyError:
+                    # Must have already been deleted, OK with that
+                    pass
+            for i in list(self._alerts.iterkeys()):
+                try:
+                    alert = self._alerts[i]
+                    if alert.expired():
+                        self._alerts.delete(i)
+                except KeyError:
+                    # Must have already been deleted, OK with that
+                    pass
+
+            # Clean up web server SSE spot/alert queues
+            self._web_server.clean_up_sse_queues()
+
+            self.status = "OK"
+            self.last_cleanup_time = datetime.now(pytz.UTC)
+
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in Cleanup thread")
+            self._stop_event.wait(timeout=1)

core/constants.py

@@ -3,7 +3,7 @@ from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "1.4-pre"
+SOFTWARE_VERSION = "1.4"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": "Spothole v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}

core/data_store.py

@@ -1,69 +0,0 @@
-from pathlib import Path
-
-import diskcache
-
-from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
-from core.live_data_cache import LiveDataCache
-from data.solar_conditions import SolarConditions
-
-
-class DataStore:
-    """Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
-    lookup data using different caching strategies for each."""
-
-    def __init__(self):
-        cache_dir = "./cache"
-        self.MAX_SPOT_COUNT = 10000
-        self.MAX_ALERT_COUNT = 10000
-        self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
-        self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
-
-        Path(cache_dir).mkdir(parents=True, exist_ok=True)
-
-        # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval
-        self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
-                                   snapshot_dir=cache_dir + "/spots",
-                                   snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
-        self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
-                                   snapshot_dir=cache_dir + "/alerts",
-                                    snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
-
-        # Standard disk cache for solar data and status data, but each cache contains only a single object which we
-        # expose to the wider application
-        self.solar = diskcache.Cache(cache_dir + "/solar")
-        if "solar_conditions" not in self.solar:
-            self.solar.add("solar_conditions", SolarConditions())
-        self.solar_conditions = self.solar.get("solar_conditions")
-        self.status = diskcache.Cache(cache_dir + "/status")
-        if "status_data" not in self.status:
-            self.status.add("status_data", {})
-        self.status_data = self.status.get("status_data")
-
-        # Standard disk caches for SIG ref data. Separate provider threads will repopulate these on a regular basis
-        # but there's no need for a TTL since old data is better than no data.
-        self.sigrefs_wwff = diskcache.Cache(cache_dir + "/sigrefs/wwff")
-        self.sigrefs_siota = diskcache.Cache(cache_dir + "/sigrefs/siota")
-        self.sigrefs_wota = diskcache.Cache(cache_dir + "/sigrefs/wota")
-        self.sigrefs_zlota = diskcache.Cache(cache_dir + "/sigrefs/zlota")
-        self.sigrefs_llota = diskcache.Cache(cache_dir + "/sigrefs/llota")
-        self.sigrefs_dme = diskcache.Cache(cache_dir + "/sigrefs/dme")
-
-        # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
-        # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
-        # we've seen, so we rely on them timing out and this triggering another lookup.
-        self.callsigns = diskcache.Cache(cache_dir + "/callsigns")
-
-    def close(self):
-        self.spots.close()
-        self.alerts.close()
-        self.solar.close()
-        self.status.close()
-        self.sigrefs_wwff.close()
-        self.sigrefs_siota.close()
-        self.sigrefs_wota.close()
-        self.sigrefs_zlota.close()
-        self.sigrefs_llota.close()
-        self.callsigns.close()
-
-# Global object
-DATA_STORE = DataStore()

core/live_data_cache.py

@@ -1,100 +0,0 @@
-import logging
-import threading
-import time
-
-import diskcache
-from cachetools import TTLCache
-
-
-class LiveDataCache:
-    """Cache for spots and alerts. Uses the fast in-memory TTLCache for normal data I/O, including the TTL to enforce
-    maximum lifetime, and adds a separate diskcache to which we can save and load the TTLCache to provide persistence.
-    Also adds thread safety so spots and alerts can come from any thread, and a listener mechanism so the web server
-    can get a callback when new spots/alerts are added, and send them to any SSE clients."""
-
-    def __init__(self, maxsize, ttl, snapshot_dir, snapshot_interval_sec):
-        self._cache = TTLCache(maxsize=maxsize, ttl=ttl)
-        self._lock = threading.Lock()
-        self._ttl = ttl
-        self._listeners = []
-        self._listeners_lock = threading.Lock()
-        self._snapshot_dir = snapshot_dir
-        self._disk_cache = diskcache.Cache(str(snapshot_dir))
-        self._load_snapshot()
-        self._start_periodic_snapshot(snapshot_interval_sec)
-
-    def set(self, key, value):
-        with self._lock:
-            self._cache[key] = value
-
-        # Notify listeners
-        with self._listeners_lock:
-            listeners = list(self._listeners)
-        for callback in listeners:
-            try:
-                callback(value)
-            except Exception:
-                logging.error("Listener raised an exception for key %s", key, exc_info=True)
-
-
-    def get(self, key, default=None):
-        with self._lock:
-            return self._cache.get(key, default)
-
-    def delete(self, key):
-        with self._lock:
-            self._cache.pop(key, None)
-
-    def keys(self):
-        with self._lock:
-            return list(self._cache.keys())
-
-    def values(self):
-        with self._lock:
-            return list(self._cache.values())
-
-    def add_listener(self, callback):
-        """Register callback(value) which will be called whenever a new spot/alert item is added via set(). Used by the
-        web server (via SSEBroadcaster) to send SSE clients an update on every new spot."""
-
-        with self._listeners_lock:
-            self._listeners.append(callback)
-
-    def remove_listener(self, callback):
-        with self._listeners_lock:
-            self._listeners.remove(callback)
-
-    def save_snapshot(self):
-        with self._lock:
-            # Store the time with the data so we can avoid loading anything nxt time that's older than TTL
-            data = [(k, v, time.time()) for k, v in self._cache.items()]
-        try:
-            self._disk_cache.set("snapshot", data)
-        except Exception as e:
-            logging.error("Failed to write snapshot to %s", self._snapshot_dir, e, exc_info=True)
-
-    def _load_snapshot(self):
-        data = self._disk_cache.get("snapshot")
-        if not data:
-            return
-
-        now = time.time()
-        with self._lock:
-            for key, value, saved_at in data:
-                # Only restore entries that would still be within TTL
-                if now - saved_at < self._ttl:
-                    self._cache[key] = value
-        logging.info("Loaded snapshot from %s", self._snapshot_dir)
-
-    def _start_periodic_snapshot(self, interval):
-        def loop():
-            while True:
-                time.sleep(interval)
-                self.save_snapshot()
-
-        t = threading.Thread(target=loop, daemon=True, name=f"snapshot-{self._snapshot_dir}")
-        t.start()
-
-    def close(self):
-        self.save_snapshot()
-        self._disk_cache.close()

core/lookup_helper.py

@@ -14,10 +14,10 @@ from pyhamtools.locator import latlong_to_locator
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
+from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.config import config
 from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
     HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
-from core.url_data_cache import URL_DATA_CACHE
 
 # QRZ XML field names differ from pyhamtools' normalised names; map them here.
 _QRZ_FIELD_MAP = {
@@ -142,8 +142,8 @@ class LookupHelper:
 
         try:
             logging.info("Downloading Country-files.com cty.plist...")
-            response = URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
-                                          headers=HTTP_HEADERS)
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
+                                                      headers=HTTP_HEADERS)
 
             if response.ok:
                 with open(self._country_files_cty_plist_download_location, "w") as f:
@@ -167,7 +167,7 @@ class LookupHelper:
 
         try:
             logging.info("Downloading dxcc.json...")
-            response = URL_DATA_CACHE.get(
+            response = SEMI_STATIC_URL_DATA_CACHE.get(
                 "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
                 headers=HTTP_HEADERS)
 
@@ -515,7 +515,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = URL_DATA_CACHE.get(
+                response = SEMI_STATIC_URL_DATA_CACHE.get(
                     self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
                     headers=HTTP_HEADERS, timeout=10)
                 if response.ok:
@@ -593,7 +593,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = URL_DATA_CACHE.get(
+                response = SEMI_STATIC_URL_DATA_CACHE.get(
                     self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                         lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
                 if response.ok:

core/sig_utils.py

@@ -4,16 +4,23 @@ import logging
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
+from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import SIGS, HTTP_HEADERS
-from core.data_store import DATA_STORE
 from core.geo_utils import wab_wai_square_to_lat_lon
-from core.url_data_cache import URL_DATA_CACHE
 
 # Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data
 # file in Spothole and load it on startup.
 with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
-    for row in csv.DictReader(_f, delimiter=";"):
-        DATA_STORE.sigrefs_dme.add(row["COD_INE"][:5], row)
+    _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
+# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
+# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
+# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
+_WWFF_INDEX_CACHE = {}
+_SIOTA_INDEX_CACHE = {}
+_WOTA_INDEX_CACHE = {}
+_ZLOTA_INDEX_CACHE = {}
+_LLOTA_INDEX_CACHE = {}
+
 
 def get_ref_regex_for_sig(sig):
     """Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
@@ -46,7 +53,7 @@ def populate_sig_ref_info(sig_ref):
     ref_id = sig_ref.id
     try:
         if sig.upper() == "POTA":
-            response = URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if data:
@@ -64,7 +71,7 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "SOTA":
-            response = URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
@@ -81,7 +88,7 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "WWBOTA":
-            response = URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
@@ -97,7 +104,7 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
-            response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
@@ -123,14 +130,15 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "WWFF":
-            response = URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
                                                       headers=HTTP_HEADERS)
             if response.ok:
-                if not bool(DATA_STORE.sigrefs_wwff) or not response.from_cache:
+                global _WWFF_INDEX_CACHE
+                if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
                     # New data from WWFF, update our internal map
-                    for row in csv.DictReader(response.content.decode().splitlines()):
-                        DATA_STORE.sigrefs_wwff.add(row["reference"], row)
-                row = DATA_STORE.sigrefs_wwff.get(ref_id)
+                    _WWFF_INDEX_CACHE = {row["reference"]: row for row in
+                                         csv.DictReader(response.content.decode().splitlines())}
+                row = _WWFF_INDEX_CACHE.get(ref_id)
                 if row:
                     sig_ref.name = row["name"] if "name" in row else None
                     sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
@@ -144,14 +152,15 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "SIOTA":
-            response = URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
                                                       headers=HTTP_HEADERS)
             if response.ok:
-                if not bool(DATA_STORE.sigrefs_siota) or not response.from_cache:
+                global _SIOTA_INDEX_CACHE
+                if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
                     # New data from SIOTA, update our internal map
-                    for row in csv.DictReader(response.content.decode().splitlines()):
-                        DATA_STORE.sigrefs_siota.add(row["SILO_CODE"], row)
-                row = DATA_STORE.sigrefs_siota.get(ref_id)
+                    _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
+                                          csv.DictReader(response.content.decode().splitlines())}
+                row = _SIOTA_INDEX_CACHE.get(ref_id)
                 if row:
                     sig_ref.name = row["NAME"] if "NAME" in row else None
                     sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
@@ -163,16 +172,17 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "WOTA":
-            response = URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if data:
-                    if not bool(DATA_STORE.sigrefs_wota) or not response.from_cache:
+                    global _WOTA_INDEX_CACHE
+                    if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
                         # New data from WOTA, update our internal map
-                        for feature in data.get("features", []):
-                            DATA_STORE.sigrefs_wota.add(feature["properties"]["wotaId"], feature)
-                    feature = DATA_STORE.sigrefs_wota.get(ref_id)
+                        _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
+                                             data.get("features", [])}
+                    feature = _WOTA_INDEX_CACHE.get(ref_id)
                     if feature:
                         sig_ref.name = feature["properties"]["title"]
                         # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -190,24 +200,24 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "ZLOTA":
-            response = URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if isinstance(data, list):
-                    if not bool(DATA_STORE.sigrefs_zlota) or not response.from_cache:
+                    global _ZLOTA_INDEX_CACHE
+                    if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
                         # New data from ZLOTA, update our internal map
-                        for ref in data:
-                            DATA_STORE.sigrefs_zlota.add(ref["code"], ref)
-                    ref = DATA_STORE.sigrefs_zlota.get(ref_id)
-                    if ref:
-                        sig_ref.name = ref["name"]
+                        _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
+                    asset = _ZLOTA_INDEX_CACHE.get(ref_id)
+                    if asset:
+                        sig_ref.name = asset["name"]
                         sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
                         try:
-                            sig_ref.grid = latlong_to_locator(ref["y"], ref["x"], 6)
+                            sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
                         except:
                             logging.debug("Invalid lat/lon received for reference")
-                            sig_ref.latitude = ref["y"]
-                            sig_ref.longitude = ref["x"]
+                            sig_ref.latitude = asset["y"]
+                            sig_ref.longitude = asset["x"]
                 elif not response.from_cache:
                     logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
             elif not response.from_cache:
@@ -219,16 +229,16 @@ def populate_sig_ref_info(sig_ref):
             sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
 
         elif sig.upper() == "LLOTA":
-            response = URL_DATA_CACHE.get("https://llota.app/api/public/references",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if isinstance(data, list):
-                    if not bool(DATA_STORE.sigrefs_llota) or not response.from_cache:
+                    global _LLOTA_INDEX_CACHE
+                    if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
                         # New data from LLOTA, update our internal map
-                        for ref in data:
-                            DATA_STORE.sigrefs_llota.add(ref["reference_code"], ref)
-                    ref = DATA_STORE.sigrefs_llota.get(ref_id)
+                        _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
+                    ref = _LLOTA_INDEX_CACHE.get(ref_id)
                     if ref:
                         sig_ref.name = str(ref["name"])
                         sig_ref.url = "https://llota.app/list/ref/" + ref_id
@@ -270,7 +280,7 @@ def populate_sig_ref_info(sig_ref):
 
         elif sig.upper() == "DME":
             # Zero-pad to 5 digits to match our source data
-            row = DATA_STORE.sigrefs_dme.get(ref_id.zfill(5))
+            row = _DME_INDEX.get(ref_id.zfill(5))
             if row:
                 sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
                 sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(

core/status_reporter.py

@@ -7,27 +7,31 @@ import pytz
 
 from core.config import SERVER_OWNER_CALLSIGN
 from core.constants import SOFTWARE_VERSION
-from core.data_store import DATA_STORE
 from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge
 
 
 class StatusReporter:
     """Provides a timed update of the application's status data."""
 
-    def __init__(self, run_interval, web_server,spot_providers, alert_providers, solar_condition_providers):
+    def __init__(self, status_data, run_interval, web_server, cleanup_timer, spots, spot_providers, alerts,
+                 alert_providers, solar_condition_providers):
         """Constructor"""
 
+        self._status_data = status_data
         self._run_interval = run_interval
         self._web_server = web_server
+        self._cleanup_timer = cleanup_timer
+        self._spots = spots
         self._spot_providers = spot_providers
+        self._alerts = alerts
         self._alert_providers = alert_providers
         self._solar_condition_providers = solar_condition_providers
         self._thread = None
         self._stop_event = Event()
         self._startup_time = datetime.now(pytz.UTC)
 
-        DATA_STORE.status_data["software-version"] = SOFTWARE_VERSION
-        DATA_STORE.status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
+        self._status_data["software-version"] = SOFTWARE_VERSION
+        self._status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
 
     def start(self):
         """Start the reporter thread"""
@@ -51,41 +55,44 @@ class StatusReporter:
     def _report(self):
         """Write status information"""
 
-        DATA_STORE.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
-        DATA_STORE.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
-        DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
-        DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
-        DATA_STORE.status_data["spot_providers"] = list(
-        map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                       "last_updated": p.last_update_time.replace(
-                           tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
-                       "last_spot": p.last_spot_time.replace(
-                           tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
-            self._spot_providers))
-        DATA_STORE.status_data["alert_providers"] = list(
-        map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                       "last_updated": p.last_update_time.replace(
-                           tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-            self._alert_providers))
-        DATA_STORE.status_data["solar_condition_providers"] = list(
-        map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                       "last_updated": p.last_update_time.replace(
-                           tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-            self._solar_condition_providers))
-        DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
-                                      "last_api_access": self._web_server.web_server_metrics[
-                                          "last_api_access_time"].replace(
-                                          tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
-                                          "last_api_access_time"] else 0,
-                                      "api_access_count": self._web_server.web_server_metrics["api_access_counter"],
-                                      "last_page_access": self._web_server.web_server_metrics[
-                                          "last_page_access_time"].replace(
-                                          tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
-                                          "last_page_access_time"] else 0,
-                                      "page_access_count": self._web_server.web_server_metrics[
-                                          "page_access_counter"]}
+        self._status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
+        self._status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
+        self._status_data["num_spots"] = len(self._spots)
+        self._status_data["num_alerts"] = len(self._alerts)
+        self._status_data["spot_providers"] = list(
+            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+                           "last_updated": p.last_update_time.replace(
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
+                           "last_spot": p.last_spot_time.replace(
+                               tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
+                self._spot_providers))
+        self._status_data["alert_providers"] = list(
+            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+                           "last_updated": p.last_update_time.replace(
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                self._alert_providers))
+        self._status_data["solar_condition_providers"] = list(
+            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+                           "last_updated": p.last_update_time.replace(
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                self._solar_condition_providers))
+        self._status_data["cleanup"] = {"status": self._cleanup_timer.status,
+                                        "last_ran": self._cleanup_timer.last_cleanup_time.replace(
+                                            tzinfo=pytz.UTC).timestamp() if self._cleanup_timer.last_cleanup_time else 0}
+        self._status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
+                                          "last_api_access": self._web_server.web_server_metrics[
+                                              "last_api_access_time"].replace(
+                                              tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
+                                              "last_api_access_time"] else 0,
+                                          "api_access_count": self._web_server.web_server_metrics["api_access_counter"],
+                                          "last_page_access": self._web_server.web_server_metrics[
+                                              "last_page_access_time"].replace(
+                                              tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
+                                              "last_page_access_time"] else 0,
+                                          "page_access_count": self._web_server.web_server_metrics[
+                                              "page_access_counter"]}
 
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)
-        spots_gauge.set(len(DATA_STORE.spots.values()))
-        alerts_gauge.set(len(DATA_STORE.alerts.values()))
+        spots_gauge.set(len(self._spots))
+        alerts_gauge.set(len(self._alerts))

core/url_data_cache.py

@@ -1,23 +0,0 @@
-import threading
-from datetime import timedelta
-
-from requests_cache import CachedSession
-
-# Cache for "semi-static" data retrieved from a URL. This is a layet of caching in addition to the normal caching of
-# spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints frequently when
-# e.g. restarting Spothole many times during testing.
-_session = CachedSession("cache/semi_static_urls", expire_after=timedelta(days=1),
-                         allowable_codes=(200, 400, 401, 403, 404))
-_lock = threading.Lock()
-
-
-class _ThreadSafeSession:
-    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads. This allows a single object
-    to be used freely across the application."""
-
-    def get(self, *args, **kwargs):
-        with _lock:
-            return _session.get(*args, **kwargs)
-
-# Global object
-URL_DATA_CACHE = _ThreadSafeSession()

core/utils.py

@@ -5,4 +5,14 @@ def safe_json_dumps(obj):
     """Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
     which are invalid in JSON."""
 
-    return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
+    return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
+
+
+def empty_queue(q):
+    """Empty a queue"""
+
+    while not q.empty():
+        try:
+            q.get_nowait()
+        except:
+            break

requirements.txt

@@ -17,5 +17,4 @@ websocket-client~=1.8.0
 tornado~=6.4.2
 tornado_eventsource~=3.0.0
 geopandas~=0.13.2
-simplejson~=4.1.1
-cachetools~=7.1.6
+simplejson~=4.1.1

server/handlers/api/addspot.py

@@ -8,7 +8,7 @@ import tornado
 from tornado import httputil
 from tornado.web import Application
 
-from core.config import ALLOW_SPOTTING
+from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
 from core.constants import UNKNOWN_BAND
 from core.lookup_helper import infer_band_from_freq
 from core.prometheus_metrics_handler import api_requests_counter
@@ -119,7 +119,7 @@ class APISpotHandler(tornado.web.RequestHandler):
             # infer missing data, and add it to our database.
             spot.source = "API"
             spot.infer_missing()
-            self._spots.set(spot.id, spot)
+            self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
 
             self.write(safe_json_dumps("OK"))
             self.set_status(201)

server/handlers/api/alerts.py

@@ -1,6 +1,7 @@
 import copy
 import logging
 from datetime import datetime
+from queue import Queue
 from typing import Any
 
 import pytz
@@ -10,9 +11,12 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import safe_json_dumps
+from core.utils import safe_json_dumps, empty_queue
 from data.lookup_credentials import extract_credentials
 
+SSE_HANDLER_MAX_QUEUE_SIZE = 100
+SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
+
 
 class APIAlertsHandler(tornado.web.RequestHandler):
     """API request handler for /api/v1/alerts"""
@@ -69,14 +73,16 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
     """API request handler for /api/v1/alerts/stream"""
 
     def __init__(self, application, request, **kwargs: Any):
-        self._sse_alert_broadcaster = None
+        self._sse_alert_queues = None
         self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
+        self._alert_queue = None
+        self._heartbeat = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, _sse_alert_broadcaster, web_server_metrics):
-        self._sse_alert_broadcaster = _sse_alert_broadcaster
+    def initialize(self, sse_alert_queues, web_server_metrics):
+        self._sse_alert_queues = sse_alert_queues
         self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
@@ -98,32 +104,59 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
             self._credentials = extract_credentials(self._query_params)
 
+            # Create a alert queue and add it to the web server's list. The web server will fill this when alerts arrive
+            self._alert_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
+            self._sse_alert_queues.append(self._alert_queue)
+
+            # Set up a timed callback to check if anything is in the queue
+            self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
+            self._heartbeat.start()
+
             # Flush headers immediately so nginx doesn't time out waiting for a response
             self.write_message("keepalive", "")
 
-            # Register to handle new alerts arriving. The callback() method will get called with the new alert as an
-            # argument.
-            self._sse_alert_broadcaster.register(self)
-
         except Exception as e:
             logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
             self.close()
 
     def close(self):
-        """When the user closes the socket, deregister ourselves from the alert broadcaster"""
-
-        self._sse_alert_broadcaster.unregister(self)
-        super().close()
-
-    def callback(self, alert):
-        """Callback when a new alert arrives"""
+        """When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
 
         try:
-            if alert_allowed_by_query(alert, self._query_params):
-                if self._credentials:
-                    alert = copy.deepcopy(alert)
-                    alert.infer_missing(self._credentials)
-                self.write_message(msg=safe_json_dumps(alert))
+            if self._alert_queue in self._sse_alert_queues:
+                self._sse_alert_queues.remove(self._alert_queue)
+            empty_queue(self._alert_queue)
+        except:
+            pass
+        try:
+            self._heartbeat.stop()
+        except:
+            pass
+        self._alert_queue = None
+        super().close()
+
+    def _callback(self):
+        """Callback to check if anything has arrived in the queue, and if so send it to the client"""
+
+        try:
+            if self._alert_queue:
+                if not self._alert_queue.empty():
+                    while not self._alert_queue.empty():
+                        alert = self._alert_queue.get()
+                        # If the new alert matches our param filters, send it to the client. If not, ignore it.
+                        if alert_allowed_by_query(alert, self._query_params):
+                            if self._credentials:
+                                alert = copy.deepcopy(alert)
+                                alert.infer_missing(self._credentials)
+                            self.write_message(msg=safe_json_dumps(alert))
+
+                else:
+                    # Send a keepalive comment if the queue was empty
+                    self.write_message("keepalive", "")
+
+                if self._alert_queue not in self._sse_alert_queues:
+                    logging.error("Web server cleared up a queue of an active connection!")
+                    self.close()
         except Exception as e:
             logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
             self.close()
@@ -136,7 +169,7 @@ def get_alert_list_with_filters(all_alerts, query):
     # Create a shallow copy of the alert list ordered by start time, then filter the list to reduce it only to alerts
     # that match the filter parameters in the query string. Finally, apply a limit to the number of alerts returned.
     # The list of query string filters is defined in the API docs.
-    alert_ids = all_alerts.keys()
+    alert_ids = list(all_alerts.iterkeys())
     alerts = []
     for k in alert_ids:
         a = all_alerts.get(k)

server/handlers/api/dxstats.py

@@ -40,7 +40,7 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
             one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
             counts = Counter()
 
-            for key in self._spots.keys():
+            for key in self._spots.iterkeys():
                 spot = self._spots.get(key)
                 if spot is None:
                     continue

server/handlers/api/spots.py

@@ -1,6 +1,7 @@
 import copy
 import logging
 from datetime import datetime, timedelta
+from queue import Queue
 from typing import Any
 
 import pytz
@@ -10,9 +11,12 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import safe_json_dumps
+from core.utils import safe_json_dumps, empty_queue
 from data.lookup_credentials import extract_credentials
 
+SSE_HANDLER_MAX_QUEUE_SIZE = 1000
+SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
+
 
 class APISpotsHandler(tornado.web.RequestHandler):
     """API request handler for /api/v1/spots"""
@@ -69,14 +73,16 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
     """API request handler for /api/v1/spots/stream"""
 
     def __init__(self, application, request, **kwargs: Any):
-        self._sse_spot_broadcaster = None
+        self._sse_spot_queues = None
         self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
+        self._spot_queue = None
+        self._heartbeat = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, sse_spot_broadcaster, web_server_metrics):
-        self._sse_spot_broadcaster = sse_spot_broadcaster
+    def initialize(self, sse_spot_queues, web_server_metrics):
+        self._sse_spot_queues = sse_spot_queues
         self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
@@ -100,33 +106,59 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
             self._credentials = extract_credentials(self._query_params)
 
+            # Create a spot queue and add it to the web server's list. The web server will fill this when spots arrive
+            self._spot_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
+            self._sse_spot_queues.append(self._spot_queue)
+
+            # Set up a timed callback to check if anything is in the queue
+            self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
+            self._heartbeat.start()
+
             # Flush headers immediately so nginx doesn't time out waiting for a response
             self.write_message("keepalive", "")
 
-            # Register to handle new spots arriving. The callback() method will get called with the new spot as an
-            # argument.
-            self._sse_spot_broadcaster.register(self)
-
         except Exception as e:
             logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
             self.close()
 
     def close(self):
-        """When the user closes the socket, deregister ourselves from the spot broadcaster"""
-
-        self._sse_spot_broadcaster.unregister(self)
-        super().close()
-
-    def callback(self, spot):
-        """Callback when a new spot arrives"""
+        """When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
 
         try:
-            # If the new spot matches our param filters, send it to the client. If not, ignore it.
-            if spot_allowed_by_query(spot, self._query_params):
-                if self._credentials:
-                    spot = copy.deepcopy(spot)
-                    spot.infer_missing(self._credentials)
-                self.write_message(msg=safe_json_dumps(spot))
+            if self._spot_queue in self._sse_spot_queues:
+                self._sse_spot_queues.remove(self._spot_queue)
+            empty_queue(self._spot_queue)
+        except:
+            pass
+        try:
+            self._heartbeat.stop()
+        except:
+            pass
+        self._spot_queue = None
+        super().close()
+
+    def _callback(self):
+        """Callback to check if anything has arrived in the queue, and if so send it to the client"""
+
+        try:
+            if self._spot_queue:
+                if not self._spot_queue.empty():
+                    while not self._spot_queue.empty():
+                        spot = self._spot_queue.get()
+                        # If the new spot matches our param filters, send it to the client. If not, ignore it.
+                        if spot_allowed_by_query(spot, self._query_params):
+                            if self._credentials:
+                                spot = copy.deepcopy(spot)
+                                spot.infer_missing(self._credentials)
+                            self.write_message(msg=safe_json_dumps(spot))
+
+                else:
+                    # Send a keepalive comment if the queue was empty
+                    self.write_message("keepalive", "")
+
+                if self._spot_queue not in self._sse_spot_queues:
+                    logging.error("Web server cleared up a queue of an active connection!")
+                    self.close()
         except Exception as e:
             logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
             self.close()
@@ -139,7 +171,7 @@ def get_spot_list_with_filters(all_spots, query):
     # Create a shallow copy of the spot list, ordered by spot time, then filter the list to reduce it only to spots
     # that match the filter parameters in the query string. Finally, apply a limit to the number of spots returned.
     # The list of query string filters is defined in the API docs.
-    spot_ids = all_spots.keys()
+    spot_ids = list(all_spots.iterkeys())
     spots = []
     for k in spot_ids:
         s = all_spots.get(k)

server/sse_broadcaster.py

@@ -1,36 +0,0 @@
-import logging
-import threading
-
-from tornado.ioloop import IOLoop
-
-
-class SSEBroadcaster:
-    """Bridge between DataStore listener callbacks (which fire on provider threads) to Tornado's async SSE handlers
-    (which live on the IOLoop thread) to avoid any interdependency between them."""
-
-    def __init__(self):
-        self._handlers = set()
-        self._lock = threading.Lock()
-        self._loop = IOLoop.current()
-
-    def register(self, handler):
-        with self._lock:
-            self._handlers.add(handler)
-
-    def unregister(self, handler):
-        with self._lock:
-            self._handlers.discard(handler)
-
-    def publish(self, value):
-        self._loop.add_callback(self._fan_out, value)
-
-    def _fan_out(self, value):
-        with self._lock:
-            handlers = list(self._handlers)
-        for handler in handlers:
-            try:
-                handler.callback(value)
-            except Exception:
-                # Connection probably dropped, ignore and de-register the handler to stop getting future items.
-                logging.debug("Failed to push to an SSE client; dropping it")
-                self.unregister(handler)

server/webserver.py

@@ -6,7 +6,7 @@ import tornado
 from tornado.web import StaticFileHandler
 
 from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL
-from core.data_store import DATA_STORE
+from core.utils import empty_queue
 from server.handlers.api.addspot import APISpotHandler
 from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
 from server.handlers.api.dxstats import APIDxStatsHandler
@@ -18,7 +18,6 @@ from server.handlers.api.status import APIStatusHandler
 from server.handlers.manifesthandler import ManifestHandler
 from server.handlers.metrics import PrometheusMetricsHandler
 from server.handlers.pagetemplate import PageTemplateHandler
-from server.sse_broadcaster import SSEBroadcaster
 
 _HERE = os.path.dirname(__file__ or "")
 
@@ -26,12 +25,15 @@ _HERE = os.path.dirname(__file__ or "")
 class WebServer:
     """Provides the public-facing web server."""
 
-    def __init__(self):
+    def __init__(self, spots, alerts, solar_conditions, status_data):
         """Constructor"""
 
-        self._data_store = DATA_STORE
-        self._spot_broadcaster = SSEBroadcaster()
-        self._alert_broadcaster = SSEBroadcaster()
+        self._spots = spots
+        self._alerts = alerts
+        self._solar_conditions = solar_conditions
+        self._sse_spot_queues = []
+        self._sse_alert_queues = []
+        self._status_data = status_data
         self._port = WEB_SERVER_PORT
         self._api_only_mode = API_ONLY_MODE
         self._shutdown_event = asyncio.Event()
@@ -43,10 +45,6 @@ class WebServer:
             "status": "Starting"
         }
 
-        # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
-        DATA_STORE.spots.add_listener(self._spot_broadcaster.publish)
-        DATA_STORE.alerts.add_listener(self._alert_broadcaster.publish)
-
     def start(self):
         """Start the web server"""
 
@@ -66,20 +64,20 @@ class WebServer:
 
         # API endpoints are always enabled
         api_routes = [
-            (r"/api/v1/spots", APISpotsHandler, {"spots": self._data_store.spots, **handler_opts}),
-            (r"/api/v1/alerts", APIAlertsHandler, {"alerts": self._data_store.alerts, **handler_opts}),
+            (r"/api/v1/spots", APISpotsHandler, {"spots": self._spots, **handler_opts}),
+            (r"/api/v1/alerts", APIAlertsHandler, {"alerts": self._alerts, **handler_opts}),
             (r"/api/v1/spots/stream", APISpotsStreamHandler,
-             {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts}),
+             {"sse_spot_queues": self._sse_spot_queues, **handler_opts}),
             (r"/api/v1/alerts/stream", APIAlertsStreamHandler,
-             {"sse_alert_broadcaster": self._alert_broadcaster, **handler_opts}),
-            (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._data_store.solar, **handler_opts}),
-            (r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._data_store.spots, **handler_opts}),
-            (r"/api/v1/options", APIOptionsHandler, {"status_data": self._data_store.status_data, **handler_opts}),
-            (r"/api/v1/status", APIStatusHandler, {"status_data": self._data_store.status_data, **handler_opts}),
+             {"sse_alert_queues": self._sse_alert_queues, **handler_opts}),
+            (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._solar_conditions, **handler_opts}),
+            (r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._spots, **handler_opts}),
+            (r"/api/v1/options", APIOptionsHandler, {"status_data": self._status_data, **handler_opts}),
+            (r"/api/v1/status", APIStatusHandler, {"status_data": self._status_data, **handler_opts}),
             (r"/api/v1/lookup/call", APILookupCallHandler, {**handler_opts}),
             (r"/api/v1/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
             (r"/api/v1/lookup/grid", APILookupGridHandler, {**handler_opts}),
-            (r"/api/v1/spot", APISpotHandler, {"spots": self._data_store.spots, **handler_opts}),
+            (r"/api/v1/spot", APISpotHandler, {"spots": self._spots, **handler_opts}),
         ]
 
         # If in API-only mode, serve a basic homepage; in normal mode, serve the usual UI routes
@@ -123,6 +121,55 @@ class WebServer:
         logging.info("You can access your copy of Spothole at " + BASE_URL)
         await self._shutdown_event.wait()
 
+    def notify_new_spot(self, spot):
+        """Internal method called when a new spot is added to the system. This is used to ping any SSE clients that are
+        awaiting a server-sent message with new spots."""
+
+        for queue in self._sse_spot_queues:
+            try:
+                queue.put(spot)
+            except:
+                # Cleanup thread was probably deleting the queue, that's fine
+                pass
+        pass
+
+    def notify_new_alert(self, alert):
+        """Internal method called when a new alert is added to the system. This is used to ping any SSE clients that are
+        awaiting a server-sent message with new spots."""
+
+        for queue in self._sse_alert_queues:
+            try:
+                queue.put(alert)
+            except:
+                # Cleanup thread was probably deleting the queue, that's fine
+                pass
+        pass
+
+    def clean_up_sse_queues(self):
+        """Clean up any SSE queues that are growing too large; probably their client disconnected and we didn't catch it
+        properly for some reason."""
+
+        for q in self._sse_spot_queues:
+            try:
+                if q.full():
+                    logging.warning(
+                        "A full SSE spot queue was found, presumably because the client disconnected strangely. It has been removed.")
+                    self._sse_spot_queues.remove(q)
+                    empty_queue(q)
+            except:
+                # Probably got deleted already on another thread
+                pass
+        for q in self._sse_alert_queues:
+            try:
+                if q.full():
+                    logging.warning(
+                        "A full SSE alert queue was found, presumably because the client disconnected strangely. It has been removed.")
+                    self._sse_alert_queues.remove(q)
+                    empty_queue(q)
+            except:
+                # Probably got deleted already on another thread
+                pass
+        pass
 
 def request_log(handler):
     """Custom log function to provide more data about requests when enabled, and to provide the ability to turn off

sigrefdataproviders/sig_ref_data_provider.py

@@ -1,28 +0,0 @@
-from datetime import datetime
-
-import pytz
-
-
-class SIGRefDataProvider:
-    """Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data."""
-
-    def __init__(self, name, provider_config):
-        """Constructor"""
-
-        self.name = name
-        self.enabled = provider_config["enabled"]
-        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
-        self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
-        self.status = "Not Started" if self.enabled else "Disabled"
-
-
-    def start(self):
-        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
-
-        raise NotImplementedError("Subclasses must implement this method")
-
-
-    def stop(self):
-        """Stop any threads and prepare for application shutdown"""
-
-        raise NotImplementedError("Subclasses must implement this method")

solarconditionsproviders/giroionosonde.py

@@ -35,18 +35,6 @@ class GIROIonosonde(SolarConditionsProvider):
         self._thread = None
         self._stop_event = Event()
 
-        # Pre-populate ionosonde_data with known station names for stations not already present,
-        # so the station dropdown is available before the first poll. Does not overwrite existing
-        # entries so KC2G cache data is preserved.
-        existing = self._solar_conditions.ionosonde_data or {}
-        new_entries = {
-            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
-                        "luf": None, "band_states": None}
-            for s in self._stations if s["ursi"] not in existing
-        }
-        if new_entries:
-            self.update_data({"ionosonde_data": {**existing, **new_entries}})
-
     @staticmethod
     def _load_stations():
         stations = []
@@ -56,6 +44,21 @@ class GIROIonosonde(SolarConditionsProvider):
                     stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
         return stations
 
+    def setup(self, solar_conditions, solar_conditions_cache):
+        """Pre-populate ionosonde_data with known station names for stations not already present,
+        so the station dropdown is available before the first poll. Does not overwrite existing
+        entries so KC2G cache data is preserved."""
+
+        super().setup(solar_conditions, solar_conditions_cache)
+        existing = solar_conditions.ionosonde_data or {}
+        new_entries = {
+            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
+                        "luf": None, "band_states": None}
+            for s in self._stations if s["ursi"] not in existing
+        }
+        if new_entries:
+            self.update_data({"ionosonde_data": {**existing, **new_entries}})
+
     def start(self):
         logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
         self._thread = Thread(target=self._run, daemon=True)

solarconditionsproviders/solar_conditions_provider.py

@@ -2,8 +2,6 @@ from datetime import datetime
 
 import pytz
 
-from core.data_store import DATA_STORE
-
 
 class SolarConditionsProvider:
     """Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
@@ -12,11 +10,18 @@ class SolarConditionsProvider:
     def __init__(self, name, provider_config):
         """Constructor"""
 
+        self._solar_conditions_cache = None
         self.name = name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._solar_conditions = DATA_STORE.solar_conditions
+        self._solar_conditions = None
+
+    def setup(self, solar_conditions, solar_conditions_cache):
+        """Set up the provider, giving it the solar conditions object and its backing cache"""
+
+        self._solar_conditions = solar_conditions
+        self._solar_conditions_cache = solar_conditions_cache
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -36,3 +41,4 @@ class SolarConditionsProvider:
                 if hasattr(self._solar_conditions, key):
                     setattr(self._solar_conditions, key, value)
             self._solar_conditions.infer_descriptions()
+            self._solar_conditions_cache['solar_conditions'] = self._solar_conditions

spothole.py

@@ -5,15 +5,23 @@ import os
 import signal
 import sys
 
+from diskcache import Cache
+
+from core.cleanup import CleanupTimer
 from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL
 from core.constants import SOFTWARE_VERSION
-from core.data_store import DATA_STORE
 from core.lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
+from data.solar_conditions import SolarConditions
 from server.webserver import WebServer
 
 # Globals
+spots = Cache('cache/spots_cache')
+alerts = Cache('cache/alerts_cache')
+solar_conditions_cache = Cache('cache/solar_conditions_cache')
+solar_conditions = solar_conditions_cache.get('solar_conditions', SolarConditions())
 web_server = None
+status_data = {}
 spot_providers = []
 alert_providers = []
 solar_condition_providers = []
@@ -38,7 +46,13 @@ def shutdown(_signum=None, _frame=None):
     for scp in solar_condition_providers:
         if scp.enabled:
             scp.stop()
-    DATA_STORE.close()
+    if cleanup_timer:
+        cleanup_timer.stop()
+    if lookup_helper:
+        lookup_helper.stop()
+    spots.close()
+    alerts.close()
+    solar_conditions_cache.close()
     os._exit(0)
 
 
@@ -89,12 +103,13 @@ if __name__ == '__main__':
     lookup_helper.start()
 
     # Set up web server
-    web_server = WebServer()
+    web_server = WebServer(spots=spots, alerts=alerts, solar_conditions=solar_conditions, status_data=status_data)
 
     # Fetch, set up and start spot providers
     for entry in config["spot-providers"]:
         spot_providers.append(get_spot_provider_from_config(entry))
     for p in spot_providers:
+        p.setup(spots=spots, web_server=web_server)
         if p.enabled:
             p.start()
 
@@ -102,6 +117,7 @@ if __name__ == '__main__':
     for entry in config["alert-providers"]:
         alert_providers.append(get_alert_provider_from_config(entry))
     for p in alert_providers:
+        p.setup(alerts=alerts, web_server=web_server)
         if p.enabled:
             p.start()
 
@@ -109,11 +125,17 @@ if __name__ == '__main__':
     for entry in config.get("solar-condition-providers", []):
         solar_condition_providers.append(get_solar_conditions_provider_from_config(entry))
     for p in solar_condition_providers:
+        p.setup(solar_conditions=solar_conditions, solar_conditions_cache=solar_conditions_cache)
         if p.enabled:
             p.start()
 
+    # Set up timer to clear spot list of old data
+    cleanup_timer = CleanupTimer(spots=spots, alerts=alerts, web_server=web_server, cleanup_interval=60)
+    cleanup_timer.start()
+
     # Set up status reporter
-    status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers,
+    status_reporter = StatusReporter(status_data=status_data, spots=spots, alerts=alerts, web_server=web_server,
+                                     cleanup_timer=cleanup_timer, spot_providers=spot_providers,
                                      alert_providers=alert_providers,
                                      solar_condition_providers=solar_condition_providers, run_interval=5)
     status_reporter.start()

spotproviders/gma.py

@@ -3,8 +3,8 @@ from datetime import datetime
 
 import pytz
 
+from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import HTTP_HEADERS
-from core.url_data_cache import URL_DATA_CACHE
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -56,8 +56,8 @@ class GMA(HTTPSpotProvider):
                 # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
                 if "REF" in source_spot:
                     try:
-                        ref_response = URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
-                                                          headers=HTTP_HEADERS)
+                        ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
+                                                                      headers=HTTP_HEADERS)
                         # Sometimes this is blank even if it's a 200 response, so handle that
                         if ref_response.ok and ref_response.text is not None and ref_response.text != "":
                             ref_info = ref_response.json()

spotproviders/spot_provider.py

@@ -2,7 +2,7 @@ from datetime import datetime
 
 import pytz
 
-from core.data_store import DATA_STORE
+from core.config import MAX_SPOT_AGE
 
 
 class SpotProvider:
@@ -16,7 +16,14 @@ class SpotProvider:
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._spots = DATA_STORE.spots
+        self._spots = None
+        self._web_server = None
+
+    def setup(self, spots, web_server):
+        """Set up the provider, e.g. giving it the spot list to work from"""
+
+        self._spots = spots
+        self._web_server = web_server
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -52,7 +59,10 @@ class SpotProvider:
 
     def _add_spot(self, spot):
         if not spot.expired():
-            self._spots.set(spot.id, spot)
+            self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
+            # Ping the web server in case we have any SSE connections that need to see this immediately
+            if self._web_server:
+                self._web_server.notify_new_spot(spot)
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1785434214"></script>
+<script src="/static/js/add-spot.js?v=1785667229"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -75,7 +75,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1785434213"></script>
+<script src="/static/js/alerts.js?v=1785667229"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -75,8 +75,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/static/js/spotsbandsandmap.js?v=1785434214"></script>
-<script src="/static/js/bands.js?v=1785434214"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1785667229"></script>
+<script src="/static/js/bands.js?v=1785667229"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1785434213" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1785667228" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
 <script src="/static/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
 <script src="/static/vendor/js/tinycolor2-1.6.0.min.js"></script>
 
-<script src="/static/js/utils.js?v=1785434213"></script>
-<script src="/static/js/ui-ham.js?v=1785434213"></script>
-<script src="/static/js/geo.js?v=1785434213"></script>
-<script src="/static/js/common.js?v=1785434213"></script>
+<script src="/static/js/utils.js?v=1785667228"></script>
+<script src="/static/js/ui-ham.js?v=1785667228"></script>
+<script src="/static/js/geo.js?v=1785667228"></script>
+<script src="/static/js/common.js?v=1785667228"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1785434213"></script>
+<script src="/static/js/conditions.js?v=1785667229"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -108,8 +108,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/static/js/spotsbandsandmap.js?v=1785434213"></script>
-<script src="/static/js/map.js?v=1785434213"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1785667228"></script>
+<script src="/static/js/map.js?v=1785667228"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -116,8 +116,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/static/js/spotsbandsandmap.js?v=1785434213"></script>
-<script src="/static/js/spots.js?v=1785434213"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1785667228"></script>
+<script src="/static/js/spots.js?v=1785667228"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -59,7 +59,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1785434213"></script>
+<script src="/static/js/status.js?v=1785667229"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");