Refactor of caching & data storage part 2 #118

alertproviders/alert_provider.py

@@ -2,7 +2,7 @@ from datetime import datetime
 
 import pytz
 
-from core.config import MAX_ALERT_AGE
+from core.data_store import DATA_STORE
 
 
 class AlertProvider:
@@ -15,14 +15,7 @@ class AlertProvider:
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._alerts = None
+        self._alerts = DATA_STORE.alerts
-        self._web_server = None
-
-    def setup(self, alerts, web_server):
-        """Set up the provider, e.g. giving it the alert list to work from"""
-
-        self._alerts = alerts
-        self._web_server = web_server
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -44,10 +37,7 @@ class AlertProvider:
 
     def _add_alert(self, alert):
         if not alert.expired():
-            self._alerts.add(alert.id, alert, expire=MAX_ALERT_AGE)
+            self._alerts.set(alert.id, alert)
-            # Ping the web server in case we have any SSE connections that need to see this immediately
-            if self._web_server:
-                self._web_server.notify_new_alert(alert)
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""

core/cache_utils.py

@@ -1,27 +0,0 @@
-import threading
-from datetime import timedelta
-
-from requests_cache import CachedSession
-
-# Cache for "semi-static" data such as the locations of parks, CSVs of reference lists, etc.
-# This has an expiry time of 30 days, so will re-request from the source after that amount
-# of time has passed. This is used throughout Spothole to cache data that does not change
-# rapidly. The ThreadSafeSession construct here protects it against some multithreading
-# contention weirdness we sometimes used to see on startup where the cache was hammered
-# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type
-# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote
-# servers for data they've told us they can't provide.
-_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30),
-                         allowable_codes=(200, 400, 401, 403, 404))
-_lock = threading.Lock()
-
-
-class _ThreadSafeSession:
-    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads."""
-
-    def get(self, *args, **kwargs):
-        with _lock:
-            return _session.get(*args, **kwargs)
-
-
-SEMI_STATIC_URL_DATA_CACHE = _ThreadSafeSession()

core/cleanup.py

@@ -1,73 +0,0 @@
-import logging
-from datetime import datetime
-from threading import Event, Thread
-
-import pytz
-
-
-class CleanupTimer:
-    """Provides a timed cleanup of the spot list."""
-
-    def __init__(self, spots, alerts, web_server, cleanup_interval):
-        """Constructor"""
-
-        self._spots = spots
-        self._alerts = alerts
-        self._web_server = web_server
-        self._cleanup_interval = cleanup_interval
-        self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC)
-        self.status = "Starting"
-        self._thread = None
-        self._stop_event = Event()
-
-    def start(self):
-        """Start the cleanup timer"""
-
-        self._thread = Thread(target=self._run, daemon=True)
-        self._thread.start()
-
-    def stop(self):
-        """Stop any threads and prepare for application shutdown"""
-
-        self._stop_event.set()
-
-    def _run(self):
-        while not self._stop_event.wait(timeout=self._cleanup_interval):
-            self._cleanup()
-
-    def _cleanup(self):
-        """Perform cleanup and reschedule next timer"""
-
-        try:
-            # Perform cleanup via letting the data expire
-            self._spots.expire()
-            self._alerts.expire()
-
-            # Explicitly clean up any spots and alerts that have expired
-            for i in list(self._spots.iterkeys()):
-                try:
-                    spot = self._spots[i]
-                    if spot.expired():
-                        self._spots.delete(i)
-                except KeyError:
-                    # Must have already been deleted, OK with that
-                    pass
-            for i in list(self._alerts.iterkeys()):
-                try:
-                    alert = self._alerts[i]
-                    if alert.expired():
-                        self._alerts.delete(i)
-                except KeyError:
-                    # Must have already been deleted, OK with that
-                    pass
-
-            # Clean up web server SSE spot/alert queues
-            self._web_server.clean_up_sse_queues()
-
-            self.status = "OK"
-            self.last_cleanup_time = datetime.now(pytz.UTC)
-
-        except Exception:
-            self.status = "Error"
-            logging.exception("Exception in Cleanup thread")
-            self._stop_event.wait(timeout=1)

core/data_store.py

@@ -0,0 +1,69 @@
+from pathlib import Path
+
+import diskcache
+
+from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
+from core.live_data_cache import LiveDataCache
+from data.solar_conditions import SolarConditions
+
+
+class DataStore:
+    """Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
+    lookup data using different caching strategies for each."""
+
+    def __init__(self):
+        cache_dir = "./cache"
+        self.MAX_SPOT_COUNT = 10000
+        self.MAX_ALERT_COUNT = 10000
+        self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
+        self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
+
+        Path(cache_dir).mkdir(parents=True, exist_ok=True)
+
+        # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval
+        self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
+                                   snapshot_dir=cache_dir + "/spots",
+                                   snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+        self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
+                                   snapshot_dir=cache_dir + "/alerts",
+                                    snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+
+        # Standard disk cache for solar data and status data, but each cache contains only a single object which we
+        # expose to the wider application
+        self.solar = diskcache.Cache(cache_dir + "/solar")
+        if "solar_conditions" not in self.solar:
+            self.solar.add("solar_conditions", SolarConditions())
+        self.solar_conditions = self.solar.get("solar_conditions")
+        self.status = diskcache.Cache(cache_dir + "/status")
+        if "status_data" not in self.status:
+            self.status.add("status_data", {})
+        self.status_data = self.status.get("status_data")
+
+        # Standard disk caches for SIG ref data. Separate provider threads will repopulate these on a regular basis
+        # but there's no need for a TTL since old data is better than no data.
+        self.sigrefs_wwff = diskcache.Cache(cache_dir + "/sigrefs/wwff")
+        self.sigrefs_siota = diskcache.Cache(cache_dir + "/sigrefs/siota")
+        self.sigrefs_wota = diskcache.Cache(cache_dir + "/sigrefs/wota")
+        self.sigrefs_zlota = diskcache.Cache(cache_dir + "/sigrefs/zlota")
+        self.sigrefs_llota = diskcache.Cache(cache_dir + "/sigrefs/llota")
+        self.sigrefs_dme = diskcache.Cache(cache_dir + "/sigrefs/dme")
+
+        # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
+        # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
+        # we've seen, so we rely on them timing out and this triggering another lookup.
+        self.callsigns = diskcache.Cache(cache_dir + "/callsigns")
+
+    def close(self):
+        self.spots.close()
+        self.alerts.close()
+        self.solar.close()
+        self.status.close()
+        self.sigrefs_wwff.close()
+        self.sigrefs_siota.close()
+        self.sigrefs_wota.close()
+        self.sigrefs_zlota.close()
+        self.sigrefs_llota.close()
+        self.callsigns.close()
+
+# Global object
+DATA_STORE = DataStore()

core/live_data_cache.py

@@ -0,0 +1,100 @@
+import logging
+import threading
+import time
+
+import diskcache
+from cachetools import TTLCache
+
+
+class LiveDataCache:
+    """Cache for spots and alerts. Uses the fast in-memory TTLCache for normal data I/O, including the TTL to enforce
+    maximum lifetime, and adds a separate diskcache to which we can save and load the TTLCache to provide persistence.
+    Also adds thread safety so spots and alerts can come from any thread, and a listener mechanism so the web server
+    can get a callback when new spots/alerts are added, and send them to any SSE clients."""
+
+    def __init__(self, maxsize, ttl, snapshot_dir, snapshot_interval_sec):
+        self._cache = TTLCache(maxsize=maxsize, ttl=ttl)
+        self._lock = threading.Lock()
+        self._ttl = ttl
+        self._listeners = []
+        self._listeners_lock = threading.Lock()
+        self._snapshot_dir = snapshot_dir
+        self._disk_cache = diskcache.Cache(str(snapshot_dir))
+        self._load_snapshot()
+        self._start_periodic_snapshot(snapshot_interval_sec)
+
+    def set(self, key, value):
+        with self._lock:
+            self._cache[key] = value
+
+        # Notify listeners
+        with self._listeners_lock:
+            listeners = list(self._listeners)
+        for callback in listeners:
+            try:
+                callback(value)
+            except Exception:
+                logging.error("Listener raised an exception for key %s", key, exc_info=True)
+
+
+    def get(self, key, default=None):
+        with self._lock:
+            return self._cache.get(key, default)
+
+    def delete(self, key):
+        with self._lock:
+            self._cache.pop(key, None)
+
+    def keys(self):
+        with self._lock:
+            return list(self._cache.keys())
+
+    def values(self):
+        with self._lock:
+            return list(self._cache.values())
+
+    def add_listener(self, callback):
+        """Register callback(value) which will be called whenever a new spot/alert item is added via set(). Used by the
+        web server (via SSEBroadcaster) to send SSE clients an update on every new spot."""
+
+        with self._listeners_lock:
+            self._listeners.append(callback)
+
+    def remove_listener(self, callback):
+        with self._listeners_lock:
+            self._listeners.remove(callback)
+
+    def save_snapshot(self):
+        with self._lock:
+            # Store the time with the data so we can avoid loading anything nxt time that's older than TTL
+            data = [(k, v, time.time()) for k, v in self._cache.items()]
+        try:
+            self._disk_cache.set("snapshot", data)
+        except Exception as e:
+            logging.error("Failed to write snapshot to %s", self._snapshot_dir, e, exc_info=True)
+
+    def _load_snapshot(self):
+        data = self._disk_cache.get("snapshot")
+        if not data:
+            return
+
+        now = time.time()
+        with self._lock:
+            for key, value, saved_at in data:
+                # Only restore entries that would still be within TTL
+                if now - saved_at < self._ttl:
+                    self._cache[key] = value
+        logging.info("Loaded snapshot from %s", self._snapshot_dir)
+
+    def _start_periodic_snapshot(self, interval):
+        def loop():
+            while True:
+                time.sleep(interval)
+                self.save_snapshot()
+
+        t = threading.Thread(target=loop, daemon=True, name=f"snapshot-{self._snapshot_dir}")
+        t.start()
+
+    def close(self):
+        self.save_snapshot()
+        self._disk_cache.close()

core/lookup_helper.py

@@ -14,10 +14,10 @@ from pyhamtools.locator import latlong_to_locator
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
-from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.config import config
 from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
     HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
+from core.url_data_cache import URL_DATA_CACHE
 
 # QRZ XML field names differ from pyhamtools' normalised names; map them here.
 _QRZ_FIELD_MAP = {
@@ -142,8 +142,8 @@ class LookupHelper:
 
         try:
             logging.info("Downloading Country-files.com cty.plist...")
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
+            response = URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
-                                                      headers=HTTP_HEADERS)
+                                          headers=HTTP_HEADERS)
 
             if response.ok:
                 with open(self._country_files_cty_plist_download_location, "w") as f:
@@ -167,7 +167,7 @@ class LookupHelper:
 
         try:
             logging.info("Downloading dxcc.json...")
-            response = SEMI_STATIC_URL_DATA_CACHE.get(
+            response = URL_DATA_CACHE.get(
                 "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
                 headers=HTTP_HEADERS)
 
@@ -515,7 +515,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = SEMI_STATIC_URL_DATA_CACHE.get(
+                response = URL_DATA_CACHE.get(
                     self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
                     headers=HTTP_HEADERS, timeout=10)
                 if response.ok:
@@ -593,7 +593,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = SEMI_STATIC_URL_DATA_CACHE.get(
+                response = URL_DATA_CACHE.get(
                     self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                         lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
                 if response.ok:

core/sig_utils.py

@@ -4,23 +4,16 @@ import logging
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
-from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import SIGS, HTTP_HEADERS
+from core.data_store import DATA_STORE
 from core.geo_utils import wab_wai_square_to_lat_lon
+from core.url_data_cache import URL_DATA_CACHE
 
 # Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data
 # file in Spothole and load it on startup.
 with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
-    _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
+    for row in csv.DictReader(_f, delimiter=";"):
-# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
+        DATA_STORE.sigrefs_dme.add(row["COD_INE"][:5], row)
-# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
-# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
-_WWFF_INDEX_CACHE = {}
-_SIOTA_INDEX_CACHE = {}
-_WOTA_INDEX_CACHE = {}
-_ZLOTA_INDEX_CACHE = {}
-_LLOTA_INDEX_CACHE = {}
-
 
 def get_ref_regex_for_sig(sig):
     """Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
@@ -53,7 +46,7 @@ def populate_sig_ref_info(sig_ref):
     ref_id = sig_ref.id
     try:
         if sig.upper() == "POTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
+            response = URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if data:
@@ -71,7 +64,7 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "SOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
+            response = URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
@@ -88,7 +81,7 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "WWBOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
+            response = URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
@@ -104,7 +97,7 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
+            response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
@@ -130,15 +123,14 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "WWFF":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
+            response = URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
                                                       headers=HTTP_HEADERS)
             if response.ok:
-                global _WWFF_INDEX_CACHE
+                if not bool(DATA_STORE.sigrefs_wwff) or not response.from_cache:
-                if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
                     # New data from WWFF, update our internal map
-                    _WWFF_INDEX_CACHE = {row["reference"]: row for row in
+                    for row in csv.DictReader(response.content.decode().splitlines()):
-                                         csv.DictReader(response.content.decode().splitlines())}
+                        DATA_STORE.sigrefs_wwff.add(row["reference"], row)
-                row = _WWFF_INDEX_CACHE.get(ref_id)
+                row = DATA_STORE.sigrefs_wwff.get(ref_id)
                 if row:
                     sig_ref.name = row["name"] if "name" in row else None
                     sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
@@ -152,15 +144,14 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "SIOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
+            response = URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
                                                       headers=HTTP_HEADERS)
             if response.ok:
-                global _SIOTA_INDEX_CACHE
+                if not bool(DATA_STORE.sigrefs_siota) or not response.from_cache:
-                if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
                     # New data from SIOTA, update our internal map
-                    _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
+                    for row in csv.DictReader(response.content.decode().splitlines()):
-                                          csv.DictReader(response.content.decode().splitlines())}
+                        DATA_STORE.sigrefs_siota.add(row["SILO_CODE"], row)
-                row = _SIOTA_INDEX_CACHE.get(ref_id)
+                row = DATA_STORE.sigrefs_siota.get(ref_id)
                 if row:
                     sig_ref.name = row["NAME"] if "NAME" in row else None
                     sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
@@ -172,17 +163,16 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "WOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
+            response = URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if data:
-                    global _WOTA_INDEX_CACHE
+                    if not bool(DATA_STORE.sigrefs_wota) or not response.from_cache:
-                    if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
                         # New data from WOTA, update our internal map
-                        _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
+                        for feature in data.get("features", []):
-                                             data.get("features", [])}
+                            DATA_STORE.sigrefs_wota.add(feature["properties"]["wotaId"], feature)
-                    feature = _WOTA_INDEX_CACHE.get(ref_id)
+                    feature = DATA_STORE.sigrefs_wota.get(ref_id)
                     if feature:
                         sig_ref.name = feature["properties"]["title"]
                         # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -200,24 +190,24 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "ZLOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
+            response = URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if isinstance(data, list):
-                    global _ZLOTA_INDEX_CACHE
+                    if not bool(DATA_STORE.sigrefs_zlota) or not response.from_cache:
-                    if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
                         # New data from ZLOTA, update our internal map
-                        _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
+                        for ref in data:
-                    asset = _ZLOTA_INDEX_CACHE.get(ref_id)
+                            DATA_STORE.sigrefs_zlota.add(ref["code"], ref)
-                    if asset:
+                    ref = DATA_STORE.sigrefs_zlota.get(ref_id)
-                        sig_ref.name = asset["name"]
+                    if ref:
+                        sig_ref.name = ref["name"]
                         sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
                         try:
-                            sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
+                            sig_ref.grid = latlong_to_locator(ref["y"], ref["x"], 6)
                         except:
                             logging.debug("Invalid lat/lon received for reference")
-                            sig_ref.latitude = asset["y"]
+                            sig_ref.latitude = ref["y"]
-                            sig_ref.longitude = asset["x"]
+                            sig_ref.longitude = ref["x"]
                 elif not response.from_cache:
                     logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
             elif not response.from_cache:
@@ -229,16 +219,16 @@ def populate_sig_ref_info(sig_ref):
             sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
 
         elif sig.upper() == "LLOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
+            response = URL_DATA_CACHE.get("https://llota.app/api/public/references",
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if isinstance(data, list):
-                    global _LLOTA_INDEX_CACHE
+                    if not bool(DATA_STORE.sigrefs_llota) or not response.from_cache:
-                    if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
                         # New data from LLOTA, update our internal map
-                        _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
+                        for ref in data:
-                    ref = _LLOTA_INDEX_CACHE.get(ref_id)
+                            DATA_STORE.sigrefs_llota.add(ref["reference_code"], ref)
+                    ref = DATA_STORE.sigrefs_llota.get(ref_id)
                     if ref:
                         sig_ref.name = str(ref["name"])
                         sig_ref.url = "https://llota.app/list/ref/" + ref_id
@@ -280,7 +270,7 @@ def populate_sig_ref_info(sig_ref):
 
         elif sig.upper() == "DME":
             # Zero-pad to 5 digits to match our source data
-            row = _DME_INDEX.get(ref_id.zfill(5))
+            row = DATA_STORE.sigrefs_dme.get(ref_id.zfill(5))
             if row:
                 sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
                 sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(

core/status_reporter.py

@@ -7,31 +7,27 @@ import pytz
 
 from core.config import SERVER_OWNER_CALLSIGN
 from core.constants import SOFTWARE_VERSION
+from core.data_store import DATA_STORE
 from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge
 
 
 class StatusReporter:
     """Provides a timed update of the application's status data."""
 
-    def __init__(self, status_data, run_interval, web_server, cleanup_timer, spots, spot_providers, alerts,
+    def __init__(self, run_interval, web_server,spot_providers, alert_providers, solar_condition_providers):
-                 alert_providers, solar_condition_providers):
         """Constructor"""
 
-        self._status_data = status_data
         self._run_interval = run_interval
         self._web_server = web_server
-        self._cleanup_timer = cleanup_timer
-        self._spots = spots
         self._spot_providers = spot_providers
-        self._alerts = alerts
         self._alert_providers = alert_providers
         self._solar_condition_providers = solar_condition_providers
         self._thread = None
         self._stop_event = Event()
         self._startup_time = datetime.now(pytz.UTC)
 
-        self._status_data["software-version"] = SOFTWARE_VERSION
+        DATA_STORE.status_data["software-version"] = SOFTWARE_VERSION
-        self._status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
+        DATA_STORE.status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
 
     def start(self):
         """Start the reporter thread"""
@@ -55,44 +51,41 @@ class StatusReporter:
     def _report(self):
         """Write status information"""
 
-        self._status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
+        DATA_STORE.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
-        self._status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
+        DATA_STORE.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
-        self._status_data["num_spots"] = len(self._spots)
+        DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
-        self._status_data["num_alerts"] = len(self._alerts)
+        DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
-        self._status_data["spot_providers"] = list(
+        DATA_STORE.status_data["spot_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+        map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
+                       "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
+                           tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
-                           "last_spot": p.last_spot_time.replace(
+                       "last_spot": p.last_spot_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
+                           tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
-                self._spot_providers))
+            self._spot_providers))
-        self._status_data["alert_providers"] = list(
+        DATA_STORE.status_data["alert_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+        map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
+                       "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                           tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-                self._alert_providers))
+            self._alert_providers))
-        self._status_data["solar_condition_providers"] = list(
+        DATA_STORE.status_data["solar_condition_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+        map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
+                       "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                           tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-                self._solar_condition_providers))
+            self._solar_condition_providers))
-        self._status_data["cleanup"] = {"status": self._cleanup_timer.status,
+        DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
-                                        "last_ran": self._cleanup_timer.last_cleanup_time.replace(
+                                      "last_api_access": self._web_server.web_server_metrics[
-                                            tzinfo=pytz.UTC).timestamp() if self._cleanup_timer.last_cleanup_time else 0}
+                                          "last_api_access_time"].replace(
-        self._status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
+                                          tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
-                                          "last_api_access": self._web_server.web_server_metrics[
+                                          "last_api_access_time"] else 0,
-                                              "last_api_access_time"].replace(
+                                      "api_access_count": self._web_server.web_server_metrics["api_access_counter"],
-                                              tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
+                                      "last_page_access": self._web_server.web_server_metrics[
-                                              "last_api_access_time"] else 0,
+                                          "last_page_access_time"].replace(
-                                          "api_access_count": self._web_server.web_server_metrics["api_access_counter"],
+                                          tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
-                                          "last_page_access": self._web_server.web_server_metrics[
+                                          "last_page_access_time"] else 0,
-                                              "last_page_access_time"].replace(
+                                      "page_access_count": self._web_server.web_server_metrics[
-                                              tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
+                                          "page_access_counter"]}
-                                              "last_page_access_time"] else 0,
-                                          "page_access_count": self._web_server.web_server_metrics[
-                                              "page_access_counter"]}
 
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)
-        spots_gauge.set(len(self._spots))
+        spots_gauge.set(len(DATA_STORE.spots.values()))
-        alerts_gauge.set(len(self._alerts))
+        alerts_gauge.set(len(DATA_STORE.alerts.values()))

core/url_data_cache.py

@@ -0,0 +1,23 @@
+import threading
+from datetime import timedelta
+
+from requests_cache import CachedSession
+
+# Cache for "semi-static" data retrieved from a URL. This is a layet of caching in addition to the normal caching of
+# spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints frequently when
+# e.g. restarting Spothole many times during testing.
+_session = CachedSession("cache/semi_static_urls", expire_after=timedelta(days=1),
+                         allowable_codes=(200, 400, 401, 403, 404))
+_lock = threading.Lock()
+
+
+class _ThreadSafeSession:
+    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads. This allows a single object
+    to be used freely across the application."""
+
+    def get(self, *args, **kwargs):
+        with _lock:
+            return _session.get(*args, **kwargs)
+
+# Global object
+URL_DATA_CACHE = _ThreadSafeSession()

core/utils.py

@@ -6,13 +6,3 @@ def safe_json_dumps(obj):
     which are invalid in JSON."""
 
     return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
-
-
-def empty_queue(q):
-    """Empty a queue"""
-
-    while not q.empty():
-        try:
-            q.get_nowait()
-        except:
-            break

requirements.txt

@@ -18,3 +18,4 @@ tornado~=6.4.2
 tornado_eventsource~=3.0.0
 geopandas~=0.13.2
 simplejson~=4.1.1
+cachetools~=7.1.6

server/handlers/api/addspot.py

@@ -8,7 +8,7 @@ import tornado
 from tornado import httputil
 from tornado.web import Application
 
-from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
+from core.config import ALLOW_SPOTTING
 from core.constants import UNKNOWN_BAND
 from core.lookup_helper import infer_band_from_freq
 from core.prometheus_metrics_handler import api_requests_counter
@@ -119,7 +119,7 @@ class APISpotHandler(tornado.web.RequestHandler):
             # infer missing data, and add it to our database.
             spot.source = "API"
             spot.infer_missing()
-            self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
+            self._spots.set(spot.id, spot)
 
             self.write(safe_json_dumps("OK"))
             self.set_status(201)

server/handlers/api/alerts.py

@@ -1,7 +1,6 @@
 import copy
 import logging
 from datetime import datetime
-from queue import Queue
 from typing import Any
 
 import pytz
@@ -11,12 +10,9 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import safe_json_dumps, empty_queue
+from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 
-SSE_HANDLER_MAX_QUEUE_SIZE = 100
-SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
-
 
 class APIAlertsHandler(tornado.web.RequestHandler):
     """API request handler for /api/v1/alerts"""
@@ -73,16 +69,14 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
     """API request handler for /api/v1/alerts/stream"""
 
     def __init__(self, application, request, **kwargs: Any):
-        self._sse_alert_queues = None
+        self._sse_alert_broadcaster = None
         self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
-        self._alert_queue = None
-        self._heartbeat = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, sse_alert_queues, web_server_metrics):
+    def initialize(self, _sse_alert_broadcaster, web_server_metrics):
-        self._sse_alert_queues = sse_alert_queues
+        self._sse_alert_broadcaster = _sse_alert_broadcaster
         self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
@@ -104,59 +98,32 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
             self._credentials = extract_credentials(self._query_params)
 
-            # Create a alert queue and add it to the web server's list. The web server will fill this when alerts arrive
-            self._alert_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
-            self._sse_alert_queues.append(self._alert_queue)
-
-            # Set up a timed callback to check if anything is in the queue
-            self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
-            self._heartbeat.start()
-
             # Flush headers immediately so nginx doesn't time out waiting for a response
             self.write_message("keepalive", "")
 
+            # Register to handle new alerts arriving. The callback() method will get called with the new alert as an
+            # argument.
+            self._sse_alert_broadcaster.register(self)
+
         except Exception as e:
             logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
             self.close()
 
     def close(self):
-        """When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
+        """When the user closes the socket, deregister ourselves from the alert broadcaster"""
 
-        try:
+        self._sse_alert_broadcaster.unregister(self)
-            if self._alert_queue in self._sse_alert_queues:
-                self._sse_alert_queues.remove(self._alert_queue)
-            empty_queue(self._alert_queue)
-        except:
-            pass
-        try:
-            self._heartbeat.stop()
-        except:
-            pass
-        self._alert_queue = None
         super().close()
 
-    def _callback(self):
+    def callback(self, alert):
-        """Callback to check if anything has arrived in the queue, and if so send it to the client"""
+        """Callback when a new alert arrives"""
 
         try:
-            if self._alert_queue:
+            if alert_allowed_by_query(alert, self._query_params):
-                if not self._alert_queue.empty():
+                if self._credentials:
-                    while not self._alert_queue.empty():
+                    alert = copy.deepcopy(alert)
-                        alert = self._alert_queue.get()
+                    alert.infer_missing(self._credentials)
-                        # If the new alert matches our param filters, send it to the client. If not, ignore it.
+                self.write_message(msg=safe_json_dumps(alert))
-                        if alert_allowed_by_query(alert, self._query_params):
-                            if self._credentials:
-                                alert = copy.deepcopy(alert)
-                                alert.infer_missing(self._credentials)
-                            self.write_message(msg=safe_json_dumps(alert))
-
-                else:
-                    # Send a keepalive comment if the queue was empty
-                    self.write_message("keepalive", "")
-
-                if self._alert_queue not in self._sse_alert_queues:
-                    logging.error("Web server cleared up a queue of an active connection!")
-                    self.close()
         except Exception as e:
             logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
             self.close()
@@ -169,7 +136,7 @@ def get_alert_list_with_filters(all_alerts, query):
     # Create a shallow copy of the alert list ordered by start time, then filter the list to reduce it only to alerts
     # that match the filter parameters in the query string. Finally, apply a limit to the number of alerts returned.
     # The list of query string filters is defined in the API docs.
-    alert_ids = list(all_alerts.iterkeys())
+    alert_ids = all_alerts.keys()
     alerts = []
     for k in alert_ids:
         a = all_alerts.get(k)

server/handlers/api/dxstats.py

@@ -40,7 +40,7 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
             one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
             counts = Counter()
 
-            for key in self._spots.iterkeys():
+            for key in self._spots.keys():
                 spot = self._spots.get(key)
                 if spot is None:
                     continue

server/handlers/api/spots.py

@@ -1,7 +1,6 @@
 import copy
 import logging
 from datetime import datetime, timedelta
-from queue import Queue
 from typing import Any
 
 import pytz
@@ -11,12 +10,9 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import safe_json_dumps, empty_queue
+from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 
-SSE_HANDLER_MAX_QUEUE_SIZE = 1000
-SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
-
 
 class APISpotsHandler(tornado.web.RequestHandler):
     """API request handler for /api/v1/spots"""
@@ -73,16 +69,14 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
     """API request handler for /api/v1/spots/stream"""
 
     def __init__(self, application, request, **kwargs: Any):
-        self._sse_spot_queues = None
+        self._sse_spot_broadcaster = None
         self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
-        self._spot_queue = None
-        self._heartbeat = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, sse_spot_queues, web_server_metrics):
+    def initialize(self, sse_spot_broadcaster, web_server_metrics):
-        self._sse_spot_queues = sse_spot_queues
+        self._sse_spot_broadcaster = sse_spot_broadcaster
         self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
@@ -106,59 +100,33 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
             self._credentials = extract_credentials(self._query_params)
 
-            # Create a spot queue and add it to the web server's list. The web server will fill this when spots arrive
-            self._spot_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
-            self._sse_spot_queues.append(self._spot_queue)
-
-            # Set up a timed callback to check if anything is in the queue
-            self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
-            self._heartbeat.start()
-
             # Flush headers immediately so nginx doesn't time out waiting for a response
             self.write_message("keepalive", "")
 
+            # Register to handle new spots arriving. The callback() method will get called with the new spot as an
+            # argument.
+            self._sse_spot_broadcaster.register(self)
+
         except Exception as e:
             logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
             self.close()
 
     def close(self):
-        """When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
+        """When the user closes the socket, deregister ourselves from the spot broadcaster"""
 
-        try:
+        self._sse_spot_broadcaster.unregister(self)
-            if self._spot_queue in self._sse_spot_queues:
-                self._sse_spot_queues.remove(self._spot_queue)
-            empty_queue(self._spot_queue)
-        except:
-            pass
-        try:
-            self._heartbeat.stop()
-        except:
-            pass
-        self._spot_queue = None
         super().close()
 
-    def _callback(self):
+    def callback(self, spot):
-        """Callback to check if anything has arrived in the queue, and if so send it to the client"""
+        """Callback when a new spot arrives"""
 
         try:
-            if self._spot_queue:
+            # If the new spot matches our param filters, send it to the client. If not, ignore it.
-                if not self._spot_queue.empty():
+            if spot_allowed_by_query(spot, self._query_params):
-                    while not self._spot_queue.empty():
+                if self._credentials:
-                        spot = self._spot_queue.get()
+                    spot = copy.deepcopy(spot)
-                        # If the new spot matches our param filters, send it to the client. If not, ignore it.
+                    spot.infer_missing(self._credentials)
-                        if spot_allowed_by_query(spot, self._query_params):
+                self.write_message(msg=safe_json_dumps(spot))
-                            if self._credentials:
-                                spot = copy.deepcopy(spot)
-                                spot.infer_missing(self._credentials)
-                            self.write_message(msg=safe_json_dumps(spot))
-
-                else:
-                    # Send a keepalive comment if the queue was empty
-                    self.write_message("keepalive", "")
-
-                if self._spot_queue not in self._sse_spot_queues:
-                    logging.error("Web server cleared up a queue of an active connection!")
-                    self.close()
         except Exception as e:
             logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
             self.close()
@@ -171,7 +139,7 @@ def get_spot_list_with_filters(all_spots, query):
     # Create a shallow copy of the spot list, ordered by spot time, then filter the list to reduce it only to spots
     # that match the filter parameters in the query string. Finally, apply a limit to the number of spots returned.
     # The list of query string filters is defined in the API docs.
-    spot_ids = list(all_spots.iterkeys())
+    spot_ids = all_spots.keys()
     spots = []
     for k in spot_ids:
         s = all_spots.get(k)

server/sse_broadcaster.py

@@ -0,0 +1,36 @@
+import logging
+import threading
+
+from tornado.ioloop import IOLoop
+
+
+class SSEBroadcaster:
+    """Bridge between DataStore listener callbacks (which fire on provider threads) to Tornado's async SSE handlers
+    (which live on the IOLoop thread) to avoid any interdependency between them."""
+
+    def __init__(self):
+        self._handlers = set()
+        self._lock = threading.Lock()
+        self._loop = IOLoop.current()
+
+    def register(self, handler):
+        with self._lock:
+            self._handlers.add(handler)
+
+    def unregister(self, handler):
+        with self._lock:
+            self._handlers.discard(handler)
+
+    def publish(self, value):
+        self._loop.add_callback(self._fan_out, value)
+
+    def _fan_out(self, value):
+        with self._lock:
+            handlers = list(self._handlers)
+        for handler in handlers:
+            try:
+                handler.callback(value)
+            except Exception:
+                # Connection probably dropped, ignore and de-register the handler to stop getting future items.
+                logging.debug("Failed to push to an SSE client; dropping it")
+                self.unregister(handler)

server/webserver.py

@@ -6,7 +6,7 @@ import tornado
 from tornado.web import StaticFileHandler
 
 from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL
-from core.utils import empty_queue
+from core.data_store import DATA_STORE
 from server.handlers.api.addspot import APISpotHandler
 from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
 from server.handlers.api.dxstats import APIDxStatsHandler
@@ -18,6 +18,7 @@ from server.handlers.api.status import APIStatusHandler
 from server.handlers.manifesthandler import ManifestHandler
 from server.handlers.metrics import PrometheusMetricsHandler
 from server.handlers.pagetemplate import PageTemplateHandler
+from server.sse_broadcaster import SSEBroadcaster
 
 _HERE = os.path.dirname(__file__ or "")
 
@@ -25,15 +26,12 @@ _HERE = os.path.dirname(__file__ or "")
 class WebServer:
     """Provides the public-facing web server."""
 
-    def __init__(self, spots, alerts, solar_conditions, status_data):
+    def __init__(self):
         """Constructor"""
 
-        self._spots = spots
+        self._data_store = DATA_STORE
-        self._alerts = alerts
+        self._spot_broadcaster = SSEBroadcaster()
-        self._solar_conditions = solar_conditions
+        self._alert_broadcaster = SSEBroadcaster()
-        self._sse_spot_queues = []
-        self._sse_alert_queues = []
-        self._status_data = status_data
         self._port = WEB_SERVER_PORT
         self._api_only_mode = API_ONLY_MODE
         self._shutdown_event = asyncio.Event()
@@ -45,6 +43,10 @@ class WebServer:
             "status": "Starting"
         }
 
+        # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
+        DATA_STORE.spots.add_listener(self._spot_broadcaster.publish)
+        DATA_STORE.alerts.add_listener(self._alert_broadcaster.publish)
+
     def start(self):
         """Start the web server"""
 
@@ -64,20 +66,20 @@ class WebServer:
 
         # API endpoints are always enabled
         api_routes = [
-            (r"/api/v1/spots", APISpotsHandler, {"spots": self._spots, **handler_opts}),
+            (r"/api/v1/spots", APISpotsHandler, {"spots": self._data_store.spots, **handler_opts}),
-            (r"/api/v1/alerts", APIAlertsHandler, {"alerts": self._alerts, **handler_opts}),
+            (r"/api/v1/alerts", APIAlertsHandler, {"alerts": self._data_store.alerts, **handler_opts}),
             (r"/api/v1/spots/stream", APISpotsStreamHandler,
-             {"sse_spot_queues": self._sse_spot_queues, **handler_opts}),
+             {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts}),
             (r"/api/v1/alerts/stream", APIAlertsStreamHandler,
-             {"sse_alert_queues": self._sse_alert_queues, **handler_opts}),
+             {"sse_alert_broadcaster": self._alert_broadcaster, **handler_opts}),
-            (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._solar_conditions, **handler_opts}),
+            (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._data_store.solar, **handler_opts}),
-            (r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._spots, **handler_opts}),
+            (r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._data_store.spots, **handler_opts}),
-            (r"/api/v1/options", APIOptionsHandler, {"status_data": self._status_data, **handler_opts}),
+            (r"/api/v1/options", APIOptionsHandler, {"status_data": self._data_store.status_data, **handler_opts}),
-            (r"/api/v1/status", APIStatusHandler, {"status_data": self._status_data, **handler_opts}),
+            (r"/api/v1/status", APIStatusHandler, {"status_data": self._data_store.status_data, **handler_opts}),
             (r"/api/v1/lookup/call", APILookupCallHandler, {**handler_opts}),
             (r"/api/v1/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
             (r"/api/v1/lookup/grid", APILookupGridHandler, {**handler_opts}),
-            (r"/api/v1/spot", APISpotHandler, {"spots": self._spots, **handler_opts}),
+            (r"/api/v1/spot", APISpotHandler, {"spots": self._data_store.spots, **handler_opts}),
         ]
 
         # If in API-only mode, serve a basic homepage; in normal mode, serve the usual UI routes
@@ -121,55 +123,6 @@ class WebServer:
         logging.info("You can access your copy of Spothole at " + BASE_URL)
         await self._shutdown_event.wait()
 
-    def notify_new_spot(self, spot):
-        """Internal method called when a new spot is added to the system. This is used to ping any SSE clients that are
-        awaiting a server-sent message with new spots."""
-
-        for queue in self._sse_spot_queues:
-            try:
-                queue.put(spot)
-            except:
-                # Cleanup thread was probably deleting the queue, that's fine
-                pass
-        pass
-
-    def notify_new_alert(self, alert):
-        """Internal method called when a new alert is added to the system. This is used to ping any SSE clients that are
-        awaiting a server-sent message with new spots."""
-
-        for queue in self._sse_alert_queues:
-            try:
-                queue.put(alert)
-            except:
-                # Cleanup thread was probably deleting the queue, that's fine
-                pass
-        pass
-
-    def clean_up_sse_queues(self):
-        """Clean up any SSE queues that are growing too large; probably their client disconnected and we didn't catch it
-        properly for some reason."""
-
-        for q in self._sse_spot_queues:
-            try:
-                if q.full():
-                    logging.warning(
-                        "A full SSE spot queue was found, presumably because the client disconnected strangely. It has been removed.")
-                    self._sse_spot_queues.remove(q)
-                    empty_queue(q)
-            except:
-                # Probably got deleted already on another thread
-                pass
-        for q in self._sse_alert_queues:
-            try:
-                if q.full():
-                    logging.warning(
-                        "A full SSE alert queue was found, presumably because the client disconnected strangely. It has been removed.")
-                    self._sse_alert_queues.remove(q)
-                    empty_queue(q)
-            except:
-                # Probably got deleted already on another thread
-                pass
-        pass
 
 def request_log(handler):
     """Custom log function to provide more data about requests when enabled, and to provide the ability to turn off

sigrefdataproviders/sig_ref_data_provider.py

@@ -0,0 +1,28 @@
+from datetime import datetime
+
+import pytz
+
+
+class SIGRefDataProvider:
+    """Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data."""
+
+    def __init__(self, name, provider_config):
+        """Constructor"""
+
+        self.name = name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")

solarconditionsproviders/giroionosonde.py

@@ -35,6 +35,18 @@ class GIROIonosonde(SolarConditionsProvider):
         self._thread = None
         self._stop_event = Event()
 
+        # Pre-populate ionosonde_data with known station names for stations not already present,
+        # so the station dropdown is available before the first poll. Does not overwrite existing
+        # entries so KC2G cache data is preserved.
+        existing = self._solar_conditions.ionosonde_data or {}
+        new_entries = {
+            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
+                        "luf": None, "band_states": None}
+            for s in self._stations if s["ursi"] not in existing
+        }
+        if new_entries:
+            self.update_data({"ionosonde_data": {**existing, **new_entries}})
+
     @staticmethod
     def _load_stations():
         stations = []
@@ -44,21 +56,6 @@ class GIROIonosonde(SolarConditionsProvider):
                     stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
         return stations
 
-    def setup(self, solar_conditions, solar_conditions_cache):
-        """Pre-populate ionosonde_data with known station names for stations not already present,
-        so the station dropdown is available before the first poll. Does not overwrite existing
-        entries so KC2G cache data is preserved."""
-
-        super().setup(solar_conditions, solar_conditions_cache)
-        existing = solar_conditions.ionosonde_data or {}
-        new_entries = {
-            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
-                        "luf": None, "band_states": None}
-            for s in self._stations if s["ursi"] not in existing
-        }
-        if new_entries:
-            self.update_data({"ionosonde_data": {**existing, **new_entries}})
-
     def start(self):
         logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
         self._thread = Thread(target=self._run, daemon=True)

solarconditionsproviders/solar_conditions_provider.py

@@ -2,6 +2,8 @@ from datetime import datetime
 
 import pytz
 
+from core.data_store import DATA_STORE
+
 
 class SolarConditionsProvider:
     """Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
@@ -10,18 +12,11 @@ class SolarConditionsProvider:
     def __init__(self, name, provider_config):
         """Constructor"""
 
-        self._solar_conditions_cache = None
         self.name = name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._solar_conditions = None
+        self._solar_conditions = DATA_STORE.solar_conditions
-
-    def setup(self, solar_conditions, solar_conditions_cache):
-        """Set up the provider, giving it the solar conditions object and its backing cache"""
-
-        self._solar_conditions = solar_conditions
-        self._solar_conditions_cache = solar_conditions_cache
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -41,4 +36,3 @@ class SolarConditionsProvider:
                 if hasattr(self._solar_conditions, key):
                     setattr(self._solar_conditions, key, value)
             self._solar_conditions.infer_descriptions()
-            self._solar_conditions_cache['solar_conditions'] = self._solar_conditions

spothole.py

@@ -5,23 +5,15 @@ import os
 import signal
 import sys
 
-from diskcache import Cache
-
-from core.cleanup import CleanupTimer
 from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL
 from core.constants import SOFTWARE_VERSION
+from core.data_store import DATA_STORE
 from core.lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
-from data.solar_conditions import SolarConditions
 from server.webserver import WebServer
 
 # Globals
-spots = Cache('cache/spots_cache')
-alerts = Cache('cache/alerts_cache')
-solar_conditions_cache = Cache('cache/solar_conditions_cache')
-solar_conditions = solar_conditions_cache.get('solar_conditions', SolarConditions())
 web_server = None
-status_data = {}
 spot_providers = []
 alert_providers = []
 solar_condition_providers = []
@@ -46,13 +38,7 @@ def shutdown(_signum=None, _frame=None):
     for scp in solar_condition_providers:
         if scp.enabled:
             scp.stop()
-    if cleanup_timer:
+    DATA_STORE.close()
-        cleanup_timer.stop()
-    if lookup_helper:
-        lookup_helper.stop()
-    spots.close()
-    alerts.close()
-    solar_conditions_cache.close()
     os._exit(0)
 
 
@@ -103,13 +89,12 @@ if __name__ == '__main__':
     lookup_helper.start()
 
     # Set up web server
-    web_server = WebServer(spots=spots, alerts=alerts, solar_conditions=solar_conditions, status_data=status_data)
+    web_server = WebServer()
 
     # Fetch, set up and start spot providers
     for entry in config["spot-providers"]:
         spot_providers.append(get_spot_provider_from_config(entry))
     for p in spot_providers:
-        p.setup(spots=spots, web_server=web_server)
         if p.enabled:
             p.start()
 
@@ -117,7 +102,6 @@ if __name__ == '__main__':
     for entry in config["alert-providers"]:
         alert_providers.append(get_alert_provider_from_config(entry))
     for p in alert_providers:
-        p.setup(alerts=alerts, web_server=web_server)
         if p.enabled:
             p.start()
 
@@ -125,17 +109,11 @@ if __name__ == '__main__':
     for entry in config.get("solar-condition-providers", []):
         solar_condition_providers.append(get_solar_conditions_provider_from_config(entry))
     for p in solar_condition_providers:
-        p.setup(solar_conditions=solar_conditions, solar_conditions_cache=solar_conditions_cache)
         if p.enabled:
             p.start()
 
-    # Set up timer to clear spot list of old data
-    cleanup_timer = CleanupTimer(spots=spots, alerts=alerts, web_server=web_server, cleanup_interval=60)
-    cleanup_timer.start()
-
     # Set up status reporter
-    status_reporter = StatusReporter(status_data=status_data, spots=spots, alerts=alerts, web_server=web_server,
+    status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers,
-                                     cleanup_timer=cleanup_timer, spot_providers=spot_providers,
                                      alert_providers=alert_providers,
                                      solar_condition_providers=solar_condition_providers, run_interval=5)
     status_reporter.start()

spotproviders/gma.py

@@ -3,8 +3,8 @@ from datetime import datetime
 
 import pytz
 
-from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import HTTP_HEADERS
+from core.url_data_cache import URL_DATA_CACHE
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -56,8 +56,8 @@ class GMA(HTTPSpotProvider):
                 # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
                 if "REF" in source_spot:
                     try:
-                        ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
+                        ref_response = URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
-                                                                      headers=HTTP_HEADERS)
+                                                          headers=HTTP_HEADERS)
                         # Sometimes this is blank even if it's a 200 response, so handle that
                         if ref_response.ok and ref_response.text is not None and ref_response.text != "":
                             ref_info = ref_response.json()

spotproviders/spot_provider.py

@@ -2,7 +2,7 @@ from datetime import datetime
 
 import pytz
 
-from core.config import MAX_SPOT_AGE
+from core.data_store import DATA_STORE
 
 
 class SpotProvider:
@@ -16,14 +16,7 @@ class SpotProvider:
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._spots = None
+        self._spots = DATA_STORE.spots
-        self._web_server = None
-
-    def setup(self, spots, web_server):
-        """Set up the provider, e.g. giving it the spot list to work from"""
-
-        self._spots = spots
-        self._web_server = web_server
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -59,10 +52,7 @@ class SpotProvider:
 
     def _add_spot(self, spot):
         if not spot.expired():
-            self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
+            self._spots.set(spot.id, spot)
-            # Ping the web server in case we have any SSE connections that need to see this immediately
-            if self._web_server:
-                self._web_server.notify_new_spot(spot)
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""