Refactor of caching & data storage part 1

core/cache_utils.py

@@ -1,27 +0,0 @@
-import threading
-from datetime import timedelta
-
-from requests_cache import CachedSession
-
-# Cache for "semi-static" data such as the locations of parks, CSVs of reference lists, etc.
-# This has an expiry time of 30 days, so will re-request from the source after that amount
-# of time has passed. This is used throughout Spothole to cache data that does not change
-# rapidly. The ThreadSafeSession construct here protects it against some multithreading
-# contention weirdness we sometimes used to see on startup where the cache was hammered
-# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type
-# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote
-# servers for data they've told us they can't provide.
-_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30),
-                         allowable_codes=(200, 400, 401, 403, 404))
-_lock = threading.Lock()
-
-
-class _ThreadSafeSession:
-    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads."""
-
-    def get(self, *args, **kwargs):
-        with _lock:
-            return _session.get(*args, **kwargs)
-
-
-SEMI_STATIC_URL_DATA_CACHE = _ThreadSafeSession()

core/cleanup.py

@@ -1,73 +0,0 @@
-import logging
-from datetime import datetime
-from threading import Event, Thread
-
-import pytz
-
-
-class CleanupTimer:
-    """Provides a timed cleanup of the spot list."""
-
-    def __init__(self, spots, alerts, web_server, cleanup_interval):
-        """Constructor"""
-
-        self._spots = spots
-        self._alerts = alerts
-        self._web_server = web_server
-        self._cleanup_interval = cleanup_interval
-        self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC)
-        self.status = "Starting"
-        self._thread = None
-        self._stop_event = Event()
-
-    def start(self):
-        """Start the cleanup timer"""
-
-        self._thread = Thread(target=self._run, daemon=True)
-        self._thread.start()
-
-    def stop(self):
-        """Stop any threads and prepare for application shutdown"""
-
-        self._stop_event.set()
-
-    def _run(self):
-        while not self._stop_event.wait(timeout=self._cleanup_interval):
-            self._cleanup()
-
-    def _cleanup(self):
-        """Perform cleanup and reschedule next timer"""
-
-        try:
-            # Perform cleanup via letting the data expire
-            self._spots.expire()
-            self._alerts.expire()
-
-            # Explicitly clean up any spots and alerts that have expired
-            for i in list(self._spots.iterkeys()):
-                try:
-                    spot = self._spots[i]
-                    if spot.expired():
-                        self._spots.delete(i)
-                except KeyError:
-                    # Must have already been deleted, OK with that
-                    pass
-            for i in list(self._alerts.iterkeys()):
-                try:
-                    alert = self._alerts[i]
-                    if alert.expired():
-                        self._alerts.delete(i)
-                except KeyError:
-                    # Must have already been deleted, OK with that
-                    pass
-
-            # Clean up web server SSE spot/alert queues
-            self._web_server.clean_up_sse_queues()
-
-            self.status = "OK"
-            self.last_cleanup_time = datetime.now(pytz.UTC)
-
-        except Exception:
-            self.status = "Error"
-            logging.exception("Exception in Cleanup thread")
-            self._stop_event.wait(timeout=1)

core/data_store.py

@@ -0,0 +1,69 @@
+from pathlib import Path
+
+import diskcache
+
+from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
+from core.live_data_cache import LiveDataCache
+from data.solar_conditions import SolarConditions
+
+
+class DataStore:
+    """Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
+    lookup data using different caching strategies for each."""
+
+    def __init__(self):
+        cache_dir = "./cache"
+        self.MAX_SPOT_COUNT = 10000
+        self.MAX_ALERT_COUNT = 10000
+        self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
+        self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
+
+        Path(cache_dir).mkdir(parents=True, exist_ok=True)
+
+        # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval
+        self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
+                                   snapshot_dir=cache_dir + "/spots",
+                                   snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+        self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
+                                   snapshot_dir=cache_dir + "/alerts",
+                                    snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+
+        # Standard disk cache for solar data and status data, but each cache contains only a single object which we
+        # expose to the wider application
+        self.solar = diskcache.Cache(cache_dir + "/solar")
+        if "solar_conditions" not in self.solar:
+            self.solar.add("solar_conditions", SolarConditions())
+        self.solar_conditions = self.solar.get("solar_conditions")
+        self.status = diskcache.Cache(cache_dir + "/status")
+        if "status_data" not in self.status:
+            self.status.add("status_data", {})
+        self.status_data = self.status.get("status_data")
+
+        # Standard disk caches for SIG ref data. Separate provider threads will repopulate these on a regular basis
+        # but there's no need for a TTL since old data is better than no data.
+        self.sigrefs_wwff = diskcache.Cache(cache_dir + "/sigrefs/wwff")
+        self.sigrefs_siota = diskcache.Cache(cache_dir + "/sigrefs/siota")
+        self.sigrefs_wota = diskcache.Cache(cache_dir + "/sigrefs/wota")
+        self.sigrefs_zlota = diskcache.Cache(cache_dir + "/sigrefs/zlota")
+        self.sigrefs_llota = diskcache.Cache(cache_dir + "/sigrefs/llota")
+        self.sigrefs_dme = diskcache.Cache(cache_dir + "/sigrefs/dme")
+
+        # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
+        # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
+        # we've seen, so we rely on them timing out and this triggering another lookup.
+        self.callsigns = diskcache.Cache(cache_dir + "/callsigns")
+
+    def close(self):
+        self.spots.close()
+        self.alerts.close()
+        self.solar.close()
+        self.status.close()
+        self.sigrefs_wwff.close()
+        self.sigrefs_siota.close()
+        self.sigrefs_wota.close()
+        self.sigrefs_zlota.close()
+        self.sigrefs_llota.close()
+        self.callsigns.close()
+
+# Global object
+DATA_STORE = DataStore()

core/live_data_cache.py

@@ -0,0 +1,72 @@
+import logging
+import threading
+import time
+
+import diskcache
+from cachetools import TTLCache
+
+
+class LiveDataCache:
+    """Cache for spots and alerts. Uses the faster in-memory TTLCache for normal data I/O, including the TTL to enforce
+    maximum lifetime, and adds a separate diskcache to which we can save and load the TTLCache to provide persistence.
+    Also adds thread safety which TTLCache doesn't do."""
+
+    def __init__(self, maxsize, ttl, snapshot_dir, snapshot_interval_sec):
+        self._cache = TTLCache(maxsize=maxsize, ttl=ttl)
+        self._lock = threading.Lock()
+        self._ttl = ttl
+        self._snapshot_dir = snapshot_dir
+        self._disk_cache = diskcache.Cache(str(snapshot_dir))
+        self._load_snapshot()
+        self._start_periodic_snapshot(snapshot_interval_sec)
+
+    def set(self, key, value):
+        with self._lock:
+            self._cache[key] = value
+
+    def get(self, key, default=None):
+        with self._lock:
+            return self._cache.get(key, default)
+
+    def delete(self, key):
+        with self._lock:
+            self._cache.pop(key, None)
+
+    def values(self):
+        with self._lock:
+            return list(self._cache.values())
+
+    def save_snapshot(self):
+        with self._lock:
+            # Store the time with the data so we can avoid loading anything nxt time that's older than TTL
+            data = [(k, v, time.time()) for k, v in self._cache.items()]
+        try:
+            self._disk_cache.set("snapshot", data)
+        except Exception as e:
+            logging.error("Failed to write snapshot to %s", self._snapshot_dir, e, exc_info=True)
+
+    def _load_snapshot(self):
+        data = self._disk_cache.get("snapshot")
+        if not data:
+            return
+
+        now = time.time()
+        with self._lock:
+            for key, value, saved_at in data:
+                # Only restore entries that would still be within TTL
+                if now - saved_at < self._ttl:
+                    self._cache[key] = value
+        logging.info("Loaded snapshot from %s", self._snapshot_dir)
+
+    def _start_periodic_snapshot(self, interval):
+        def loop():
+            while True:
+                time.sleep(interval)
+                self.save_snapshot()
+
+        t = threading.Thread(target=loop, daemon=True, name=f"snapshot-{self._snapshot_dir}")
+        t.start()
+
+    def close(self):
+        self.save_snapshot()
+        self._disk_cache.close()

core/lookup_helper.py

@@ -14,7 +14,7 @@ from pyhamtools.locator import latlong_to_locator
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
-from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
+from core.url_data_cache import URL_DATA_CACHE
 from core.config import config
 from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
     HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
@@ -142,8 +142,8 @@ class LookupHelper:
 
         try:
             logging.info("Downloading Country-files.com cty.plist...")
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
-                                                      headers=HTTP_HEADERS)
+            response = URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
+                                          headers=HTTP_HEADERS)
 
             if response.ok:
                 with open(self._country_files_cty_plist_download_location, "w") as f:
@@ -167,7 +167,7 @@ class LookupHelper:
 
         try:
             logging.info("Downloading dxcc.json...")
-            response = SEMI_STATIC_URL_DATA_CACHE.get(
+            response = URL_DATA_CACHE.get(
                 "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
                 headers=HTTP_HEADERS)
 
@@ -515,7 +515,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = SEMI_STATIC_URL_DATA_CACHE.get(
+                response = URL_DATA_CACHE.get(
                     self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
                     headers=HTTP_HEADERS, timeout=10)
                 if response.ok:
@@ -593,7 +593,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = SEMI_STATIC_URL_DATA_CACHE.get(
+                response = URL_DATA_CACHE.get(
                     self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                         lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
                 if response.ok:

core/sig_utils.py

@@ -4,23 +4,16 @@ import logging
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
-from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import SIGS, HTTP_HEADERS
+from core.data_store import DATA_STORE
 from core.geo_utils import wab_wai_square_to_lat_lon
+from core.url_data_cache import URL_DATA_CACHE
 
 # Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data
 # file in Spothole and load it on startup.
 with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
-    _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
-# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
-# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
-# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
-_WWFF_INDEX_CACHE = {}
-_SIOTA_INDEX_CACHE = {}
-_WOTA_INDEX_CACHE = {}
-_ZLOTA_INDEX_CACHE = {}
-_LLOTA_INDEX_CACHE = {}
-
+    for row in csv.DictReader(_f, delimiter=";"):
+        DATA_STORE.sigrefs_dme.add(row["COD_INE"][:5], row)
 
 def get_ref_regex_for_sig(sig):
     """Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
@@ -53,7 +46,7 @@ def populate_sig_ref_info(sig_ref):
     ref_id = sig_ref.id
     try:
         if sig.upper() == "POTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
+            response = URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if data:
@@ -71,7 +64,7 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "SOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
+            response = URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
@@ -88,7 +81,7 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "WWBOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
+            response = URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
@@ -104,7 +97,7 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
+            response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
@@ -130,15 +123,14 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "WWFF":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
+            response = URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
                                                       headers=HTTP_HEADERS)
             if response.ok:
-                global _WWFF_INDEX_CACHE
-                if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
+                if not bool(DATA_STORE.sigrefs_wwff) or not response.from_cache:
                     # New data from WWFF, update our internal map
-                    _WWFF_INDEX_CACHE = {row["reference"]: row for row in
-                                         csv.DictReader(response.content.decode().splitlines())}
-                row = _WWFF_INDEX_CACHE.get(ref_id)
+                    for row in csv.DictReader(response.content.decode().splitlines()):
+                        DATA_STORE.sigrefs_wwff.add(row["reference"], row)
+                row = DATA_STORE.sigrefs_wwff.get(ref_id)
                 if row:
                     sig_ref.name = row["name"] if "name" in row else None
                     sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
@@ -152,15 +144,14 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "SIOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
+            response = URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
                                                       headers=HTTP_HEADERS)
             if response.ok:
-                global _SIOTA_INDEX_CACHE
-                if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
+                if not bool(DATA_STORE.sigrefs_siota) or not response.from_cache:
                     # New data from SIOTA, update our internal map
-                    _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
-                                          csv.DictReader(response.content.decode().splitlines())}
-                row = _SIOTA_INDEX_CACHE.get(ref_id)
+                    for row in csv.DictReader(response.content.decode().splitlines()):
+                        DATA_STORE.sigrefs_siota.add(row["SILO_CODE"], row)
+                row = DATA_STORE.sigrefs_siota.get(ref_id)
                 if row:
                     sig_ref.name = row["NAME"] if "NAME" in row else None
                     sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
@@ -172,17 +163,16 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "WOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
+            response = URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if data:
-                    global _WOTA_INDEX_CACHE
-                    if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
+                    if not bool(DATA_STORE.sigrefs_wota) or not response.from_cache:
                         # New data from WOTA, update our internal map
-                        _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
-                                             data.get("features", [])}
-                    feature = _WOTA_INDEX_CACHE.get(ref_id)
+                        for feature in data.get("features", []):
+                            DATA_STORE.sigrefs_wota.add(feature["properties"]["wotaId"], feature)
+                    feature = DATA_STORE.sigrefs_wota.get(ref_id)
                     if feature:
                         sig_ref.name = feature["properties"]["title"]
                         # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -200,24 +190,24 @@ def populate_sig_ref_info(sig_ref):
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
 
         elif sig.upper() == "ZLOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
+            response = URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if isinstance(data, list):
-                    global _ZLOTA_INDEX_CACHE
-                    if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
+                    if not bool(DATA_STORE.sigrefs_zlota) or not response.from_cache:
                         # New data from ZLOTA, update our internal map
-                        _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
-                    asset = _ZLOTA_INDEX_CACHE.get(ref_id)
-                    if asset:
-                        sig_ref.name = asset["name"]
+                        for ref in data:
+                            DATA_STORE.sigrefs_zlota.add(ref["code"], ref)
+                    ref = DATA_STORE.sigrefs_zlota.get(ref_id)
+                    if ref:
+                        sig_ref.name = ref["name"]
                         sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
                         try:
-                            sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
+                            sig_ref.grid = latlong_to_locator(ref["y"], ref["x"], 6)
                         except:
                             logging.debug("Invalid lat/lon received for reference")
-                            sig_ref.latitude = asset["y"]
-                            sig_ref.longitude = asset["x"]
+                            sig_ref.latitude = ref["y"]
+                            sig_ref.longitude = ref["x"]
                 elif not response.from_cache:
                     logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
             elif not response.from_cache:
@@ -229,16 +219,16 @@ def populate_sig_ref_info(sig_ref):
             sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
 
         elif sig.upper() == "LLOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
+            response = URL_DATA_CACHE.get("https://llota.app/api/public/references",
                                                       headers=HTTP_HEADERS)
             if response.ok:
                 data = response.json()
                 if isinstance(data, list):
-                    global _LLOTA_INDEX_CACHE
-                    if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
+                    if not bool(DATA_STORE.sigrefs_llota) or not response.from_cache:
                         # New data from LLOTA, update our internal map
-                        _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
-                    ref = _LLOTA_INDEX_CACHE.get(ref_id)
+                        for ref in data:
+                            DATA_STORE.sigrefs_llota.add(ref["reference_code"], ref)
+                    ref = DATA_STORE.sigrefs_llota.get(ref_id)
                     if ref:
                         sig_ref.name = str(ref["name"])
                         sig_ref.url = "https://llota.app/list/ref/" + ref_id
@@ -280,7 +270,7 @@ def populate_sig_ref_info(sig_ref):
 
         elif sig.upper() == "DME":
             # Zero-pad to 5 digits to match our source data
-            row = _DME_INDEX.get(ref_id.zfill(5))
+            row = DATA_STORE.sigrefs_dme.get(ref_id.zfill(5))
             if row:
                 sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
                 sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(

core/status_reporter.py

@@ -13,25 +13,21 @@ from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alert
 class StatusReporter:
     """Provides a timed update of the application's status data."""
 
-    def __init__(self, status_data, run_interval, web_server, cleanup_timer, spots, spot_providers, alerts,
-                 alert_providers, solar_condition_providers):
+    def __init__(self, data_store, run_interval, web_server,spot_providers, alert_providers, solar_condition_providers):
         """Constructor"""
 
-        self._status_data = status_data
+        self._data_store = data_store
         self._run_interval = run_interval
         self._web_server = web_server
-        self._cleanup_timer = cleanup_timer
-        self._spots = spots
         self._spot_providers = spot_providers
-        self._alerts = alerts
         self._alert_providers = alert_providers
         self._solar_condition_providers = solar_condition_providers
         self._thread = None
         self._stop_event = Event()
         self._startup_time = datetime.now(pytz.UTC)
 
-        self._status_data["software-version"] = SOFTWARE_VERSION
-        self._status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
+        self._data_store.status_data["software-version"] = SOFTWARE_VERSION
+        self._data_store.status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
 
     def start(self):
         """Start the reporter thread"""
@@ -55,44 +51,41 @@ class StatusReporter:
     def _report(self):
         """Write status information"""
 
-        self._status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
-        self._status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
-        self._status_data["num_spots"] = len(self._spots)
-        self._status_data["num_alerts"] = len(self._alerts)
-        self._status_data["spot_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
-                           "last_spot": p.last_spot_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
-                self._spot_providers))
-        self._status_data["alert_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-                self._alert_providers))
-        self._status_data["solar_condition_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-                self._solar_condition_providers))
-        self._status_data["cleanup"] = {"status": self._cleanup_timer.status,
-                                        "last_ran": self._cleanup_timer.last_cleanup_time.replace(
-                                            tzinfo=pytz.UTC).timestamp() if self._cleanup_timer.last_cleanup_time else 0}
-        self._status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
-                                          "last_api_access": self._web_server.web_server_metrics[
-                                              "last_api_access_time"].replace(
-                                              tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
-                                              "last_api_access_time"] else 0,
-                                          "api_access_count": self._web_server.web_server_metrics["api_access_counter"],
-                                          "last_page_access": self._web_server.web_server_metrics[
-                                              "last_page_access_time"].replace(
-                                              tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
-                                              "last_page_access_time"] else 0,
-                                          "page_access_count": self._web_server.web_server_metrics[
-                                              "page_access_counter"]}
+        self._data_store.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
+        self._data_store.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
+        self._data_store.status_data["num_spots"] = len(self._data_store.spots.values())
+        self._data_store.status_data["num_alerts"] = len(self._data_store.alerts.values())
+        self._data_store.status_data["spot_providers"] = list(
+        map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+                       "last_updated": p.last_update_time.replace(
+                           tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
+                       "last_spot": p.last_spot_time.replace(
+                           tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
+            self._spot_providers))
+        self._data_store.status_data["alert_providers"] = list(
+        map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+                       "last_updated": p.last_update_time.replace(
+                           tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+            self._alert_providers))
+        self._data_store.status_data["solar_condition_providers"] = list(
+        map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+                       "last_updated": p.last_update_time.replace(
+                           tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+            self._solar_condition_providers))
+        self._data_store.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
+                                      "last_api_access": self._web_server.web_server_metrics[
+                                          "last_api_access_time"].replace(
+                                          tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
+                                          "last_api_access_time"] else 0,
+                                      "api_access_count": self._web_server.web_server_metrics["api_access_counter"],
+                                      "last_page_access": self._web_server.web_server_metrics[
+                                          "last_page_access_time"].replace(
+                                          tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
+                                          "last_page_access_time"] else 0,
+                                      "page_access_count": self._web_server.web_server_metrics[
+                                          "page_access_counter"]}
 
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)
-        spots_gauge.set(len(self._spots))
-        alerts_gauge.set(len(self._alerts))
+        spots_gauge.set(len(self._data_store.spots.values()))
+        alerts_gauge.set(len(self._data_store.alerts.values()))

core/url_data_cache.py

@@ -0,0 +1,23 @@
+import threading
+from datetime import timedelta
+
+from requests_cache import CachedSession
+
+# Cache for "semi-static" data retrieved from a URL. This is a layet of caching in addition to the normal caching of
+# spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints frequently when
+# e.g. restarting Spothole many times during testing.
+_session = CachedSession("cache/semi_static_urls", expire_after=timedelta(days=1),
+                         allowable_codes=(200, 400, 401, 403, 404))
+_lock = threading.Lock()
+
+
+class _ThreadSafeSession:
+    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads. This allows a single object
+    to be used freely across the application."""
+
+    def get(self, *args, **kwargs):
+        with _lock:
+            return _session.get(*args, **kwargs)
+
+# Global object
+URL_DATA_CACHE = _ThreadSafeSession()