Fix a bug where solar conditions and status data were not properly persisted to disk between restarts. Release v2.0.5

.idea/modules.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ProjectModuleManager">
     <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/Spothole.iml" filepath="$PROJECT_DIR$/.idea/Spothole.iml" />
+      <module fileurl="file://$PROJECT_DIR$/.idea/spothole.iml" filepath="$PROJECT_DIR$/.idea/spothole.iml" />
     </modules>
   </component>
 </project>

.idea/runConfigurations/Run.xml

@@ -1,6 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="Run" type="PythonConfigurationType" factoryName="Python">
-    <module name="Spothole" />
+    <module name="spothole" />
     <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />

config-example.yml

@@ -68,7 +68,7 @@ spot_providers:
   - class: "DXCluster"
     # Clusters have a "name" property in case you want to say which cluster node the data is from, or connect to
     # several clusters and name them separately. The name will be used as the "source" field for the spots received.
-    name: "HRD Cluster"
+    name: "Cluster"
     enabled: true
     host: "hrd.wa9pie.net"
     port: 8000
@@ -84,7 +84,7 @@ spot_providers:
     allow_rbn_spots: false
 
   - class: "DXCluster"
-    name: "W3LPL Cluster"
+    name: "Cluster"
     enabled: false
     host: "w3lpl.net"
     port: 7373

core/constants.py

@@ -3,7 +3,7 @@ from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "2.0.3"
+SOFTWARE_VERSION = "2.0.5"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"}
@@ -54,7 +54,7 @@ SIGS = [
         ref_regex=r"[A-Z]{2}\-\d{3}",
     ),
     SIG(
-        name="GMA Island",
+        name="GMA Islands",
         comment_names=[],
         description="Global Mountain Activity - Islands",
         ref_regex=r"(([A-Z]{2}\-\d{3})|([A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}))",

core/data_store.py

@@ -6,6 +6,7 @@ import diskcache
 
 from core.config import MAX_ALERT_AGE, MAX_SPOT_AGE
 from core.live_data_cache import LiveDataCache
+from core.single_object_data_cache import SingleObjectDataCache
 from data.solar_conditions import SolarConditions
 
 logger = logging.getLogger(__name__)
@@ -34,10 +35,8 @@ class DataStore:
         self.dxcc_data = None
         self.dxcc_lookup_by_call_regex = []
         self.sigrefs = None
-        self.status_data = {}
-        self._status = None
-        self.solar_conditions = {}
-        self._solar = None
+        self.status = None
+        self.solar_conditions = None
         # ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
         # caches, they can just be straight objects
         self.cq_zone_data = None
@@ -46,16 +45,11 @@ class DataStore:
     def setup(self):
         Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
 
-        # Standard disk cache for solar data and status data, but each cache contains only a single object which we
-        # expose to the wider application
-        self._solar = diskcache.Cache(f"{CACHE_DIR}solar")
-        if "solar_conditions" not in self._solar:
-            self._solar.add("solar_conditions", SolarConditions())
-        self.solar_conditions = self._solar.get("solar_conditions")
-        self._status = diskcache.Cache(f"{CACHE_DIR}status")
-        if "status_data" not in self._status:
-            self._status.add("status_data", {})
-        self.status_data = self._status.get("status_data")
+        # For solar data and status data, we use a wrapper around disk cache where each cache contains only a single
+        # object exposed to the wider application, and provides a store() method for callers to notify diskcache that
+        # the object has changed and needs to be re-cached.
+        self.solar_conditions = SingleObjectDataCache(f"{CACHE_DIR}solar", SolarConditions())
+        self.status = SingleObjectDataCache(f"{CACHE_DIR}status", {})
 
         # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
         # a regular basis but there's no need for a TTL since old data is better than no data.
@@ -116,13 +110,10 @@ class DataStore:
             self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
 
     def close(self):
-        self.spots.save_snapshot()
-        self.alerts.save_snapshot()
-
         self.spots.close()
         self.alerts.close()
-        self._solar.close()
-        self._status.close()
+        self.solar.close()
+        self.status.close()
         self.dxcc_data.close()
         self.sigrefs.close()
         self.callsign_data_countryfiles.close()

core/single_object_data_cache.py

@@ -0,0 +1,39 @@
+import logging
+import threading
+
+import diskcache
+
+logger = logging.getLogger(__name__)
+
+
+class SingleObjectDataCache:
+    """Cache for status and solar conditions. This uses DiskCache, but unlike the standard DiskCache users like SIG and
+    callsign lookup handlers, status and solar conditions are persisted as a single object. If we just load the object
+    from DiskCache and modify it, DiskCache doesn't know that it's been updated and needs re-caching, so we provide a
+    store() method that any functions updating the object can call afterwards."""
+
+    def __init__(self, cache_dir, object_if_empty):
+        """Initialize a SingleObjectDataCache. Provide the directory to load the cache from and save it to. If the cache
+        is empty, the provided object_if_empty parameter will be used to initialise it."""
+
+        self._lock = threading.Lock()
+        self._cache = diskcache.Cache(cache_dir)
+        # This cache stores a single object, doesn't matter what it's called so "object" will do
+        if "object" not in self._cache:
+            self._cache.add("object", object_if_empty)
+        self._obj = self._cache.get("object")
+
+    def get(self):
+        """Get the data object. This can then be manipulated as necessary across multiple threads. Any function
+        modifying the object must remember to call store() afterwards."""
+        return self._obj
+
+    def store(self):
+        """Store the updated object in the cache. Any function modifying the object must remember to call this
+        afterwards."""
+        with self._lock:
+            self._cache.set("object", self._obj)
+
+    def close(self):
+        self.store()
+        self._cache.close()

core/status_reporter.py

@@ -25,8 +25,9 @@ class StatusReporter:
         self._stop_event = Event()
         self._startup_time = datetime.now(pytz.UTC)
 
-        DATA_STORE.status_data["software_version"] = SOFTWARE_VERSION
-        DATA_STORE.status_data["server_owner_callsign"] = SERVER_OWNER_CALLSIGN
+        DATA_STORE.status.get()["software_version"] = SOFTWARE_VERSION
+        DATA_STORE.status.get()["server_owner_callsign"] = SERVER_OWNER_CALLSIGN
+        DATA_STORE.status.store()
 
     def start(self):
         """Start the reporter thread"""
@@ -50,11 +51,11 @@ class StatusReporter:
     def _report(self):
         """Write status information"""
 
-        DATA_STORE.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
-        DATA_STORE.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
-        DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
-        DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
-        DATA_STORE.status_data["spot_providers"] = [
+        DATA_STORE.status.get()["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
+        DATA_STORE.status.get()["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
+        DATA_STORE.status.get()["num_spots"] = len(DATA_STORE.spots.values())
+        DATA_STORE.status.get()["num_alerts"] = len(DATA_STORE.alerts.values())
+        DATA_STORE.status.get()["spot_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -69,7 +70,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.spot_providers
         ]
-        DATA_STORE.status_data["alert_providers"] = [
+        DATA_STORE.status.get()["alert_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -80,7 +81,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.alert_providers
         ]
-        DATA_STORE.status_data["solar_condition_providers"] = [
+        DATA_STORE.status.get()["solar_condition_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -91,7 +92,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.solar_condition_providers
         ]
-        DATA_STORE.status_data["static_data_providers"] = [
+        DATA_STORE.status.get()["static_data_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -102,7 +103,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.static_data_providers
         ]
-        DATA_STORE.status_data["sig_ref_data_providers"] = [
+        DATA_STORE.status.get()["sig_ref_data_providers"] = [
             {
                 "sig_name": p.sig_name,
                 "enabled": p.enabled,
@@ -114,7 +115,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.sig_ref_data_providers
         ]
-        DATA_STORE.status_data["callsign_data_providers"] = [
+        DATA_STORE.status.get()["callsign_data_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -126,13 +127,13 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.callsign_data_providers
         ]
-        DATA_STORE.status_data["cleanup"] = {
+        DATA_STORE.status.get()["cleanup"] = {
             "status": CLEANUP_TIMER.status,
             "last_ran": CLEANUP_TIMER.last_cleanup_time.replace(tzinfo=pytz.UTC).timestamp()
             if CLEANUP_TIMER.last_cleanup_time
             else 0,
         }
-        DATA_STORE.status_data["webserver"] = {
+        DATA_STORE.status.get()["webserver"] = {
             "status": WEB_SERVER.web_server_metrics["status"],
             "last_api_access": WEB_SERVER.web_server_metrics["last_api_access_time"]
             .replace(tzinfo=pytz.UTC)
@@ -147,6 +148,7 @@ class StatusReporter:
             else 0,
             "page_access_count": WEB_SERVER.web_server_metrics["page_access_counter"],
         }
+        DATA_STORE.status.store()
 
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)

providers/solarconditions/solar_conditions_provider.py

@@ -16,7 +16,7 @@ class SolarConditionsProvider:
         self.enabled = provider_config.get("enabled", True)
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._solar_conditions = DATA_STORE.solar_conditions
+        self._solar_conditions = DATA_STORE.solar_conditions.get()
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -36,3 +36,4 @@ class SolarConditionsProvider:
                 if hasattr(self._solar_conditions, key):
                     setattr(self._solar_conditions, key, value)
             self._solar_conditions.infer_descriptions()
+            DATA_STORE.solar_conditions.store()

providers/spot/gma.py

@@ -118,8 +118,8 @@ class GMA(HTTPSpotProvider):
                                         spot.sig_refs[0].sig = "IOTA"
                                         spot.sig = "IOTA"
                                     case "GMA Island":
-                                        spot.sig_refs[0].sig = "GMA Island"
-                                        spot.sig = "GMA Island"
+                                        spot.sig_refs[0].sig = "GMA Islands"
+                                        spot.sig = "GMA Islands"
                                     case "Lighthouse (ILLW)":
                                         spot.sig_refs[0].sig = "ILLW"
                                         spot.sig = "ILLW"

pyproject.toml

@@ -1,8 +1,8 @@
 [project]
-name = "Spothole"
-version = "2.0.3"
+name = "spothole"
+version = "2.0.5"
 authors = [
-  { name="Ian Renton", email="ian@ianrenton.com" },
+    { name = "Ian Renton", email = "ian@ianrenton.com" },
 ]
 description = "Spothole is a utility to aggregate \"spots\" from amateur radio DX clusters and xOTA spotting sites, and provide an open JSON API as well as a website to browse the data."
 readme = "README.md"

server/webserver.py

@@ -111,7 +111,7 @@ class WebServer:
             (
                 r"/api/v2/solar",
                 APISolarConditionsHandler,
-                {"solar_conditions": self._data_store.solar_conditions, **handler_opts},
+                {"solar_conditions": self._data_store.solar_conditions.get(), **handler_opts},
             ),
             (
                 r"/api/v2/dxstats",
@@ -121,12 +121,12 @@ class WebServer:
             (
                 r"/api/v2/options",
                 APIOptionsHandler,
-                {"status_data": self._data_store.status_data, **handler_opts},
+                {"status_data": self._data_store.status.get(), **handler_opts},
             ),
             (
                 r"/api/v2/status",
                 APIStatusHandler,
-                {"status_data": self._data_store.status_data, **handler_opts},
+                {"status_data": self._data_store.status.get(), **handler_opts},
             ),
             (r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}),
             (r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),

static/js/spotsbandsandmap.js

@@ -40,7 +40,7 @@ function setHamHFBandToggles() {
 
 // Generate SIGs filter card. This one is also a special case.
 function generateSIGsMultiToggleFilterCard(sig_options) {
-    const $grid = $('<div class="row row-cols-2 row-cols-xxl-3 g-1 mb-1">');
+    const $grid = $('<div class="row row-cols-2 g-1 mb-1">');
     sig_options.forEach(o => {
         const domSafeName = o["name"].replace(/^[^A-Za-z0-9]+|[^\w]+/gi, "");
         $grid.append(`<div class="col"><div class="form-check"><input type="checkbox" class="form-check-input filter-button-sig storeable-checkbox" id="filter-button-sig-${domSafeName}" value="${o['name']}" autocomplete="off" onClick="filtersUpdated()" checked><label class="form-check-label" id="filter-button-label-sig-${domSafeName}" for="filter-button-sig-${domSafeName}" title="${o['description']}"><i class="fa-solid ${sigToIcon(o['name'], 'fa-tower-cell')}"></i> ${o['name']}</label></div></div>`);

static/js/ui-ham.js

@@ -345,7 +345,7 @@ const SIG_ICONS = {
     "SOTA": "fa-mountain-sun",
     "WWFF": "fa-seedling",
     "GMA": "fa-person-hiking",
-    "GMA Island": "fa-person-hiking",
+    "GMA Islands": "fa-person-hiking",
     "WWBOTA": "fa-radiation",
     "HEMA": "fa-mound",
     "IOTA": "fa-book-atlas",
@@ -373,7 +373,7 @@ const SIG_NAMES = {
     "SOTA": "Summits on the Air",
     "WWFF": "Worldwide Flora & Fauna",
     "GMA": "Global Mountain Activity",
-    "GMA Island": "Global Mountain Activity - Islands",
+    "GMA Islands": "Global Mountain Activity - Islands",
     "WWBOTA": "Bunkers on the Air",
     "HEMA": "Humps Excluding Marilyns Award",
     "IOTA": "Islands on the Air",

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1787950046"></script>
+<script src="/static/js/add-spot.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -84,7 +84,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1787950046"></script>
+<script src="/static/js/alerts.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -76,8 +76,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1787950046"></script>
-<script src="/static/js/bands.js?v=1787950046"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1787991444"></script>
+<script src="/static/js/bands.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1787950046" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1787991444" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -15,10 +15,10 @@
   window.fetchEventSource = fetchEventSource;
 </script>
 
-<script src="/static/js/utils.js?v=1787950046"></script>
-<script src="/static/js/ui-ham.js?v=1787950046"></script>
-<script src="/static/js/geo.js?v=1787950046"></script>
-<script src="/static/js/common.js?v=1787950046"></script>
+<script src="/static/js/utils.js?v=1787991444"></script>
+<script src="/static/js/ui-ham.js?v=1787991444"></script>
+<script src="/static/js/geo.js?v=1787991444"></script>
+<script src="/static/js/common.js?v=1787991444"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1787950046"></script>
+<script src="/static/js/conditions.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -113,8 +113,8 @@
     const CARTODB_API_KEY = "{{ web_ui_options.get('cartodb_api_key', '') }}";
 </script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1787950045"></script>
-<script src="/static/js/map.js?v=1787950045"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1787991444"></script>
+<script src="/static/js/map.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -113,8 +113,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1787950045"></script>
-<script src="/static/js/spots.js?v=1787950045"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1787991444"></script>
+<script src="/static/js/spots.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -86,7 +86,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1787950046"></script>
+<script src="/static/js/status.js?v=1787991444"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");