Update map and bands pages to use SSE endpoint after initial load. Closes #43.

.idea/spothole.iml

@@ -2,6 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
       <excludeFolder url="file://$MODULE_DIR$/.venv" />
       <excludeFolder url="file://$MODULE_DIR$/webassets/vendor" />
     </content>

README.md

@@ -95,9 +95,9 @@ Various approaches exist to writing your own client, but in general:
   `https://spothole.app/api/v1/spots` once every few minutes. Apply filters if necessary.
 * Call the "options" API to get an idea of which bands, modes etc. the server knows about. You might want to do that
   first before calling the spots/alerts APIs, to allow you to populate your filters correctly.
-* Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "map" and "bands"
+* Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "alerts"/"upcoming"
-  pages simply query the main spot API on a timer, whereas the main/spots page combines this approach with using the
+  page simply query the main spot API on a timer, whereas the spots, map and bands pages combine this approach with
-  Server-Sent Events (SSE) endpoint to update live.
+  using the Server-Sent Events (SSE) endpoint to update live.
 * Let me know if you get stuck, I'm happy to help.
 
 Please don't hammer the API with an unnecessarily high request rate. For example, Spothole only queries the POTA API

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/js/add-spot.js?v=1782411905"></script>
+<script src="/js/add-spot.js?v=1782461355"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -75,7 +75,7 @@
 
 </div>
 
-<script src="/js/alerts.js?v=1782411905"></script>
+<script src="/js/alerts.js?v=1782461355"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -3,9 +3,7 @@
 
 <div class="mt-3">
     <div id="settingsButtonRow" class="row mb-3">
-        <div class="col-auto me-auto pt-3">
+        <div class="col-auto me-auto pt-3"></div>
-            {% module Template("widgets/refresh-timer.html", web_ui_options=web_ui_options) %}
-        </div>
         <div class="col-auto">
             <div class="d-inline-flex gap-1">
                 {% module Template("widgets/filters-display-data-buttons.html", web_ui_options=web_ui_options) %}
@@ -77,8 +75,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/js/spotsbandsandmap.js?v=1782411905"></script>
+<script src="/js/spotsbandsandmap.js?v=1782461355"></script>
-<script src="/js/bands.js?v=1782411905"></script>
+<script src="/js/bands.js?v=1782461355"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/css/style.css?v=1782411905" type="text/css">
+<link rel="stylesheet" href="/css/style.css?v=1782461355" type="text/css">
 <link href="/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
 <script src="/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
 <script src="/vendor/js/tinycolor2-1.6.0.min.js"></script>
 
-<script src="/js/utils.js?v=1782411905"></script>
+<script src="/js/utils.js?v=1782461355"></script>
-<script src="/js/ui-ham.js?v=1782411905"></script>
+<script src="/js/ui-ham.js?v=1782461355"></script>
-<script src="/js/geo.js?v=1782411905"></script>
+<script src="/js/geo.js?v=1782461355"></script>
-<script src="/js/common.js?v=1782411905"></script>
+<script src="/js/common.js?v=1782461355"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/js/conditions.js?v=1782411905"></script>
+<script src="/js/conditions.js?v=1782461355"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -95,8 +95,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/js/spotsbandsandmap.js?v=1782411905"></script>
+<script src="/js/spotsbandsandmap.js?v=1782461355"></script>
-<script src="/js/map.js?v=1782411905"></script>
+<script src="/js/map.js?v=1782461355"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -116,8 +116,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/js/spotsbandsandmap.js?v=1782411905"></script>
+<script src="/js/spotsbandsandmap.js?v=1782461355"></script>
-<script src="/js/spots.js?v=1782411905"></script>
+<script src="/js/spots.js?v=1782461355"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -59,7 +59,7 @@
     </div>
 </div>
 
-<script src="/js/status.js?v=1782411905"></script>
+<script src="/js/status.js?v=1782461355"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");

webassets/js/alerts.js

@@ -1,5 +1,7 @@
 // How often to query the server?
 const REFRESH_INTERVAL_SEC = 60 * 10;
+// Last time the alerts list was updated on display.
+let lastUpdateTime;
 
 // Storage for the alert data that the server gives us.
 let alerts = [];
@@ -309,6 +311,27 @@ function filtersUpdated() {
     saveSettings();
 }
 
+// Update the refresh timing display
+function updateRefreshDisplay() {
+    if (lastUpdateTime != null) {
+        let secSinceUpdate = moment.duration(moment().diff(lastUpdateTime)).asSeconds();
+        let count = REFRESH_INTERVAL_SEC;
+        let updatingString = "Updating..."
+        if (secSinceUpdate < REFRESH_INTERVAL_SEC) {
+            count = REFRESH_INTERVAL_SEC - secSinceUpdate;
+            let number;
+            if (count <= 60) {
+                number = count.toFixed(0);
+                updatingString = "<span class='nowrap'>Updating in " + number + " second" + (number !== "1" ? "s" : "") + ".</span>";
+            } else {
+                number = Math.round(count / 60.0).toFixed(0);
+                updatingString = "<span class='nowrap'>Updating in " + number + " minute" + (number !== "1" ? "s" : "") + ".</span>";
+            }
+        }
+        $("#timing-container").html("Last updated at " + lastUpdateTime.format('HH:mm') + " UTC. " + updatingString);
+    }
+}
+
 // Startup
 $(document).ready(function () {
     // Call loadOptions(), this will then trigger loading alerts and setting up timers.

webassets/js/bands.js

@@ -1,5 +1,9 @@
-// How often to query the server?
+// SSE connection for live spot updates
-const REFRESH_INTERVAL_SEC = 60;
+let evtSource;
+let restartSSEOnErrorTimeoutId;
+// Debounce timer so rapid SSE bursts only trigger one updateBands() call, as these could trigger a flash of the user
+// visible content
+let updateBandsDebounceId;
 
 // A couple of constants that must match what's in CSS. We need to know them before the content actually renders, so we
 // can't just ask the elements themselves for their dimensions.
@@ -12,19 +16,62 @@ BAND_COLUMN_SPOT_DIV_HEIGHT_PX = BAND_COLUMN_FONT_SIZE * 1.6;
 
 // Load spots and populate the bands display.
 function loadSpots() {
-    $.getJSON('/api/v1/spots' + buildQueryString(false), function (jsonData) {
+    // Close any existing SSE connection before fetching fresh data
-        // Store last updated time
+    if (evtSource != null) {
-        lastUpdateTime = moment.utc();
+        evtSource.close();
-        updateRefreshDisplay();
+    }
+    $.getJSON('/api/v1/spots' + buildQueryString(), function (jsonData) {
         // Store data
         spots = jsonData;
         // Update bands display
         updateBands();
+        // Start the ongoing SSE connection
+        startSSEConnection();
     });
 }
 
-// Build a query string for the API, based on the filters that the user has selected.
+// Start an SSE connection to receive new spots as they arrive.
-function buildQueryString(includeCredentials) {
+function startSSEConnection() {
+    if (evtSource != null) {
+        evtSource.close();
+    }
+    evtSource = new EventSource('/api/v1/spots/stream' + buildQueryString());
+
+    evtSource.onmessage = function (event) {
+        const newSpot = JSON.parse(event.data);
+
+        // Replace any existing spot for this callsign
+        spots = spots.filter(s => newSpot["dx_call"] !== s["dx_call"]);
+        spots.unshift(newSpot);
+
+        // Debounce. Wait 500ms after the last message before re-rendering to avoid too many ugly flashes
+        clearTimeout(updateBandsDebounceId);
+        updateBandsDebounceId = setTimeout(updateBands, 500);
+    };
+
+    evtSource.onerror = function () {
+        if (evtSource != null) {
+            evtSource.close();
+        }
+        clearTimeout(restartSSEOnErrorTimeoutId);
+        restartSSEOnErrorTimeoutId = setTimeout(startSSEConnection, 1000);
+    };
+}
+
+// Remove spots from the display that are older than the selected max age.
+function expireOldSpots() {
+    const maxAgeSeconds = parseInt($("#max-spot-age option:selected").val());
+    const cutoff = (Date.now() / 1000) - maxAgeSeconds;
+    const before = spots.length;
+    spots = spots.filter(s => s["time"] && s["time"] >= cutoff);
+    if (spots.length !== before) {
+        updateBands();
+    }
+}
+
+// Build a query string for the API, based on the filters that the user has selected. There's no need for credentials
+// in the bands page's version of this, because nothing QRZ.com/HamQTH can provide will affect the display.
+function buildQueryString() {
     let str = "?";
     ["dx_continent", "de_continent", "mode", "source", "band", "sig"].forEach(fn => {
         if (!allFilterOptionsSelected(fn)) {
@@ -34,9 +81,6 @@ function buildQueryString(includeCredentials) {
     str = str + "max_age=" + $("#max-spot-age option:selected").val();
     // Additional filters for the bands view: No dupes, no QRT
     str = str + "&dedupe=true&allow_qrt=false";
-    if (includeCredentials) {
-        str = str + getCredentialQueryString();
-    }
     return str;
 }
 
@@ -257,22 +301,28 @@ function loadOptions() {
         // Load settings from settings storage now all the controls are available
         loadSettings();
 
-        // Load spots and set up the timer
+        // Load spots and start SSE for live updates
         loadSpots();
-        setInterval(loadSpots, REFRESH_INTERVAL_SEC * 1000);
+        // Set up spot expiry checker
+        setInterval(expireOldSpots, 60 * 1000);
     });
 }
 
-// Method called when any display property is changed to reload the map and persist the display settings.
+// Method called when any display property is changed to reload the bands display and persist settings.
 function displayUpdated() {
-    updateMap();
+    updateBands();
     saveSettings();
 }
 
 // Startup
 $(document).ready(function () {
+    // Close SSE connection cleanly when navigating away
+    window.addEventListener('beforeunload', function () {
+        if (evtSource != null) {
+            evtSource.close();
+        }
+    });
+
     // Call loadOptions(), this will then trigger loading spots and setting up timers.
     loadOptions();
-    // Update the refresh timing display every second
-    setInterval(updateRefreshDisplay, 1000);
 });

webassets/js/common.js

@@ -1,7 +1,5 @@
 // Storage for the options that the server gives us. This will define our filters.
 let options = {};
-// Last time we updated the spots/alerts list on display.
-let lastUpdateTime;
 // Normally load user settings from local storage, unless embedded mode is in use
 let useLocalStorage = true;
 
@@ -145,27 +143,6 @@ function toggleFilterButtons(filterQuery, state) {
     filtersUpdated();
 }
 
-// Update the refresh timing display
-function updateRefreshDisplay() {
-    if (lastUpdateTime != null) {
-        let secSinceUpdate = moment.duration(moment().diff(lastUpdateTime)).asSeconds();
-        let count = REFRESH_INTERVAL_SEC;
-        let updatingString = "Updating..."
-        if (secSinceUpdate < REFRESH_INTERVAL_SEC) {
-            count = REFRESH_INTERVAL_SEC - secSinceUpdate;
-            let number;
-            if (count <= 60) {
-                number = count.toFixed(0);
-                updatingString = "<span class='nowrap'>Updating in " + number + " second" + (number !== "1" ? "s" : "") + ".</span>";
-            } else {
-                number = Math.round(count / 60.0).toFixed(0);
-                updatingString = "<span class='nowrap'>Updating in " + number + " minute" + (number !== "1" ? "s" : "") + ".</span>";
-            }
-        }
-        $("#timing-container").html("Last updated at " + lastUpdateTime.format('HH:mm') + " UTC. " + updatingString);
-    }
-}
-
 // When the "use local time" field is changed, reload the table and save settings
 function timeZoneUpdated() {
     updateTable();

webassets/js/map.js

@@ -1,6 +1,3 @@
-// How often to query the server?
-const REFRESH_INTERVAL_SEC = 60;
-
 // Colours
 const MAIDENHEAD_GRID_COLOR_LIGHT = 'rgba(200, 140, 140, 1.0)';
 const CQ_ZONES_COLOR_LIGHT = 'rgba(140, 200, 140, 1.0)';
@@ -11,6 +8,15 @@ const CQ_ZONES_COLOR_DARK = 'rgba(60, 120, 60, 1.0)';
 const ITU_ZONES_COLOR_DARK = 'rgba(120, 120, 60, 1.0)';
 const WAB_WAI_GRID_COLOR_DARK = 'rgba(60, 60, 120, 1.0)';
 
+// SSE connection for live spot updates
+let evtSource;
+let restartSSEOnErrorTimeoutId;
+// Map dx_call to a pair of marker & geodesic line, so we can de-duplicate spots as they arrive and only show
+// one marker per dx_call. The key here is actually dx_call + SSID if the spot gives us an SSID; this is mostly for
+// if APRS spots are enabled so we can ID a home and mobile station separately rather than our marker oscillating
+// between the two as updates come in.
+let spotMarkers = new Map();
+
 // Map layers
 let backgroundTileLayer;
 let markersLayer;
@@ -28,7 +34,13 @@ let firstLoad = true;
 
 // Load spots and populate the map.
 function loadSpots() {
-    $.getJSON('/api/v1/spots' + buildQueryString(true), function (jsonData) {
+    // Close any existing SSE connection before fetching fresh data
+    if (evtSource != null) {
+        evtSource.close();
+    }
+    // On first fetch, don't include QRZ/HamQTH credentials. This will cause some inaccurate
+    // marker positions but avoids having to wait a minute or more for all the lookups to fire.
+    $.getJSON('/api/v1/spots' + buildQueryString(false), function (jsonData) {
         // Store data
         spots = jsonData;
         // Update map
@@ -36,9 +48,102 @@ function loadSpots() {
         if ($("#showTerminator")[0].checked) {
             terminator.setTime();
         }
+        // Start the ongoing SSE connection
+        startSSEConnection();
     });
 }
 
+// Start the SSE connection to receive new spots as they arrive
+function startSSEConnection() {
+    if (evtSource != null) {
+        evtSource.close();
+    }
+    // SSE is going to fetch only a few spots at a time, so now we include QRZ/HamQTH credentials because the delay won't be significant.
+    evtSource = new EventSource('/api/v1/spots/stream' + buildQueryString(true));
+
+    evtSource.onmessage = function (event) {
+        const newSpot = JSON.parse(event.data);
+        const key = spotKey(newSpot);
+
+        // Remove existing marker/geodesic for this callsign if present
+        removeSpotFromMap(key);
+        spots = spots.filter(s => spotKey(s) !== key);
+
+        // Skip spots with no map coordinates
+        if (newSpot["dx_latitude"] == null || newSpot["dx_longitude"] == null) {
+            return;
+        }
+
+        // Add to data store and map
+        spots.unshift(newSpot);
+        addSpotToMap(newSpot);
+    };
+
+    evtSource.onerror = function () {
+        if (evtSource != null) {
+            evtSource.close();
+        }
+        clearTimeout(restartSSEOnErrorTimeoutId);
+        restartSSEOnErrorTimeoutId = setTimeout(startSSEConnection, 1000);
+    };
+}
+
+// Remove spots from the map that are older than the selected max age.
+function expireOldSpots() {
+    const maxAgeSeconds = parseInt($("#max-spot-age option:selected").val());
+    const cutoff = (Date.now() / 1000) - maxAgeSeconds;
+    spots = spots.filter(function (s) {
+        if (s["time"] && s["time"] < cutoff) {
+            removeSpotFromMap(spotKey(s));
+            return false;
+        }
+        return true;
+    });
+}
+
+// Returns a unique key for a spot based on its callsign and SSID. This is in case the APRS spot source is enabled and
+// we want to display all SSIDs for a given callsign, not just the latest one.
+function spotKey(s) {
+    return s["dx_call"] + (s["dx_ssid"] ? "-" + s["dx_ssid"] : "");
+}
+
+// Add a single spot's marker and geodesic to the map and to spotMarkers
+function addSpotToMap(s) {
+    const m = L.marker([s["dx_latitude"], s["dx_longitude"]], {icon: getIcon(s)});
+    m.bindPopup(getTooltipText(s));
+    markersLayer.addLayer(m);
+    oms.addMarker(m);
+
+    let geodesic = null;
+    if ($("#mapShowGeodesics")[0].checked && s["de_latitude"] != null && s["de_longitude"] != null) {
+        try {
+            geodesic = L.geodesic([[s["de_latitude"], s["de_longitude"]], m.getLatLng()], {
+                color: bandToColor(s['band']),
+                wrap: false,
+                steps: 5
+            });
+            geodesicsLayer.addLayer(geodesic);
+        } catch (e) {
+            // Not sure what causes these but better to continue than to crash out
+        }
+    }
+
+    spotMarkers.set(spotKey(s), {marker: m, geodesic: geodesic});
+}
+
+// Remove a spot's marker and geodesic from the map and from spotMarkers.
+function removeSpotFromMap(key) {
+    const entry = spotMarkers.get(key);
+    if (entry) {
+        markersLayer.removeLayer(entry.marker);
+        oms.removeMarker(entry.marker);
+        if (entry.geodesic) {
+            geodesicsLayer.removeLayer(entry.geodesic);
+        }
+        spotMarkers.delete(key);
+    }
+}
+
 // Build a query string for the API, based on the filters that the user has selected.
 function buildQueryString(includeCredentials) {
     let str = "?";
@@ -62,27 +167,14 @@ function updateMap() {
     markersLayer.clearLayers();
     geodesicsLayer.clearLayers();
     oms.clearMarkers();
+    spotMarkers.clear();
 
-    // Make new markers for all spots that match the filter
+    // Make new markers for all spots
     spots.forEach(function (s) {
-        const m = L.marker([s["dx_latitude"], s["dx_longitude"]], {icon: getIcon(s)});
+        if (s["dx_latitude"] == null || s["dx_longitude"] == null) {
-        m.bindPopup(getTooltipText(s));
+            return;
-        markersLayer.addLayer(m);
-        oms.addMarker(m);
-
-        // Create geodesics if required
-        if ($("#mapShowGeodesics")[0].checked && s["de_latitude"] != null && s["de_longitude"] != null) {
-            try {
-                const geodesic = L.geodesic([[s["de_latitude"], s["de_longitude"]], m.getLatLng()], {
-                    color: bandToColor(s['band']),
-                    wrap: false,
-                    steps: 5
-                });
-                geodesicsLayer.addLayer(geodesic);
-            } catch (e) {
-                // Not sure what causes these but better to continue than to crash out
-            }
         }
+        addSpotToMap(s);
     });
 
     // On first load, zoom to the extent of the markers
@@ -239,9 +331,10 @@ function loadOptions() {
         enableITUZones($("#showITUZones")[0].checked);
         enableWABWAIGrid($("#showWABWAIGrid")[0].checked);
 
-        // Load spots and set up the timer
+        // Load spots and start SSE for live updates
         loadSpots();
-        setInterval(loadSpots, REFRESH_INTERVAL_SEC * 1000);
+        // Set up spot expiry checker
+        setInterval(expireOldSpots, 60 * 1000);
     });
 }
 
@@ -483,6 +576,13 @@ function setUpMap() {
 
 // Startup
 $(document).ready(function () {
+    // Close SSE connection cleanly when navigating away
+    window.addEventListener('beforeunload', function () {
+        if (evtSource != null) {
+            evtSource.close();
+        }
+    });
+
     // Hide the extra things that need to be hidden on this page
     $(".hideonmap").hide();
     // Set up map