Use SSE frontend #3

README.md

@@ -55,12 +55,14 @@ More will be added soon to allow customisation of filters and other display prop
 
 One of the key strengths of Spothole is that the API is well-defined and open to anyone to use. This means you can build your own software that uses data from Spothole.
 
+As well as the main API endpoints to fetch spots and alerts, with various possible query parameters, there are also Server-Sent Events (SSE) API endpoints to receive a live feed, plus various utility lookup endpoints for things like callsign and park data.
+
 Various approaches exist to writing your own client, but in general:
 
 * Refer to the API docs. These are built on an OpenAPI definition file (`/webassets/apidocs/openapi.yml`), which you can automatically use to generate a client skeleton using various software.
 * Call the main "spots" or "alerts" API endpoints to get the data you want. Apply filters if necessary.
 * Call the "options" API to get an idea of which bands, modes etc. the server knows about. You might want to do that first before calling the spots/alerts APIs, to allow you to populate your filters correctly.
-* Refer to the provided HTML/JS interface for a reference
+* Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "map" and "bands" pages simply query the main spot API on a timer, whereas the main/spots page combines this approach with using the Server-Sent Events (SSE) endpoint to update live.
 * Let me know if you get stuck, I'm happy to help!
 
 ## Running your own copy

server/webserver.py

@@ -110,34 +110,40 @@ class WebServer:
 
     # Serve the SSE JSON API /spots/stream endpoint
     def serve_sse_spots_api(self):
-        response.content_type  = 'text/event-stream'
-        response.cache_control = 'no-cache'
-        yield 'retry: 1000\n\n'
+        try:
+            response.content_type  = 'text/event-stream'
+            response.cache_control = 'no-cache'
+            yield 'retry: 1000\n\n'
 
-        spot_queue = Queue(maxsize=100)
-        self.sse_spot_queues.append(spot_queue)
-        while True:
-            if spot_queue.empty():
-                gevent.sleep(1)
-            else:
-                spot = spot_queue.get()
-                yield 'data: ' + json.dumps(spot, default=serialize_everything) + '\n\n'
+            spot_queue = Queue(maxsize=100)
+            self.sse_spot_queues.append(spot_queue)
+            while True:
+                if spot_queue.empty():
+                    gevent.sleep(1)
+                else:
+                    spot = spot_queue.get()
+                    yield 'data: ' + json.dumps(spot, default=serialize_everything) + '\n\n'
+        except Exception as e:
+            logging.warn("Exception when serving SSE socket", e)
 
 
     # Serve the SSE JSON API /alerts/stream endpoint
     def serve_sse_alerts_api(self):
-        response.content_type  = 'text/event-stream'
-        response.cache_control = 'no-cache'
-        yield 'retry: 1000\n\n'
+        try:
+            response.content_type  = 'text/event-stream'
+            response.cache_control = 'no-cache'
+            yield 'retry: 1000\n\n'
 
-        alert_queue = Queue(maxsize=100)
-        self.sse_alert_queues.append(alert_queue)
-        while True:
-            if alert_queue.empty():
-                gevent.sleep(1)
-            else:
-                alert = alert_queue.get()
-                yield 'data: ' + json.dumps(alert, default=serialize_everything) + '\n\n'
+            alert_queue = Queue(maxsize=100)
+            self.sse_alert_queues.append(alert_queue)
+            while True:
+                if alert_queue.empty():
+                    gevent.sleep(1)
+                else:
+                    alert = alert_queue.get()
+                    yield 'data: ' + json.dumps(alert, default=serialize_everything) + '\n\n'
+        except Exception as e:
+            logging.warn("Exception when serving SSE socket", e)
 
     # Look up data for a callsign
     def serve_call_lookup_api(self):

spothole.py

@@ -1,14 +1,18 @@
 # Main script
+from time import sleep
+
+import gevent
+from gevent import monkey; monkey.patch_all()
+
 import importlib
 import logging
 import signal
 import sys
 
 from diskcache import Cache
-from gevent import monkey; monkey.patch_all()
 
 from core.cleanup import CleanupTimer
-from core.config import config, WEB_SERVER_PORT, SERVER_OWNER_CALLSIGN, MAX_SPOT_AGE
+from core.config import config, WEB_SERVER_PORT, SERVER_OWNER_CALLSIGN
 from core.constants import SOFTWARE_NAME, SOFTWARE_VERSION
 from core.lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
@@ -22,10 +26,13 @@ status_data = {}
 spot_providers = []
 alert_providers = []
 cleanup_timer = None
+run = True
 
 
 # Shutdown function
 def shutdown(sig, frame):
+    global run
+
     logging.info("Stopping program, this may take a few seconds...")
     for p in spot_providers:
         if p.enabled:
@@ -37,6 +44,7 @@ def shutdown(sig, frame):
     lookup_helper.stop()
     spots.close()
     alerts.close()
+    run = False
 
 
 # Utility method to get a spot provider based on the class specified in its config entry.
@@ -105,3 +113,7 @@ if __name__ == '__main__':
     status_reporter.start()
 
     logging.info("Startup complete.")
+
+    while run:
+        gevent.sleep(1)
+    exit(0)

webassets/apidocs/openapi.yml

@@ -155,7 +155,7 @@ paths:
         - Spots
       summary: Get spot stream
       description: Request a Server-Sent Event stream which will return individual spots immediately when they are added to the system. Only spots that match the provided filters will be returned.
-      operationId: spots/stream
+      operationId: spots-stream
       parameters:
         - name: source
           in: query
@@ -321,7 +321,7 @@ paths:
         - Alerts
       summary: Get alert stream
       description: Request a Server-Sent Event stream which will return individual alerts immediately when they are added to the system. Only alerts that match the provided filters will be returned.
-      operationId: alerts/stream
+      operationId: alerts-stream
       parameters:
         - name: max_duration
           in: query

webassets/js/bands.js

@@ -1,3 +1,6 @@
+// How often to query the server?
+const REFRESH_INTERVAL_SEC = 60;
+
 // A couple of constants that must match what's in CSS. We need to know them before the content actually renders, so we
 // can't just ask the elements themselves for their dimensions.
 BAND_COLUMN_HEIGHT_EM = 62;

webassets/js/common.js

@@ -116,17 +116,20 @@ function toggleFilterButtons(filterQuery, state) {
 // Update the refresh timing display
 function updateRefreshDisplay() {
     if (lastUpdateTime != null) {
-        let count = REFRESH_INTERVAL_SEC;
         let secSinceUpdate = moment.duration(moment().diff(lastUpdateTime)).asSeconds();
-        updatingString = "Updating..."
-        if (secSinceUpdate < REFRESH_INTERVAL_SEC) {
-            count = REFRESH_INTERVAL_SEC - secSinceUpdate;
-            if (count <= 60) {
-                var number = count.toFixed(0);
-                updatingString = "<span class='nowrap'>Updating in " + number + " second" + (number != "1" ? "s" : "") + ".</span>";
-            } else {
-                var number = Math.round(count / 60.0).toFixed(0);
-                updatingString = "<span class='nowrap'>Updating in " + number + " minute" + (number != "1" ? "s" : "") + ".</span>";
+        let updatingString = " Connected to live spot server.";
+        if (typeof REFRESH_INTERVAL_SEC !== 'undefined' && REFRESH_INTERVAL_SEC != null) {
+            let count = REFRESH_INTERVAL_SEC;
+            updatingString = "Updating..."
+            if (secSinceUpdate < REFRESH_INTERVAL_SEC) {
+                count = REFRESH_INTERVAL_SEC - secSinceUpdate;
+                if (count <= 60) {
+                    var number = count.toFixed(0);
+                    updatingString = "<span class='nowrap'>Updating in " + number + " second" + (number != "1" ? "s" : "") + ".</span>";
+                } else {
+                    var number = Math.round(count / 60.0).toFixed(0);
+                    updatingString = "<span class='nowrap'>Updating in " + number + " minute" + (number != "1" ? "s" : "") + ".</span>";
+                }
             }
         }
         $("#timing-container").html("Last updated at " + lastUpdateTime.format('HH:mm') + " UTC. " + updatingString);

webassets/js/map.js

@@ -1,3 +1,6 @@
+// How often to query the server?
+const REFRESH_INTERVAL_SEC = 60;
+
 // Map layers
 var markersLayer;
 var geodesicsLayer;

webassets/js/spots.js

@@ -1,3 +1,6 @@
+// SSE event source
+let evtSource;
+
 // Load spots and populate the table.
 function loadSpots() {
     $.getJSON('/api/v1/spots' + buildQueryString(), function(jsonData) {
@@ -8,9 +11,36 @@ function loadSpots() {
         spots = jsonData;
         // Update table
         updateTable();
+        // Start SSE connection to fetch updates in the background
+        restartSSEConnection();
     });
 }
 
+// Start an SSE connection (closing an existing one if it exists). This will then be used to add to the table on the
+// fly.
+function restartSSEConnection() {
+    if (evtSource != null) {
+        evtSource.close();
+    }
+    evtSource = new EventSource('/api/v1/spots/stream');
+
+    evtSource.onmessage = function(event) {
+        // Store last updated time
+        lastUpdateTime = moment.utc();
+        updateRefreshDisplay();
+        // Add spot to table
+        newSpot = JSON.parse(event.data);
+        console.log(newSpot);
+        spots.unshift(newSpot);
+        spots = spots.slice(0, -1);
+        updateTable();
+    };
+
+    evtSource.onerror = function(err) {
+        setTimeout(restartSSEConnection(), 1000);
+    };
+}
+
 // Build a query string for the API, based on the filters that the user has selected.
 function buildQueryString() {
     var str =  "?";
@@ -316,9 +346,8 @@ function loadOptions() {
             $("#add-spot-button").show();
         }
 
-        // Load spots and set up the timer
+        // Load spots (this will also set up the SSE connection to update them too)
         loadSpots();
-        setInterval(loadSpots, REFRESH_INTERVAL_SEC * 1000);
     });
 }
 
@@ -384,8 +413,6 @@ function displayIntroBox() {
 $(document).ready(function() {
     // Call loadOptions(), this will then trigger loading spots and setting up timers.
     loadOptions();
-    // Update the refresh timing display every second
-    setInterval(updateRefreshDisplay, 1000);
     // Display intro box
     displayIntroBox();
 });

webassets/js/spotsbandsandmap.js

@@ -1,6 +1,3 @@
-// How often to query the server?
-const REFRESH_INTERVAL_SEC = 60;
-
 // Storage for the spot data that the server gives us.
 var spots = []