First attempt at SSE backend #3

alertproviders/alert_provider.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import pytz
 
 from core.config import MAX_ALERT_AGE
+from spothole import add_alert
 
 
 # Generic alert provider class. Subclasses of this query the individual APIs for alerts.
@@ -29,11 +30,9 @@ class AlertProvider:
     # to deal with duplicates.
     def submit_batch(self, alerts):
         for alert in alerts:
-            # Fill in any blanks
+            # Fill in any blanks and add to the list
             alert.infer_missing()
-            # Add to the list, provided it heas not already expired.
+            add_alert(alert)
-            if not alert.expired():
-                self.alerts.add(alert.id, alert, expire=MAX_ALERT_AGE)
 
     # Stop any threads and prepare for application shutdown
     def stop(self):

core/cleanup.py

@@ -10,9 +10,10 @@ import pytz
 class CleanupTimer:
 
     # Constructor
-    def __init__(self, spots, alerts, cleanup_interval):
+    def __init__(self, spots, alerts, web_server, cleanup_interval):
         self.spots = spots
         self.alerts = alerts
+        self.web_server = web_server
         self.cleanup_interval = cleanup_interval
         self.cleanup_timer = None
         self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC)
@@ -51,6 +52,9 @@ class CleanupTimer:
                     # Must have already been deleted, OK with that
                     pass
 
+            # Clean up web server SSE spot/alert queues
+            self.web_server.clean_up_sse_queues()
+
             self.status = "OK"
             self.last_cleanup_time = datetime.now(pytz.UTC)
 

requirements.txt

@@ -13,4 +13,5 @@ rss-parser~=2.1.1
 pyproj~=3.7.2
 prometheus_client~=0.23.1
 beautifulsoup4~=4.14.2
-websocket-client~=1.9.0
+websocket-client~=1.9.0
+gevent~=25.9.1

server/webserver.py

@@ -2,6 +2,7 @@ import json
 import logging
 import re
 from datetime import datetime, timedelta
+from queue import Queue
 from threading import Thread
 
 import bottle
@@ -28,6 +29,8 @@ class WebServer:
         self.api_access_counter = 0
         self.spots = spots
         self.alerts = alerts
+        self.sse_spot_queues = []
+        self.sse_alert_queues = []
         self.status_data = status_data
         self.port = port
         self.thread = Thread(target=self.run)
@@ -41,6 +44,8 @@ class WebServer:
         # Routes for API calls
         bottle.get("/api/v1/spots")(lambda: self.serve_spots_api())
         bottle.get("/api/v1/alerts")(lambda: self.serve_alerts_api())
+        bottle.get("/api/v1/spots/stream")(lambda: self.serve_sse_spots_api())
+        bottle.get("/api/v1/alerts/stream")(lambda: self.serve_sse_alerts_api())
         bottle.get("/api/v1/options")(lambda: self.serve_api(self.get_options()))
         bottle.get("/api/v1/status")(lambda: self.serve_api(self.status_data))
         bottle.get("/api/v1/lookup/call")(lambda: self.serve_call_lookup_api())
@@ -102,6 +107,31 @@ class WebServer:
             response.status = 500
             return json.dumps("Error - " + str(e), default=serialize_everything)
 
+    # Serve the SSE JSON API /spots/stream endpoint
+    def serve_sse_spots_api(self):
+        response.content_type  = 'text/event-stream'
+        response.cache_control = 'no-cache'
+        yield 'retry: 1000\n\n'
+
+        spot_queue = Queue(maxsize=100)
+        self.sse_spot_queues.append(spot_queue)
+        while True:
+            spot = spot_queue.get()
+            yield 'data: ' + json.dumps(spot, default=serialize_everything) + '\n\n'
+
+
+    # Serve the SSE JSON API /alerts/stream endpoint
+    def serve_sse_alerts_api(self):
+        response.content_type  = 'text/event-stream'
+        response.cache_control = 'no-cache'
+        yield 'retry: 1000\n\n'
+
+        alert_queue = Queue(maxsize=100)
+        self.sse_alert_queues.append(alert_queue)
+        while True:
+            alert = alert_queue.get()
+            yield 'data: ' + json.dumps(alert, default=serialize_everything) + '\n\n'
+
     # Look up data for a callsign
     def serve_call_lookup_api(self):
         try:
@@ -303,16 +333,9 @@ class WebServer:
         # Get the query (and the right one, with Bottle magic. This is a MultiDict object)
         query = bottle.request.query
 
-        # Create a shallow copy of the spot list, ordered by spot time. We'll then filter it accordingly.
+        # Create a shallow copy of the spot list, ordered by spot time, then filter the list to reduce it only to spots
-        # We can filter by spot time and received time with "since" and "received_since", which take a UNIX timestamp
+        # that match the filter parameters in the query string. Finally, apply a limit to the number of spots returned.
-        # in seconds UTC.
+        # The list of query string filters is defined in the API docs.
-        # We can also filter by source, sig, band, mode, dx_continent and de_continent. Each of these accepts a single
-        # value or a comma-separated list.
-        # We can filter by comments, accepting a single string, where the API will only return spots where the comment
-        # contains the provided value (case-insensitive).
-        # We can "de-dupe" spots, so only the latest spot will be sent for each callsign.
-        # We can provide a "limit" number as well. Spots are always returned newest-first; "limit" limits to only the
-        # most recent X spots.
         spot_ids = list(self.spots.iterkeys())
         spots = []
         for k in spot_ids:
@@ -320,87 +343,29 @@ class WebServer:
             if s is not None:
                 spots.append(s)
         spots = sorted(spots, key=lambda spot: (spot.time if spot and spot.time else 0), reverse=True)
-        for k in query.keys():
+        spots = list(filter(lambda spot: spot_allowed_by_query(spot, query), spots))
-            match k:
-                case "since":
-                    since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC).timestamp()
-                    spots = [s for s in spots if s.time and s.time > since]
-                case "max_age":
-                    max_age = int(query.get(k))
-                    since = (datetime.now(pytz.UTC) - timedelta(seconds=max_age)).timestamp()
-                    spots = [s for s in spots if s.time and s.time > since]
-                case "received_since":
-                    since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC).timestamp()
-                    spots = [s for s in spots if s.received_time and s.received_time > since]
-                case "source":
-                    sources = query.get(k).split(",")
-                    spots = [s for s in spots if s.source and s.source in sources]
-                case "sig":
-                    # If a list of sigs is provided, the spot must have a sig and it must match one of them.
-                    # The special "sig" "NO_SIG", when supplied in the list, mathches spots with no sig.
-                    sigs = query.get(k).split(",")
-                    include_no_sig = "NO_SIG" in sigs
-                    spots = [s for s in spots if (s.sig and s.sig in sigs) or (include_no_sig and not s.sig)]
-                case "needs_sig":
-                    # If true, a sig is required, regardless of what it is, it just can't be missing. Mutually
-                    # exclusive with supplying the special "NO_SIG" parameter to the "sig" query param.
-                    needs_sig = query.get(k).upper() == "TRUE"
-                    if needs_sig:
-                        spots = [s for s in spots if s.sig]
-                case "needs_sig_ref":
-                    # If true, at least one sig ref is required, regardless of what it is, it just can't be missing.
-                    needs_sig_ref = query.get(k).upper() == "TRUE"
-                    if needs_sig_ref:
-                        spots = [s for s in spots if s.sig_refs and len(s.sig_refs) > 0]
-                case "band":
-                    bands = query.get(k).split(",")
-                    spots = [s for s in spots if s.band and s.band in bands]
-                case "mode":
-                    modes = query.get(k).split(",")
-                    spots = [s for s in spots if s.mode in modes]
-                case "mode_type":
-                    mode_families = query.get(k).split(",")
-                    spots = [s for s in spots if s.mode_type and s.mode_type in mode_families]
-                case "dx_continent":
-                    dxconts = query.get(k).split(",")
-                    spots = [s for s in spots if s.dx_continent and s.dx_continent in dxconts]
-                case "de_continent":
-                    deconts = query.get(k).split(",")
-                    spots = [s for s in spots if s.de_continent and s.de_continent in deconts]
-                case "comment_includes":
-                    comment_includes = query.get(k).strip()
-                    spots = [s for s in spots if s.comment and comment_includes.upper() in s.comment.upper()]
-                case "dx_call_includes":
-                    dx_call_includes = query.get(k).strip()
-                    spots = [s for s in spots if s.dx_call and dx_call_includes.upper() in s.dx_call.upper()]
-                case "allow_qrt":
-                    # If false, spots that are flagged as QRT are not returned.
-                    prevent_qrt = query.get(k).upper() == "FALSE"
-                    if prevent_qrt:
-                        spots = [s for s in spots if not s.qrt or s.qrt == False]
-                case "needs_good_location":
-                    # If true, spots require a "good" location to be returned
-                    needs_good_location = query.get(k).upper() == "TRUE"
-                    if needs_good_location:
-                        spots = [s for s in spots if s.dx_location_good]
-                case "dedupe":
-                    # Ensure only the latest spot of each callsign-SSID combo is present in the list. This relies on the
-                    # list being in reverse time order, so if any future change allows re-ordering the list, that should
-                    # be done *after* this. SSIDs are deliberately included here (see issue #68) because e.g. M0TRT-7
-                    # and M0TRT-9 APRS transponders could well be in different locations, on different frequencies etc.
-                    dedupe = query.get(k).upper() == "TRUE"
-                    if dedupe:
-                        spots_temp = []
-                        already_seen = []
-                        for s in spots:
-                            call_plus_ssid = s.dx_call + (s.dx_ssid if s.dx_ssid else "")
-                            if call_plus_ssid not in already_seen:
-                                spots_temp.append(s)
-                                already_seen.append(call_plus_ssid)
-                        spots = spots_temp
-        # If we have a "limit" parameter, we apply that last, regardless of where it appeared in the list of keys.
         if "limit" in query.keys():
             spots = spots[:int(query.get("limit"))]
+
+        # Ensure only the latest spot of each callsign-SSID combo is present in the list. This relies on the
+        # list being in reverse time order, so if any future change allows re-ordering the list, that should
+        # be done *after* this. SSIDs are deliberately included here (see issue #68) because e.g. M0TRT-7
+        # and M0TRT-9 APRS transponders could well be in different locations, on different frequencies etc.
+        # This is a special consideration for the geo map and band map views (and Field Spotter) because while
+        # duplicates are fine in the main spot list (e.g. different cluster spots of the same DX) this doesn't
+        # work well for the other views.
+        if "dedupe" in query.keys():
+                dedupe = query.get("dedupe").upper() == "TRUE"
+                if dedupe:
+                    spots_temp = []
+                    already_seen = []
+                    for s in spots:
+                        call_plus_ssid = s.dx_call + (s.dx_ssid if s.dx_ssid else "")
+                        if call_plus_ssid not in already_seen:
+                            spots_temp.append(s)
+                            already_seen.append(call_plus_ssid)
+                    spots = spots_temp
+
         return spots
 
     # Utility method to apply filters to the overall alert list and return only a subset. Enables query parameters in
@@ -409,50 +374,17 @@ class WebServer:
         # Get the query (and the right one, with Bottle magic. This is a MultiDict object)
         query = bottle.request.query
 
-        # Create a shallow copy of the alert list, ordered by start time. We'll then filter it accordingly.
+        # Create a shallow copy of the alert list ordered by start time, then filter the list to reduce it only to alerts
-        # We can filter by received time with "received_since", which take a UNIX timestamp in seconds UTC.
+        # that match the filter parameters in the query string. Finally, apply a limit to the number of alerts returned.
-        # We can also filter by source, sig, and dx_continent. Each of these accepts a single
+        # The list of query string filters is defined in the API docs.
-        # value or a comma-separated list.
-        # We can provide a "limit" number as well. Alerts are always returned newest-first; "limit" limits to only the
-        # most recent X alerts.
         alert_ids = list(self.alerts.iterkeys())
         alerts = []
         for k in alert_ids:
             a = self.alerts.get(k)
             if a is not None:
                 alerts.append(a)
-        # We never want alerts that seem to be in the past
         alerts = sorted(alerts, key=lambda alert: (alert.start_time if alert and alert.start_time else 0))
-        for k in query.keys():
+        alerts = list(filter(lambda alert: alert_allowed_by_query(alert, query), alerts))
-            match k:
-                case "received_since":
-                    since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC)
-                    alerts = [a for a in alerts if a.received_time and a.received_time > since]
-                case "max_duration":
-                    max_duration = int(query.get(k))
-                    # Check the duration if end_time is provided. If end_time is not provided, assume the activation is
-                    # "short", i.e. it always passes this check. If dxpeditions_skip_max_duration_check is true and
-                    # the alert is a dxpedition, it also always passes the check.
-                    dxpeditions_skip_check = bool(query.get(
-                        "dxpeditions_skip_max_duration_check")) if "dxpeditions_skip_max_duration_check" in query.keys() else False
-                    alerts = [a for a in alerts if (a.end_time and a.end_time - a.start_time <= max_duration) or
-                              not a.end_time or (dxpeditions_skip_check and a.is_dxpedition)]
-                case "source":
-                    sources = query.get(k).split(",")
-                    alerts = [a for a in alerts if a.source and a.source in sources]
-                case "sig":
-                    # If a list of sigs is provided, the alert must have a sig and it must match one of them.
-                    # The special "sig" "NO_SIG", when supplied in the list, mathches alerts with no sig.
-                    sigs = query.get(k).split(",")
-                    include_no_sig = "NO_SIG" in sigs
-                    spots = [a for a in alerts if (a.sig and a.sig in sigs) or (include_no_sig and not a.sig)]
-                case "dx_continent":
-                    dxconts = query.get(k).split(",")
-                    alerts = [a for a in alerts if a.dx_continent and a.dx_continent in dxconts]
-                case "dx_call_includes":
-                    dx_call_includes = query.get(k).strip()
-                    spots = [a for a in alerts if a.dx_call and dx_call_includes.upper() in a.dx_call.upper()]
-        # If we have a "limit" parameter, we apply that last, regardless of where it appeared in the list of keys.
         if "limit" in query.keys():
             alerts = alerts[:int(query.get("limit"))]
         return alerts
@@ -481,6 +413,155 @@ class WebServer:
 
         return options
 
+    # Internal method called when a new spot is added to the system. This is used to ping any SSE clients that are
+    # awaiting a server-sent message with new spots.
+    def notify_new_spot(self, spot):
+        for queue in self.sse_spot_queues:
+            try:
+                queue.put(spot)
+            except:
+                # Cleanup thread was probably deleting the queue, that's fine
+                pass
+
+    # Internal method called when a new alert is added to the system. This is used to ping any SSE clients that are
+    # awaiting a server-sent message with new spots.
+    def notify_new_alert(self, alert):
+        for queue in self.sse_alert_queues:
+            try:
+                queue.put(alert)
+            except:
+                # Cleanup thread was probably deleting the queue, that's fine
+                pass
+
+    # Clean up any SSE queues that are growing too large; probably their client disconnected.
+    def clean_up_sse_queues(self):
+        self.sse_spot_queues = [q for q in self.sse_spot_queues if not q.full()]
+        self.sse_alert_queues = [q for q in self.sse_alert_queues if not q.full()]
+
+
+# Given URL query params and a spot, figure out if the spot "passes" the requested filters or is rejected. The list
+# of query parameters and their function is defined in the API docs.
+def spot_allowed_by_query(spot, query):
+    for k in query.keys():
+        match k:
+            case "since":
+                since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC).timestamp()
+                if not spot.time or spot.time <= since:
+                    return False
+            case "max_age":
+                max_age = int(query.get(k))
+                since = (datetime.now(pytz.UTC) - timedelta(seconds=max_age)).timestamp()
+                if not spot.time or spot.time <= since:
+                    return False
+            case "received_since":
+                since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC).timestamp()
+                if not spot.received_time or spot.received_time <= since:
+                    return False
+            case "source":
+                sources = query.get(k).split(",")
+                if not spot.source or spot.source not in sources:
+                    return False
+            case "sig":
+                # If a list of sigs is provided, the spot must have a sig and it must match one of them.
+                # The special "sig" "NO_SIG", when supplied in the list, mathches spots with no sig.
+                sigs = query.get(k).split(",")
+                include_no_sig = "NO_SIG" in sigs
+                if not spot.sig and not include_no_sig:
+                    return False
+                if spot.sig and spot.sig not in sigs:
+                    return False
+            case "needs_sig":
+                # If true, a sig is required, regardless of what it is, it just can't be missing. Mutually
+                # exclusive with supplying the special "NO_SIG" parameter to the "sig" query param.
+                needs_sig = query.get(k).upper() == "TRUE"
+                if needs_sig and not spot.sig:
+                    return False
+            case "needs_sig_ref":
+                # If true, at least one sig ref is required, regardless of what it is, it just can't be missing.
+                needs_sig_ref = query.get(k).upper() == "TRUE"
+                if needs_sig_ref and (not spot.sig_refs or len(spot.sig_refs) == 0):
+                    return False
+            case "band":
+                bands = query.get(k).split(",")
+                if not spot.band or spot.band not in bands:
+                    return False
+            case "mode":
+                modes = query.get(k).split(",")
+                if not spot.mode or spot.mode not in modes:
+                    return False
+            case "mode_type":
+                mode_types = query.get(k).split(",")
+                if not spot.mode_type or spot.mode_type not in mode_types:
+                    return False
+            case "dx_continent":
+                dxconts = query.get(k).split(",")
+                if not spot.dx_continent or spot.dx_continent not in dxconts:
+                    return False
+            case "de_continent":
+                deconts = query.get(k).split(",")
+                if not spot.de_continent or spot.de_continent not in deconts:
+                    return False
+            case "comment_includes":
+                comment_includes = query.get(k).strip()
+                if not spot.comment or comment_includes.upper() not in spot.comment.upper():
+                    return False
+            case "dx_call_includes":
+                dx_call_includes = query.get(k).strip()
+                if not spot.dx_call or dx_call_includes.upper() not in spot.dx_call.upper():
+                    return False
+            case "allow_qrt":
+                # If false, spots that are flagged as QRT are not returned.
+                prevent_qrt = query.get(k).upper() == "FALSE"
+                if prevent_qrt and spot.qrt and spot.qrt == True:
+                    return False
+            case "needs_good_location":
+                # If true, spots require a "good" location to be returned
+                needs_good_location = query.get(k).upper() == "TRUE"
+                if needs_good_location and not spot.dx_location_good:
+                    return False
+    return True
+
+# Given URL query params and an alert, figure out if the alert "passes" the requested filters or is rejected. The list
+# of query parameters and their function is defined in the API docs.
+def alert_allowed_by_query(alert, query):
+    for k in query.keys():
+        match k:
+            case "received_since":
+                since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC)
+                if not alert.received_time or alert.received_time <= since:
+                    return False
+            case "max_duration":
+                max_duration = int(query.get(k))
+                # Check the duration if end_time is provided. If end_time is not provided, assume the activation is
+                # "short", i.e. it always passes this check. If dxpeditions_skip_max_duration_check is true and
+                # the alert is a dxpedition, it also always passes the check.
+                if alert.is_dxpedition and (bool(query.get(
+                        "dxpeditions_skip_max_duration_check")) if "dxpeditions_skip_max_duration_check" in query.keys() else False):
+                    continue
+                if alert.end_time and alert.start_time and alert.end_time - alert.start_time > max_duration:
+                    return False
+            case "source":
+                sources = query.get(k).split(",")
+                if not alert.source or alert.source not in sources:
+                    return False
+            case "sig":
+                # If a list of sigs is provided, the alert must have a sig and it must match one of them.
+                # The special "sig" "NO_SIG", when supplied in the list, mathches alerts with no sig.
+                sigs = query.get(k).split(",")
+                include_no_sig = "NO_SIG" in sigs
+                if not alert.sig and not include_no_sig:
+                    return False
+                if alert.sig and alert.sig not in sigs:
+                    return False
+            case "dx_continent":
+                dxconts = query.get(k).split(",")
+                if not alert.dx_continent or alert.dx_continent not in dxconts:
+                    return False
+            case "dx_call_includes":
+                dx_call_includes = query.get(k).strip()
+                if not alert.dx_call or dx_call_includes.upper() not in alert.dx_call.upper():
+                    return False
+    return True
 
 # Convert objects to serialisable things. Used by JSON serialiser as a default when it encounters unserializable things.
 # Just converts objects to dict. Try to avoid doing anything clever here when serialising spots, because we also need

spothole.py

@@ -3,11 +3,13 @@ import importlib
 import logging
 import signal
 import sys
+from time import sleep
 
 from diskcache import Cache
+from gevent import monkey; monkey.patch_all()
 
 from core.cleanup import CleanupTimer
-from core.config import config, WEB_SERVER_PORT, SERVER_OWNER_CALLSIGN
+from core.config import config, WEB_SERVER_PORT, SERVER_OWNER_CALLSIGN, MAX_SPOT_AGE
 from core.constants import SOFTWARE_NAME, SOFTWARE_VERSION
 from core.lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
@@ -16,14 +18,17 @@ from server.webserver import WebServer
 # Globals
 spots = Cache('cache/spots_cache')
 alerts = Cache('cache/alerts_cache')
+web_server = None
 status_data = {}
 spot_providers = []
 alert_providers = []
 cleanup_timer = None
+run = True
 
 
 # Shutdown function
 def shutdown(sig, frame):
+    global spot_providers, alert_providers, cleanup_timer, spots, alerts, run
     logging.info("Stopping program, this may take a few seconds...")
     for p in spot_providers:
         if p.enabled:
@@ -35,6 +40,7 @@ def shutdown(sig, frame):
     lookup_helper.stop()
     spots.close()
     alerts.close()
+    run = False
 
 
 # Utility method to get a spot provider based on the class specified in its config entry.
@@ -50,6 +56,24 @@ def get_alert_provider_from_config(config_providers_entry):
     provider_class = getattr(module, config_providers_entry["class"])
     return provider_class(config_providers_entry)
 
+# Utility method to add a spot, notifying the web server in case any Server-Sent Event connections need to have data
+# sent immediately. If the spot has already expired due to loading old data, it will be ignored.
+def add_spot(spot):
+    global web_server
+    if not spot.expired():
+        spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
+        if web_server:
+            web_server.notify_new_spot(spot)
+
+# Utility method to add an alert, notifying the web server in case any Server-Sent Event connections need to have data
+# sent immediately. If the alert has already expired due to loading old data, it will be ignored.
+def add_alert(alert):
+    global web_server
+    if not alert.expired():
+        alerts.add(alert.id, alert, expire=MAX_SPOT_AGE)
+        if web_server:
+            web_server.notify_new_alert(alert)
+
 
 # Main function
 if __name__ == '__main__':
@@ -72,6 +96,10 @@ if __name__ == '__main__':
     # Set up lookup helper
     lookup_helper.start()
 
+    # Set up web server
+    web_server = WebServer(spots=spots, alerts=alerts, status_data=status_data, port=WEB_SERVER_PORT)
+    web_server.start()
+
     # Fetch, set up and start spot providers
     for entry in config["spot-providers"]:
         spot_providers.append(get_spot_provider_from_config(entry))
@@ -89,13 +117,9 @@ if __name__ == '__main__':
             p.start()
 
     # Set up timer to clear spot list of old data
-    cleanup_timer = CleanupTimer(spots=spots, alerts=alerts, cleanup_interval=60)
+    cleanup_timer = CleanupTimer(spots=spots, alerts=alerts, web_server=web_server, cleanup_interval=60)
     cleanup_timer.start()
 
-    # Set up web server
-    web_server = WebServer(spots=spots, alerts=alerts, status_data=status_data, port=WEB_SERVER_PORT)
-    web_server.start()
-
     # Set up status reporter
     status_reporter = StatusReporter(status_data=status_data, spots=spots, alerts=alerts, web_server=web_server,
                                      cleanup_timer=cleanup_timer, spot_providers=spot_providers,
@@ -103,3 +127,6 @@ if __name__ == '__main__':
     status_reporter.start()
 
     logging.info("Startup complete.")
+
+    while (run):
+        sleep(1)

spotproviders/spot_provider.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import pytz
 
 from core.config import MAX_SPOT_AGE
+from spothole import add_spot
 
 
 # Generic spot provider class. Subclasses of this query the individual APIs for data.
@@ -32,23 +33,19 @@ class SpotProvider:
     def submit_batch(self, spots):
         for spot in spots:
             if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time:
-                # Fill in any blanks
+                # Fill in any blanks and add to the list
                 spot.infer_missing()
-                # Add to the list, provided it heas not already expired.
+                add_spot(spot)
-                if not spot.expired():
-                    self.spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
         self.last_spot_time = datetime.fromtimestamp(max(map(lambda s: s.time, spots)), pytz.UTC)
 
     # Submit a single spot retrieved from the provider. This will be added to the list regardless of its age. Spots
     # passing the check will also have their infer_missing() method called to complete their data set. This is called by
     # the data streaming subclasses, which can be relied upon not to re-provide old spots.
     def submit(self, spot):
-        # Fill in any blanks
+        # Fill in any blanks and add to the list
         spot.infer_missing()
-        # Add to the list, provided it heas not already expired.
+        add_spot(spot)
-        if not spot.expired():
+        self.last_spot_time = datetime.fromtimestamp(spot.time, pytz.UTC)
-            self.spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
-            self.last_spot_time = datetime.fromtimestamp(spot.time, pytz.UTC)
 
     # Stop any threads and prepare for application shutdown
     def stop(self):

webassets/apidocs/openapi.yml

@@ -149,6 +149,104 @@ paths:
                 items:
                   $ref: '#/components/schemas/Spot'
 
+  /spots/stream:
+    get:
+      tags:
+        - Spots
+      summary: Get spot stream
+      description: Request a Server-Sent Event stream which will return individual spots immediately when they are added to the system. Only spots that match the provided filters will be returned.
+      operationId: spots/stream
+      parameters:
+        - name: source
+          in: query
+          description: "Limit the spots to only ones from one or more sources. To select more than one source, supply a comma-separated list."
+          required: false
+          schema:
+            $ref: "#/components/schemas/Source"
+        - name: sig
+          in: query
+          description: "Limit the spots to only ones from one or more Special Interest Groups provided as an argument. To select more than one SIG, supply a comma-separated list. The special `sig` name `NO_SIG` matches spots with no sig set. You can use `sig=NO_SIG` to specifically only return generic spots with no associated SIG. You can also use combinations to request for example POTA + no SIG, but reject other SIGs. If you want to request 'every SIG and not No SIG', see the `needs_sig` query parameter for a shortcut."
+          required: false
+          schema:
+            $ref: "#/components/schemas/SIGNameIncludingNoSIG"
+        - name: needs_sig
+          in: query
+          description: "Limit the spots to only ones with a Special Interest Group such as POTA. Because supplying all known SIGs as a `sigs` parameter is unwieldy, and leaving `sigs` blank will also return spots with *no* SIG, this parameter can be set true to return only spots with a SIG, regardless of what it is, so long as it's not blank. This is the equivalent of supplying the `sig` query param with a list of every known SIG apart from the special `NO_SIG` value. This is what Field Spotter uses to exclude generic cluster spots and only retrieve xOTA things."
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: needs_sig_ref
+          in: query
+          description: "Limit the spots to only ones which have at least one reference (e.g. a park reference) for Special Interest Groups such as POTA."
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: band
+          in: query
+          description: "Limit the spots to only ones from one or more bands. To select more than one band, supply a comma-separated list."
+          required: false
+          schema:
+            $ref: "#/components/schemas/BandName"
+        - name: mode
+          in: query
+          description: "Limit the spots to only ones from one or more modes. To select more than one mode, supply a comma-separated list."
+          required: false
+          schema:
+            $ref: "#/components/schemas/Mode"
+        - name: mode_type
+          in: query
+          description: "Limit the spots to only ones from one or more mode families. To select more than one mode family, supply a comma-separated list."
+          required: false
+          schema:
+            $ref: "#/components/schemas/Mode"
+        - name: dx_continent
+          in: query
+          description: "Limit the spots to only ones where the DX (the operator being spotted) is on the given continent(s). To select more than one continent, supply a comma-separated list."
+          required: false
+          schema:
+            $ref: "#/components/schemas/Continent"
+        - name: de_continent
+          in: query
+          description: "Limit the spots to only ones where the spotteris on the given continent(s). To select more than one continent, supply a comma-separated list."
+          required: false
+          schema:
+            $ref: "#/components/schemas/Continent"
+        - name: dx_call_includes
+          in: query
+          description: "Limit the alerts to only ones where the DX callsign includes the supplied string (case-insensitive). Generally a complete callsign, but you can supply a shorter string for partial matches."
+          required: false
+          schema:
+            type: string
+        - name: comment_includes
+          in: query
+          description: "Return only spots where the comment includes the provided string (case-insensitive)."
+          required: false
+          schema:
+            type: string
+        - name: needs_good_location
+          in: query
+          description: "Return only spots with a 'good' location. (See the spot `dx_location_good` parameter for details. Useful for map-based clients, to avoid spots with 'bad' locations e.g. loads of cluster spots ending up in the centre of the DXCC entitity.)"
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: allow_qrt
+          in: query
+          description: Allow spots that are known to be QRT to be returned.
+          required: false
+          schema:
+            type: boolean
+            default: true
+      responses:
+        '200':
+          description: Success
+          content:
+            text/event-stream:
+              schema:
+                $ref: '#/components/schemas/SpotStream'
+
 
   /alerts:
     get:
@@ -156,7 +254,7 @@ paths:
         - Alerts
       summary: Get alerts
       description: Retrieves alerts (indications of upcoming activations) from the system. Supply this with no query parameters to retrieve all alerts known to the system. Supply query parameters to filter what is retrieved.
-      operationId: spots
+      operationId: alerts
       parameters:
         - name: limit
           in: query
@@ -217,6 +315,59 @@ paths:
                   $ref: '#/components/schemas/Alert'
 
 
+  /alerts/stream:
+    get:
+      tags:
+        - Alerts
+      summary: Get alert stream
+      description: Request a Server-Sent Event stream which will return individual alerts immediately when they are added to the system. Only alerts that match the provided filters will be returned.
+      operationId: alerts/stream
+      parameters:
+        - name: max_duration
+          in: query
+          description: Limit the alerts to only ones with a duration of this many seconds or less. Duration is end time minus start time, if end time is set, otherwise the activation is assumed to be short and therefore to always pass this check. This is useful to filter out people who alert POTA activations lasting months or even years, but note it will also include multi-day or multi-week DXpeditions that you might otherwise be interested in. See the dxpeditions_skip_max_duration_check parameter for the workaround.
+          required: false
+          schema:
+            type: integer
+        - name: dxpeditions_skip_max_duration_check
+          in: query
+          description: Return DXpedition alerts even if they last longer than max_duration. This allows the user to filter out multi-day/multi-week POTA alerts where the operator likely won't be on the air most of the time, but keep multi-day/multi-week DXpeditions where the operator(s) likely *will* be on the air most of the time.
+          required: false
+          schema:
+            type: boolean
+        - name: source
+          in: query
+          description: "Limit the alerts to only ones from one or more sources. To select more than one source, supply a comma-separated list."
+          required: false
+          schema:
+            $ref: "#/components/schemas/Source"
+        - name: sig
+          in: query
+          description: "Limit the alerts to only ones from one or more Special Interest Groups. To select more than one SIG, supply a comma-separated list. The special value 'NO_SIG' can be included to return alerts specifically without an associated SIG (i.e. general DXpeditions)."
+          required: false
+          schema:
+            $ref: "#/components/schemas/SIGNameIncludingNoSIG"
+        - name: dx_continent
+          in: query
+          description: "Limit the alerts to only ones where the DX operator is on the given continent(s). To select more than one continent, supply a comma-separated list."
+          required: false
+          schema:
+            $ref: "#/components/schemas/Continent"
+        - name: dx_call_includes
+          in: query
+          description: "Limit the alerts to only ones where the DX callsign includes the supplied string (case-insensitive). Generally a complete callsign, but you can supply a shorter string for partial matches."
+          required: false
+          schema:
+            type: string
+      responses:
+        '200':
+          description: Success
+          content:
+            text/event-stream:
+              schema:
+                $ref: '#/components/schemas/AlertStream'
+
+
   /status:
     get:
       tags:
@@ -933,6 +1084,15 @@ components:
           example: "GUID-123456"
 
 
+    SpotStream:
+      type: object
+      description: A server-sent event containing a spot
+      required: [data]
+      properties:
+        data:
+          $ref: "#/components/schemas/Spot"
+
+
     Alert:
       type: object
       properties:
@@ -1032,6 +1192,16 @@ components:
           description: The ID the source gave it, if any.
           example: "GUID-123456"
 
+
+    AlertStream:
+      type: object
+      description: A server-sent event containing an alert
+      required: [data]
+      properties:
+        data:
+          $ref: "#/components/schemas/Alert"
+
+
     SpotProviderStatus:
       type: object
       properties: