Release 2.1.3

# Conflicts:
#	templates/add_spot.html
#	templates/alerts.html
#	templates/bands.html
#	templates/base.html
#	templates/conditions.html
#	templates/map.html
#	templates/spots.html
#	templates/status.html

core/cleanup.py

@@ -34,6 +34,10 @@ class CleanupTimer:
         """Stop any threads and prepare for application shutdown"""
 
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=5)
+            if self._thread.is_alive():
+                logger.warning("Cleanup worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while not self._stop_event.wait(timeout=self._cleanup_interval):

core/constants.py

@@ -4,7 +4,7 @@ from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "2.1-pre"
+SOFTWARE_VERSION = "2.1.3"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"}
@@ -263,6 +263,16 @@ SIGS = [
         region_flag="🇮🇪",
         refs_globally_unique=False,
     ),
+    SIG(
+        name="DMF",
+        comment_names=["DMF"],
+        description="Diplôme des Moulins de France",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.MILL,
+        icon="fa-fan",
+        region_flag="🇫🇷",
+        refs_globally_unique=False,
+    ),
     SIG(
         name="DME",
         comment_names=["DME"],

core/data_providers.py

@@ -1,5 +1,6 @@
 import logging
 import threading
+import time
 
 from core.config import config, create_provider_from_config
 
@@ -16,6 +17,7 @@ class DataProviders:
         self.static_data_providers = []
         self.sig_ref_data_providers = []
         self.callsign_data_providers = []
+        self._startup_timers = []
 
     def setup(self):
         for entry in config["spot_providers"]:
@@ -43,41 +45,58 @@ class DataProviders:
     def start(self):
         # Start data providers before spot/alert providers so the lookup data is there already for incoming spots.
         # Each category is fired off after a small delay to give the rest of Spothole chance to start up.
-        threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")).start()
+        self._startup_timers = [
-        threading.Timer(
+            threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")),
-            10.0,
+            threading.Timer(10.0, lambda: self.start_providers(self.callsign_data_providers, "callsign data")),
-            lambda: self.start_providers(self.callsign_data_providers, "callsign data"),
+            threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")),
-        ).start()
+            threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")),
-        threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")).start()
-        threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")).start()
             threading.Timer(
                 25.0,
                 lambda: self.start_providers(self.solar_condition_providers, "solar condition"),
-        ).start()
+            ),
-        threading.Timer(
+            threading.Timer(30.0, lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data")),
-            30.0,
+        ]
-            lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data"),
+        for t in self._startup_timers:
-        ).start()
+            t.daemon = True
+            t.start()
 
     def stop(self):
-        for sp in self.spot_providers:
+        # Cancel any startup timers that haven't fired yet
-            if sp.enabled:
+        for t in self._startup_timers:
-                sp.stop()
+            t.cancel()
-        for ap in self.alert_providers:
+
-            if ap.enabled:
+        # Stop all providers
-                ap.stop()
+        all_providers = [
-        for scp in self.solar_condition_providers:
+            p
-            if scp.enabled:
+            for p in (
-                scp.stop()
+                self.spot_providers
-        for srdp in self.sig_ref_data_providers:
+                + self.alert_providers
-            if srdp.enabled:
+                + self.solar_condition_providers
-                srdp.stop()
+                + self.sig_ref_data_providers
-        for sdp in self.static_data_providers:
+                + self.static_data_providers
-            if sdp.enabled:
+                + self.callsign_data_providers
-                sdp.stop()
+            )
-        for cdp in self.callsign_data_providers:
+            if p.enabled
-            if cdp.enabled:
+        ]
-                cdp.stop()
+        if not all_providers:
+            return
+
+        def stop_provider(p):
+            try:
+                p.stop()
+            except Exception:
+                logger.exception("Exception stopping provider")
+
+        threads = [threading.Thread(target=stop_provider, args=(p,), daemon=True) for p in all_providers]
+        for t in threads:
+            t.start()
+
+        deadline = time.monotonic() + 15
+        for t in threads:
+            t.join(timeout=max(0.0, deadline - time.monotonic()))
+        still_running = [t for t in threads if t.is_alive()]
+        if still_running:
+            logger.warning("Some threads did not stop in time!")
 
 
 # Global object

core/data_store.py

@@ -106,6 +106,7 @@ class DataStore:
         and testing the callsign every time is expensive. So instead we build a separate in-memory lookup of compiled
         regex against DXCC entity code, as a list of tuples we can iterate through."""
 
+        self.dxcc_lookup_by_call_regex = []
         for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]:
             self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
 

core/live_data_cache.py

@@ -22,6 +22,8 @@ class LiveDataCache:
         self._listeners_lock = threading.Lock()
         self._snapshot_dir = snapshot_dir
         self._disk_cache = diskcache.Cache(str(snapshot_dir))
+        self._stop_event = threading.Event()
+        self._snapshot_thread = None
         self._load_snapshot()
         self._start_periodic_snapshot(snapshot_interval_sec)
 
@@ -75,7 +77,12 @@ class LiveDataCache:
             logger.exception(f"Failed to write snapshot to {self._snapshot_dir}")
 
     def _load_snapshot(self):
+        try:
             data = self._disk_cache.get("snapshot")
+        except Exception:
+            logger.warning(f"Failed to load snapshot from {self._snapshot_dir}, clearing it.")
+            self._disk_cache.clear()
+            return
         if not data:
             return
 
@@ -89,13 +96,19 @@ class LiveDataCache:
 
     def _start_periodic_snapshot(self, interval):
         def loop():
-            while True:
+            while not self._stop_event.wait(timeout=interval):
-                time.sleep(interval)
                 self.save_snapshot()
 
-        t = threading.Thread(target=loop, name=f"LiveDataCache-Snapshot-{self._snapshot_dir}")
+        self._snapshot_thread = threading.Thread(
-        t.start()
+            target=loop, name=f"LiveDataCache-Snapshot-{self._snapshot_dir}", daemon=True
+        )
+        self._snapshot_thread.start()
 
     def close(self):
+        self._stop_event.set()
+        if self._snapshot_thread:
+            self._snapshot_thread.join(timeout=15)
+            if self._snapshot_thread.is_alive():
+                logger.warning(f"LiveDataCache snapshot thread for {self._snapshot_dir} did not exit on time.")
         self.save_snapshot()
         self._disk_cache.close()

core/sig_lookup_helper.py

@@ -24,7 +24,7 @@ def get_sig_ref_info(sig_name, ref_id):
 
     # Sometimes we allow spaces instead of dashes in references due to common usage that way, but official reference
     # lists never do, so convert them here.
-    ref_id.replace(" ", "-")
+    ref_id = ref_id.replace(" ", "-")
 
     # Prepare the object to be returned
     sig_ref = SIGRef(sig=sig_name, id=ref_id)
@@ -114,17 +114,24 @@ def get_sig_ref_info(sig_name, ref_id):
         # OK, this is something we have to look up. Now check to see if our data store contains reference data and if
         # so, copy the data into the sig_ref object
         key = f"{sig_name}:{ref_id}"
+        try:
             lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
             if lookup_data:
-            for key, value in lookup_data.__dict__.items():
+                for attr, value in lookup_data.__dict__.items():
-                if value is not None and sig_ref.__dict__.get(key) is None:
+                    if value is not None and sig_ref.__dict__.get(attr) is None:
-                    sig_ref.__dict__[key] = value
+                        sig_ref.__dict__[attr] = value
 
             else:
                 # Maybe a super new reference we don't know about yet, but more likely a typo or a test reference,
                 # just silently ignore it.
                 logger.debug(f"{sig_name} database did not contain data for ref {ref_id}")
 
+        except (ValueError, KeyError):
+            # Catch exceptions due to e.g. old versions of objects in the cache that are no longer compatible,
+            # and remove them from the cache.
+            del DATA_STORE.sigrefs[key]
+            return None
+
     except Exception:
         logger.exception(f"Exception when looking up sig_ref info for {sig_name} ref {ref_id}")
     return sig_ref

core/single_object_data_cache.py

@@ -21,7 +21,13 @@ class SingleObjectDataCache:
         # This cache stores a single object, doesn't matter what it's called so "object" will do
         if "object" not in self._cache:
             self._cache.add("object", object_if_empty)
+        try:
             self._obj = self._cache.get("object")
+        except Exception:
+            logger.warning(f"Failed to load cache from {cache_dir}, clearing it.")
+            self._cache.clear()
+            self._cache.add("object", object_if_empty)
+            self._obj = object_if_empty
 
     def get(self):
         """Get the data object. This can then be manipulated as necessary across multiple threads. Any function

core/status_reporter.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from datetime import datetime
 from threading import Event, Thread
@@ -11,8 +12,11 @@ from core.constants import SOFTWARE_VERSION
 from core.data_providers import DATA_PROVIDERS
 from core.data_store import DATA_STORE
 from core.prometheus_metrics_handler import alerts_gauge, memory_use_gauge, spots_gauge
+from telnetserver.telnetserver import TELNET_SERVER
 from webserver.webserver import WEB_SERVER
 
+logger = logging.getLogger(__name__)
+
 
 class StatusReporter:
     """Provides a timed update of the application's status data."""
@@ -32,13 +36,17 @@ class StatusReporter:
     def start(self):
         """Start the reporter thread"""
 
-        self._thread = Thread(target=self._run, name="StatusReporter")
+        self._thread = Thread(target=self._run, name="StatusReporter", daemon=True)
         self._thread.start()
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""
 
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("Status reporter worker thread did not exit on time and will be killed.")
 
     def _run(self):
         """Thread entry point: report immediately on startup, then on each interval until stopped"""
@@ -134,19 +142,13 @@ class StatusReporter:
             else 0,
         }
         DATA_STORE.status.get()["webserver"] = {
-            "status": WEB_SERVER.web_server_metrics["status"],
+            "status": WEB_SERVER.web_server_metrics.status,
-            "last_api_access": WEB_SERVER.web_server_metrics["last_api_access_time"]
+            "api_requests_per_hour": WEB_SERVER.web_server_metrics.api_requests_per_hour(),
-            .replace(tzinfo=pytz.UTC)
+            "page_requests_per_hour": WEB_SERVER.web_server_metrics.page_requests_per_hour(),
-            .timestamp()
+            "sse_client_count": WEB_SERVER.sse_client_count,
-            if WEB_SERVER.web_server_metrics["last_api_access_time"]
+        }
-            else 0,
+        DATA_STORE.status.get()["telnet"] = {
-            "api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"],
+            "client_count": TELNET_SERVER.client_count,
-            "last_page_access": WEB_SERVER.web_server_metrics["last_page_access_time"]
-            .replace(tzinfo=pytz.UTC)
-            .timestamp()
-            if WEB_SERVER.web_server_metrics["last_page_access_time"]
-            else 0,
-            "page_access_count": WEB_SERVER.web_server_metrics["page_access_counter"],
         }
         DATA_STORE.status.store()
 

core/url_data_cache.py

@@ -14,14 +14,13 @@ class URLDataCache(CachedSession):
     used across multiple threads, though note that URL lookups will block each other this way, so it is still better to
     create one of these objects per thread if possible."""
 
-    _lock = threading.Lock()
-
     def __init__(self, name):
         super().__init__(
             f"{CACHE_DIR}urls/{name}",
             expire_after=timedelta(days=1),
             allowable_codes=(200, 400, 401, 403, 404),
         )
+        self._lock = threading.Lock()
 
     def get(self, *args, **kwargs):
         with self._lock:

data/alert.py

@@ -158,7 +158,7 @@ class Alert:
                 self.icon = "fa-globe-africa"
             elif self.alert_type == AlertType.CONTEST:
                 self.icon = "fa-trophy"
-            elif self.alert_type == AlertType.CONTEST:
+            elif self.alert_type == AlertType.SATELLITE:
                 self.icon = "fa-satellite"
             elif self.sig_refs and self.sig_refs[0].icon:
                 self.icon = self.sig_refs[0].icon

data/spot.py

@@ -12,6 +12,7 @@ from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 from core.call_lookup_helper import get_call_info
 from core.config import MAX_SPOT_AGE
 from core.constants import PROPAGATION_MODES, SIGS
+from core.data_store import DATA_STORE
 from core.enums import Continent, LocationSourceForSpot, Mode, ModeSource, ModeType
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.sig_lookup_helper import populate_missing_sig_ref_info
@@ -363,10 +364,12 @@ class Spot:
             if self.propagation_mode == "Satellite":
                 if not self.sig:
                     self.sig = "AMSAT"
+                if not any(sig_ref.sig == "AMSAT" for sig_ref in self.sig_refs):
                     self.sig_refs.append(SIGRef(sig="AMSAT"))
             if self.propagation_mode == "Earth-Moon-Earth":
                 if not self.sig:
                     self.sig = "EME"
+                if not any(sig_ref.sig == "EME" for sig_ref in self.sig_refs):
                     self.sig_refs.append(SIGRef(sig="EME"))
 
             # Parse "de_grid -> dx_grid" structures from the comment
@@ -445,6 +448,15 @@ class Spot:
                 elif self.dx_call:
                     self.dx_itu_zone = dx_call_info.itu_zone
 
+            # DXCC lookup from callsign if nothing else has provided it
+            if self.dx_call and not self.dx_dxcc_id:
+                for regex, entity_code in DATA_STORE.dxcc_lookup_by_call_regex:
+                    if regex.pattern and regex.match(self.dx_call):
+                        self.dx_dxcc_id = entity_code
+                        break
+            if self.dx_dxcc_id and not self.dx_flag:
+                self.dx_flag = get_flag_for_dxcc(self.dx_dxcc_id)
+
             # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
             # is likely at home.
             self.dx_location_good = bool(

providers/alert/http_alert_provider.py

@@ -4,7 +4,7 @@ from threading import Event, Thread
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, JSONDecodeError, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from providers.alert.alert_provider import AlertProvider
@@ -27,11 +27,15 @@ class HTTPAlertProvider(AlertProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} alert API every {self._poll_interval!s} seconds.")
-        self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=12)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} alert worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:
@@ -60,9 +64,14 @@ class HTTPAlertProvider(AlertProvider):
                 logger.warning(f"HTTP {http_response.status_code} when calling {self.name} alerts API.")
 
         except ConnectionError:
+            self.status = "Error"
             logger.warning(f"Connection error when accessing {self.name} alerts API.")
         except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
             logger.warning(f"Timeout when accessing {self.name} alerts API.")
+        except JSONDecodeError:
+            self.status = "Error"
+            logger.warning(f"Invalid or empty JSON response from {self.name} alert API.")
         except Exception:
             self.status = "Error"
             logger.exception(f"Exception in HTTP JSON Alert Provider ({self.name})")

providers/callsigndata/callsign_data_provider.py

@@ -42,7 +42,13 @@ class CallsignDataProvider:
 
         if self.enabled:
             if callsign in self._storage:
+                try:
                     return self._storage[callsign]
+                except (ValueError, KeyError):
+                    # Catch exceptions due to e.g. old versions of objects in the cache that are no longer compatible,
+                    # and remove them from the cache.
+                    del self._storage[callsign]
+                    return None
             else:
                 c = self._perform_new_lookup(callsign, lookup_credentials)
                 if c:

providers/callsigndata/file_download_callsign_data_provider.py

@@ -32,11 +32,15 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
-        self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=12)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} callsign data worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/callsigndata/hamqth.py

@@ -124,8 +124,8 @@ class HamQTH(APIQueryCallsignDataProvider):
         lat = None
         lon = None
         if (
-            "latitude" in data
+            data.get("latitude") is not None
-            and "longitude" in data
+            and data.get("longitude") is not None
             and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
             and -89.9 < float(data["latitude"]) < 89.9
         ):
@@ -134,7 +134,7 @@ class HamQTH(APIQueryCallsignDataProvider):
 
         # Check for sensible grids
         grid = None
-        if "grid" in data and not data["grid"].startswith("AA00"):
+        if data.get("grid") and not data["grid"].startswith("AA00"):
             grid = data["grid"]
 
         return Callsign(
@@ -147,8 +147,8 @@ class HamQTH(APIQueryCallsignDataProvider):
             latitude=lat,
             longitude=lon,
             grid=grid,
-            dxcc_id=int(data["adif"]) if "adif" in data else None,
+            dxcc_id=int(data["adif"]) if data.get("adif") is not None else None,
-            cq_zone=int(data["cq"]) if "cq" in data else None,
+            cq_zone=int(data["cq"]) if data.get("cq") is not None else None,
-            itu_zone=int(data["itu"]) if "itu" in data else None,
+            itu_zone=int(data["itu"]) if data.get("itu") is not None else None,
             location_source=LocationSourceForCallsign.HOME_QTH,
         )

providers/callsigndata/qrz.py

@@ -150,8 +150,8 @@ class QRZ(APIQueryCallsignDataProvider):
         lat = None
         lon = None
         if (
-            "latitude" in data
+            data.get("latitude") is not None
-            and "longitude" in data
+            and data.get("longitude") is not None
             and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
             and -89.9 < float(data["latitude"]) < 89.9
         ):
@@ -160,7 +160,7 @@ class QRZ(APIQueryCallsignDataProvider):
 
         # Check for sensible grids
         grid = None
-        if "grid" in data and not data["grid"].startswith("AA00"):
+        if data.get("grid") and not data["grid"].startswith("AA00"):
             grid = data["grid"]
 
         return Callsign(
@@ -173,8 +173,8 @@ class QRZ(APIQueryCallsignDataProvider):
             latitude=lat,
             longitude=lon,
             grid=grid,
-            dxcc_id=int(data["adif"]) if "adif" in data else None,
+            dxcc_id=int(data["adif"]) if data.get("adif") is not None else None,
-            cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
+            cq_zone=int(data["cqzone"]) if data.get("cqzone") is not None else None,
-            itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
+            itu_zone=int(data["ituzone"]) if data.get("ituzone") is not None else None,
             location_source=LocationSourceForCallsign.HOME_QTH,
         )

providers/sigrefdata/dme.py

@@ -53,7 +53,7 @@ class DME(LocalFileSIGRefDataProvider):
 
                 # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
                 # of the data in this case
-                if self._stop:
+                if self._stop_event.is_set():
                     break
 
                 # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time

providers/sigrefdata/file_download_sig_ref_data_provider.py

@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Event, Thread
+from threading import Thread
 
 import pytz
 from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
@@ -21,19 +21,21 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         self._url = url
         self._poll_interval = poll_interval
         self._thread = None
-        self._stop_event = Event()
         self._url_data_cache = URLDataCache(f"sigrefdata_{sig_name}")
 
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
-        self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}")
+        self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         super().stop()
-        self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=12)
+            if self._thread.is_alive():
+                logger.warning(f"{self.sig_name} SIG ref data worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/sigrefdata/sig_ref_data_provider.py

@@ -1,5 +1,6 @@
 import logging
 from datetime import datetime
+from threading import Event
 
 import pytz
 
@@ -19,7 +20,7 @@ class SIGRefDataProvider:
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
         self.reference_count = 0
-        self._stop = False
+        self._stop_event = Event()
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -30,20 +31,22 @@ class SIGRefDataProvider:
         """Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
         super()."""
 
-        self._stop = True
+        self._stop_event.set()
 
     def _add_data(self, new_data):
         """Add all the provided reference data objects to the data store."""
 
-        # with transact() batches all writes together to save making thousands of individual sqlite writes
+        # with transact() batches all writes together to save making thousands of individual sqlite writes. However,
-        with DATA_STORE.sigrefs.transact():
+        # that means that each provider holds the lock while it writes, and the default behaviour for other attempted
+        # transact()s is to fail if they can't get the lock (?!). This behaviour is fixed by retry=True.
+        with DATA_STORE.sigrefs.transact(retry=True):
             for d in new_data:
                 DATA_STORE.sigrefs.set(f"{self.sig_name}:{d.id}", d)
 
                 # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
                 # within the first few minutes of startup, we need a way to abort this expensive process of filling up the
                 # disk cache.
-                if self._stop:
+                if self._stop_event.is_set():
                     break
 
         self.reference_count = len(new_data)

providers/sigrefdata/toilets.py

@@ -35,7 +35,7 @@ class Toilets(LocalFileSIGRefDataProvider):
 
                 # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
                 # of the data in this case
-                if self._stop:
+                if self._stop_event.is_set():
                     break
 
         return new_data

providers/solarconditions/giroionosonde.py

@@ -67,11 +67,15 @@ class GIROIonosonde(SolarConditionsProvider):
 
     def start(self):
         logger.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
-        self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider")
+        self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=12)
+            if self._thread.is_alive():
+                logger.warning("GIRO ionosonde worker thread did not exit on time and will be killed.")
 
     def _run(self):
         # Real interval at which we poll is the "once per hour" divided by the number of stations, so each one gets

providers/solarconditions/http_solar_conditions_provider.py

@@ -25,11 +25,15 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
 
     def start(self):
         logger.info(f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
-        self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=12)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} solar conditions worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/solarconditions/kc2gprop.py

@@ -32,11 +32,15 @@ class KC2GProp(SolarConditionsProvider):
 
     def start(self):
         logger.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
-        self._thread = Thread(target=self._run, name="KC2GPropProvider")
+        self._thread = Thread(target=self._run, name="KC2GPropProvider", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=12)
+            if self._thread.is_alive():
+                logger.warning("KC2G ionosonde worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/spot/aprsis.py

@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Thread
+from threading import Event, Thread
 
 import aprslib
 import pytz
@@ -17,27 +17,44 @@ class APRSIS(SpotProvider):
 
     def __init__(self, provider_config):
         super().__init__("APRS-IS", provider_config)
-        self._thread = Thread(target=self._connect, name="APRSISSpotProvider")
+        self._thread = None
-        self._thread.daemon = True
         self._aprsis = None
+        self._stop_event = Event()
 
     def start(self):
+        self._thread = Thread(target=self._run, name="APRSISSpotProvider", daemon=True)
         self._thread.start()
 
-    def _connect(self):
+    def _run(self):
+        while not self._stop_event.is_set():
+            try:
                 self._aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN)
                 self.status = "Connecting"
                 logger.info("APRS-IS connecting...")
                 self._aprsis.connect()
-        self._aprsis.consumer(self._handle)
                 logger.info("APRS-IS connected.")
+                self._aprsis.consumer(self._handle, immortal=True)
+
+            except Exception:
+                if not self._stop_event.is_set():
+                    self.status = "Error"
+                    logger.exception("Exception in APRS-IS provider")
+
+            if not self._stop_event.is_set():
+                self._stop_event.wait(timeout=5)
 
     def stop(self):
         self.status = "Shutting down"
+        self._stop_event.set()
+        if self._aprsis:
             self._aprsis.close()
-        self._thread.join()
+        if self._thread:
+            self._thread.join(timeout=5)
+            if self._thread.is_alive():
+                logger.warning("APRS-IS worker thread did not exit on time and will be killed.")
 
     def _handle(self, data):
+        try:
             # Split SSID in "from" call and store separately
             from_parts = str(data["from"]).split("-")
             dx_call = from_parts[0].upper()
@@ -52,8 +69,8 @@ class APRSIS(SpotProvider):
                 de_call=de_call,
                 de_ssid=de_ssid,
                 comment=str(data["comment"]) if "comment" in data else None,
-            dx_latitude=float(data["latitude"]) if "latitude" in data else None,
+                dx_latitude=float(data["latitude"]) if data.get("latitude") is not None else None,
-            dx_longitude=float(data["longitude"]) if "longitude" in data else None,
+                dx_longitude=float(data["longitude"]) if data.get("longitude") is not None else None,
                 time=datetime.now(pytz.UTC).timestamp(),
             )  # APRS-IS spots are live so we can assume spot time is "now"
 
@@ -63,3 +80,6 @@ class APRSIS(SpotProvider):
             self.status = "OK"
             self.last_update_time = datetime.now(pytz.UTC)
             logger.debug("Data received from APRS-IS.")
+
+        except Exception:
+            logger.exception("Exception handling APRS-IS packet")

providers/spot/dxcluster.py

@@ -1,8 +1,8 @@
 import logging
 import re
+import socket
 from datetime import datetime
-from threading import Thread
+from threading import Event, Lock, Thread
-from time import sleep
 
 import pytz
 import telnetlib3
@@ -41,27 +41,43 @@ class DXCluster(SpotProvider):
             self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
         )
         self._telnet = None
-        self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}")
+        self._telnet_lock = Lock()
-        self._thread.daemon = True
+        self._thread = None
-        self._running = True
+        self._stop_event = Event()
 
     def start(self):
+        self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
-        self._running = False
+        self._stop_event.set()
+        with self._telnet_lock:
             if self._telnet:
+                try:
+                    self._telnet.sock.shutdown(socket.SHUT_RDWR)
+                except (AttributeError, OSError):
+                    pass
                 self._telnet.close()
-        self._thread.join()
+        if self._thread:
+            self._thread.join(timeout=5)
+            if self._thread.is_alive():
+                logger.warning(f"DX Cluster {self._hostname} worker thread did not exit on time and will be killed.")
 
     def _handle(self):
-        while self._running:
+        while not self._stop_event.is_set():
             connected = False
-            while not connected and self._running:
+            while not connected and not self._stop_event.is_set():
                 try:
                     self.status = "Connecting"
                     logger.info(f"DX Cluster {self._hostname} connecting...")
-                    self._telnet = telnetlib3.Telnet(self._hostname, self._port)
+                    new_telnet = telnetlib3.Telnet(self._hostname, self._port)
+                    with self._telnet_lock:
+                        self._telnet = new_telnet
+                        if self._stop_event.is_set():
+                            # stop() was called while we were connecting, close the connection rather than trying to
+                            # read when we know it won't work
+                            new_telnet.close()
+                            break
                     self._telnet.read_until(self._login_prompt.encode("latin-1"))
                     self._telnet.write(f"{self._login_callsign}\n".encode("latin-1"))
                     connected = True
@@ -69,14 +85,14 @@ class DXCluster(SpotProvider):
                 except ConnectionRefusedError:
                     self.status = "Error"
                     logger.warning(f"Connection refused to DX cluster {self._hostname}")
-                    sleep(300)
+                    self._stop_event.wait(timeout=300)
                 except Exception:
                     self.status = "Error"
                     logger.exception(f"Exception while connecting to DX Cluster Provider ({self._hostname}).")
-                    sleep(5)
+                    self._stop_event.wait(timeout=5)
 
             self.status = "Waiting for Data"
-            while connected and self._running:
+            while connected and not self._stop_event.is_set():
                 try:
                     # Check new telnet info against regular expression
                     telnet_output = self._telnet.read_until("\n".encode("latin-1"))
@@ -106,19 +122,19 @@ class DXCluster(SpotProvider):
 
                 except EOFError:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Restarting"
                         logger.warning(f"Disconnected from DX Cluster {self._hostname}. Reconnecting...")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"DX Cluster {self._hostname} shutting down...")
                         self.status = "Shutting down"
                 except Exception:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Error"
                         logger.exception(f"Exception in DX Cluster Provider ({self._hostname})")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"DX Cluster {self._hostname} shutting down...")
                         self.status = "Shutting down"

providers/spot/http_spot_provider.py

@@ -4,7 +4,7 @@ from threading import Event, Thread
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, JSONDecodeError, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from providers.spot.spot_provider import SpotProvider
@@ -28,12 +28,16 @@ class HTTPSpotProvider(SpotProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} spot API every {self._poll_interval!s} seconds.")
-        self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
         self._wakeup_event.set()
+        if self._thread:
+            self._thread.join(timeout=12)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} spot worker thread did not exit on time and will be killed.")
 
     def force_poll(self):
         """Trigger an immediate poll without waiting for the normal interval."""
@@ -69,9 +73,14 @@ class HTTPSpotProvider(SpotProvider):
                 logger.warning(f"HTTP {http_response.status_code} when calling {self.name} spot API.")
 
         except ConnectionError:
+            self.status = "Error"
             logger.warning(f"Connection error when accessing {self.name} spots API.")
         except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
             logger.warning(f"Timeout when accessing {self.name} spots API.")
+        except JSONDecodeError:
+            self.status = "Error"
+            logger.warning(f"Invalid or empty JSON response from {self.name} spots API.")
         except Exception:
             self.status = "Error"
             logger.exception(f"Exception in HTTP Spot Provider ({self.name})")

providers/spot/rbn.py

@@ -1,8 +1,8 @@
 import logging
 import re
+import socket
 from datetime import datetime
-from threading import Thread
+from threading import Event, Lock, Thread
-from time import sleep
 
 import pytz
 import telnetlib3
@@ -30,27 +30,43 @@ class RBN(SpotProvider):
         super().__init__(name, provider_config)
         self._port = provider_config["port"]
         self._telnet = None
-        self._thread = Thread(target=self._handle, name=f"RBNSpotProvider-{self.name}")
+        self._telnet_lock = Lock()
-        self._thread.daemon = True
+        self._thread = None
-        self._running = True
+        self._stop_event = Event()
 
     def start(self):
+        self._thread = Thread(target=self._handle, name=f"RBNSpotProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
-        self._running = False
+        self._stop_event.set()
+        with self._telnet_lock:
             if self._telnet:
+                try:
+                    self._telnet.sock.shutdown(socket.SHUT_RDWR)
+                except (AttributeError, OSError):
+                    pass
                 self._telnet.close()
-        self._thread.join()
+        if self._thread:
+            self._thread.join(timeout=5)
+            if self._thread.is_alive():
+                logger.warning(f"RBN (port {self._port!s}) worker thread did not exit on time and will be killed.")
 
     def _handle(self):
-        while self._running:
+        while not self._stop_event.is_set():
             connected = False
-            while not connected and self._running:
+            while not connected and not self._stop_event.is_set():
                 try:
                     self.status = "Connecting"
                     logger.info(f"RBN port {self._port!s} connecting...")
-                    self._telnet = telnetlib3.Telnet("telnet.reversebeacon.net", self._port)
+                    new_telnet = telnetlib3.Telnet("telnet.reversebeacon.net", self._port)
+                    with self._telnet_lock:
+                        self._telnet = new_telnet
+                        if self._stop_event.is_set():
+                            # stop() was called while we were connecting, close the connection rather than trying to
+                            # read when we know it won't work
+                            new_telnet.close()
+                            break
                     self._telnet.read_until("Please enter your call: ".encode("latin-1"))
                     self._telnet.write(f"{SERVER_OWNER_CALLSIGN}\n".encode("latin-1"))
                     connected = True
@@ -58,10 +74,10 @@ class RBN(SpotProvider):
                 except Exception:
                     self.status = "Error"
                     logger.exception(f"Exception while connecting to RBN (port {self._port!s}).")
-                    sleep(5)
+                    self._stop_event.wait(timeout=5)
 
             self.status = "Waiting for Data"
-            while connected and self._running:
+            while connected and not self._stop_event.is_set():
                 try:
                     # Check new telnet info against regular expression
                     telnet_output = self._telnet.read_until("\n".encode("latin-1"))
@@ -91,19 +107,19 @@ class RBN(SpotProvider):
 
                 except EOFError:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Restarting"
                         logger.warning(f"Disconnected from RBN provider (port {self._port!s}). Reconnecting...")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"RBN provider (port {self._port!s}) shutting down...")
                         self.status = "Shutting down"
                 except Exception:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Error"
                         logger.exception(f"Exception in RBN provider (port {self._port!s})")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"RBN provider (port {self._port!s}) shutting down...")
                         self.status = "Shutting down"

providers/spot/sse_spot_provider.py

@@ -42,7 +42,7 @@ class SSESpotProvider(SpotProvider):
                 logger.exception(f"Exception closing SSE connection for {self.name} during stop()")
 
         if self._thread:
-            self._thread.join(timeout=15)
+            self._thread.join(timeout=5)
             if self._thread.is_alive():
                 logger.warning(f"{self.name} SSE worker thread did not exit on time and will be killed.")
 

providers/spot/ukpacketnet.py

@@ -42,7 +42,7 @@ class UKPacketNet(HTTPSpotProvider):
                                 )
                                 comment = (
                                     f"{comment} {listed_port['baud']!s} baud"
-                                    if "baud" in listed_port and listed_port["baud"] > 0
+                                    if listed_port.get("baud") and listed_port["baud"] > 0
                                     else comment
                                 )
 
@@ -50,7 +50,7 @@ class UKPacketNet(HTTPSpotProvider):
                                 # very hacky but a lot of node comments contain their frequency as the first or second
                                 # word of their comment, but not in the proper data structure field.
                                 freq = (
-                                    listed_port["freq"] if "freq" in listed_port and listed_port["freq"] > 0 else None
+                                    listed_port["freq"] if listed_port.get("freq") and listed_port["freq"] > 0 else None
                                 )
                                 if not freq and comment:
                                     possible_freq = comment.split(" ")[0].upper().replace("MHZ", "")

providers/spot/websocket_spot_provider.py

@@ -1,7 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Thread
+from threading import Event, Thread
-from time import sleep
 
 import pytz
 from websocket import create_connection
@@ -20,22 +19,24 @@ class WebsocketSpotProvider(SpotProvider):
         self._url = url
         self._ws = None
         self._thread = None
-        self._stopped = False
+        self._stop_event = Event()
         self._last_event_id = None
 
     def start(self):
         logger.info(f"Set up websocket connection to {self.name} spot API.")
-        self._stopped = False
+        self._stop_event.clear()
         self._thread = Thread(target=self._run, name=f"WebsocketSpotProvider-{self.name}")
         self._thread.daemon = True
         self._thread.start()
 
     def stop(self):
-        self._stopped = True
+        self._stop_event.set()
         if self._ws:
             self._ws.close()
         if self._thread:
-            self._thread.join()
+            self._thread.join(timeout=5)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} websocket worker thread did not exit on time and will be killed.")
 
     def _on_open(self):
         self.status = "Waiting for Data"
@@ -44,14 +45,19 @@ class WebsocketSpotProvider(SpotProvider):
         self.status = "Connecting"
 
     def _run(self):
-        while not self._stopped:
+        while not self._stop_event.is_set():
             try:
                 logger.debug(f"Connecting to {self.name} spot API...")
                 self.status = "Connecting"
                 self._ws = create_connection(self._url, header=HTTP_HEADERS)
                 self.status = "Connected"
+
+                # Keep reading from this same connection until it drops or we're asked to stop, rather than
+                # reconnecting for every message.
+                while not self._stop_event.is_set():
                     data = self._ws.recv()
-                if data:
+                    if not data:
+                        break
                     try:
                         new_spot = self._ws_message_to_spot(data)
                         if new_spot:
@@ -69,7 +75,16 @@ class WebsocketSpotProvider(SpotProvider):
                 logger.exception(f"Exception in Websocket Spot Provider ({self.name})")
             else:
                 self.status = "Disconnected"
-            sleep(5)  # Wait before trying to reconnect
+            finally:
+                if self._ws:
+                    try:
+                        self._ws.close()
+                    except Exception:
+                        # No problem, we were getting rid of this object anyway.
+                        pass
+                    self._ws = None
+            if not self._stop_event.is_set():
+                self._stop_event.wait(timeout=5)  # Wait before trying to reconnect
 
     def _ws_message_to_spot(self, b):
         """Convert a WS message received from the API into a spot. The exact message data (in bytes) is provided here so the

providers/spot/wwbota.py

@@ -36,7 +36,7 @@ class WWBOTA(SSESpotProvider):
             dx_call=source_spot["call"].upper(),
             de_call=source_spot["spotter"].upper(),
             freq=float(source_spot["freq"]) * 1000000,
-            mode=Mode.from_name(source_spot["mode"].upper()) if "mode" in source_spot else None,
+            mode=Mode.from_name(source_spot["mode"].upper()) if source_spot.get("mode") else None,
             comment=source_spot["comment"],
             sig="WWBOTA",
             sig_refs=refs,

providers/staticdata/file_download_static_data_provider.py

@@ -29,11 +29,15 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.")
-        self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=12)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} static data worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "spothole"
-version = "2.1-pre"
+version = "2.1.3"
 authors = [
     { name = "Ian Renton", email = "ian@ianrenton.com" },
 ]

spothole.py

@@ -15,17 +15,25 @@ from webserver.webserver import WEB_SERVER
 
 logger = logging.getLogger(__name__)
 
+_shutdown_in_progress = False
+
 
 def shutdown(_signum=None, _frame=None):
     """Shutdown function"""
 
+    # Check if this is the second time a shutdown was asked for, if so immediately kill the program.
+    global _shutdown_in_progress
+    if _shutdown_in_progress:
+        os._exit(1)
+    _shutdown_in_progress = True
+
     logger.info("Stopping program...")
     WEB_SERVER.stop()
     TELNET_SERVER.stop()
     DATA_PROVIDERS.stop()
     CLEANUP_TIMER.stop()
     DATA_STORE.close()
-    os._exit(0)
+    logger.info("Stopped.")
 
 
 # Main function
@@ -43,8 +51,9 @@ if __name__ == "__main__":
     logger.info("Starting...")
     logger.info(f"This is Spothole version {SOFTWARE_VERSION}. This instance is run by {SERVER_OWNER_CALLSIGN}.")
 
-    # Shut down gracefully on SIGINT
+    # Shut down gracefully on SIGINT or SIGTERM
     signal.signal(signal.SIGINT, shutdown)
+    signal.signal(signal.SIGTERM, shutdown)
 
     # Set up data store
     DATA_STORE.setup()
@@ -59,14 +68,16 @@ if __name__ == "__main__":
     status_reporter = StatusReporter(run_interval=5)
     status_reporter.start()
 
-    # Set up the web server
-    WEB_SERVER.setup()
-
     # Run the telnet server
     if TELNET_SERVER_ENABLED:
         TELNET_SERVER.start(port=TELNET_SERVER_PORT)
 
-    # Run the web server. This is the blocking call that keeps the application running in the main thread, so this must
+    # Set up the web server
-    # be the last thing we do. web_server.stop() triggers an await condition in the web server which finishes the main
+    WEB_SERVER.setup()
-    # thread.
+
+    # Run the web server
     WEB_SERVER.start()
+
+    # Block the main thread until a termination signal arrives and shutdown() is running.
+    while not _shutdown_in_progress:
+        signal.pause()

static/apidocs/openapi.yml

@@ -25,6 +25,8 @@ info:
     * Added `contests_skip_max_duration_check` to alert query parameters
     * SIG reference types (e.g. "Park") are now capitalised to match other enums
     * Added the ability to get only certain fields of spots and alerts from the API by using the `fields` query parameter.
+    * Replace `last_page_access` with `page_requests_per_hour` and `last_api_access` with `api_requests_per_hour` in the web server stats.
+    * Added `sse_client_count` to the `webserver` stats, and a new `telnet` object with `client_count`.
 
     ### 2.0
 
@@ -899,7 +901,12 @@ components:
         - WAB
         - WAI
         - DME
+        - DMF
         - FEA
+        - DMUE
+        - DMVE
+        - DCE
+        - DEFE
         - DTMBA
         - BIWOTA
         - COTA
@@ -2100,14 +2107,25 @@ components:
               type: string
               description: The status of the web server
               example: OK
-            last_page_access:
+            page_requests_per_hour:
-              type: number
+              type: integer
-              description: The last time a page was accessed on the web server, UTC seconds since UNIX epoch.
+              description: The number of page requests handled by the web server in the last hour
-              example: 1759579508
+              example: 123
-            last_api_access:
+            api_requests_per_hour:
-              type: number
+              type: integer
-              description: The last time an API endpoint was accessed on the web server, UTC seconds since UNIX epoch.
+              description: The number of API requests handled by the web server in the last hour
-              example: 1759579508
+              example: 123
+            sse_client_count:
+              type: integer
+              description: The number of clients currently connected to SSE streams
+              example: 5
+        "telnet":
+          type: object
+          properties:
+            client_count:
+              type: integer
+              description: The number of clients currently connected to the telnet server
+              example: 2
         spot_providers:
           type: array
           description: An array of all the spot providers.

static/css/style.css

@@ -414,8 +414,8 @@ div.band-spot:hover span.band-spot-info {
 
     /* Make map stretch to horizontal screen edges */
     div#map, div#table-container, div#bands-container {
-        margin-left: -1em;
+        margin-left: -0.75rem;
-        margin-right: -1em;
+        margin-right: -0.75rem;
     }
 
     /* Avoid map page filters panel being larger than the map itself */

static/js/status.js

@@ -9,8 +9,10 @@ function loadStatus() {
         $("#total-alerts").text(jsonData["num_alerts"]);
 
         $("#web-server-status").text(jsonData["webserver"]["status"]);
-        $("#web-server-last-api").text(moment.unix(jsonData["webserver"]["last_api_access"]).utc().fromNow());
+        $("#web-server-api-rate").text(jsonData["webserver"]["api_requests_per_hour"] + " / hour");
-        $("#web-server-last-page").text(moment.unix(jsonData["webserver"]["last_page_access"]).utc().fromNow());
+        $("#web-server-page-rate").text(jsonData["webserver"]["page_requests_per_hour"] + " / hour");
+        $("#web-server-sse-clients").text(jsonData["webserver"]["sse_client_count"]);
+        $("#telnet-server-clients").text(jsonData["telnet"]["client_count"]);
 
         $("#cleanup-status").text(jsonData["cleanup"]["status"]);
         $("#cleanup-last-ran").text((jsonData["cleanup"]["last_ran"] > 0) ? moment.unix(jsonData["cleanup"]["last_ran"]).utc().fromNow() : "N/A");

telnetserver/telnetserver.py

@@ -4,6 +4,7 @@ import threading
 from datetime import datetime
 
 import pytz
+from pyhamtools import callinfo
 
 from core.config import SERVER_OWNER_CALLSIGN
 from core.constants import SOFTWARE_VERSION
@@ -12,6 +13,33 @@ from data.spot import Spot
 
 logger = logging.getLogger(__name__)
 
+BANNER = (
+    """
+           ==                                                                  
+          ====                      #####    #####               ####          
+         ==  ==                    ..###    ..###               ..###          
+        == ###==########   ######  #######   .#######    ######  .###   ###### 
+       ==###.. ..###..### ###..###...###.    .###..###  ###..### .###  ###..###
+      ==###.### .### .###.### .###  .###     .### .### .### .### .### .####### 
+     == .###.###.### .###.### .###  .### ### .### .### .### .### .### .###...  
+    ==   ...### .####### ..######   ..#####  #### #####..######  #####..###### 
+   ==    .###.  .###=..   ......     .....  .... .....  ......  .....  ......  
+  == %%  ... %%%%###==                                                         
+ ==  %%%%%%%%%%%#####==                                                        \r\n"""
+    + "==             .....  =="
+    + f"Welcome to Spothole v{SOFTWARE_VERSION}".rjust(56)
+    + "\r\n"
+    + "========================"
+    + f"This server is run by {SERVER_OWNER_CALLSIGN}".rjust(56)
+)
+
+MOTD = (
+    "Spothole's telnet server is a new feature and may not work properly in all\r\n"
+    + "loggers. Please give it a try in your logger of choice and let me know if it\r\n"
+    + "(or doesn't!) Please note that DXSpider-like commands are not yet supported\r\n"
+    + "so you are not yet able to filter spots server-side or log in at all."
+)
+
 
 class TelnetServer:
     """A telnet server designed to provide spots in the same format as DXSpider, for compatibility with desktop loggers."""
@@ -21,22 +49,20 @@ class TelnetServer:
         self._running = False
         self._clients = set()
         self._loop = None
+        self._thread = None
+        self._shutdown_event = asyncio.Event()
 
     def start(self, port=7373):
         """Starts the telnet server"""
 
         self._port = port
 
-        # Start telnet server on the async loop
+        # Start the telnet server. asyncio.run() needs a coroutine, and threading.Thread needs a plain callable, so
-        def run_loop():
+        # hand Thread the bridge between the two directly rather than writing a one-line wrapper method for it.
-            self._loop = asyncio.new_event_loop()
+        self._thread = threading.Thread(
-            asyncio.set_event_loop(self._loop)
+            target=asyncio.run, args=(self._start_internal(),), name="TelnetServer", daemon=True
-            self._loop.run_until_complete(self._start_internal())
+        )
-            self._loop.run_forever()
+        self._thread.start()
-
-        # Start the network thread as a daemon so it exits cleanly when the main script stops
-        t = threading.Thread(target=run_loop, daemon=True)
-        t.start()
         logger.debug("Telnet server background thread spawned")
 
         # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
@@ -44,8 +70,16 @@ class TelnetServer:
         self._running = True
 
     async def _start_internal(self):
+        """Start method (async). Sets up the telnet server and waits for shutdown."""
+
+        self._loop = asyncio.get_running_loop()
+
+        server = await asyncio.start_server(self._handle_client, "0.0.0.0", self._port)
         logger.info(f"Telnet server listening on port {self._port}")
-        await asyncio.start_server(self._handle_client, "0.0.0.0", self._port)
+        async with server:
+            await self._shutdown_event.wait()
+
+        await self._stop_internal()
 
     async def _handle_client(self, reader, writer):
         """Handles a new client connection"""
@@ -53,15 +87,22 @@ class TelnetServer:
         logger.debug("Telnet client connected")
         self._clients.add(writer)
 
-        # Print MOTD
+        # Print banner and MOTD
         try:
-            motd = (
+            text = (
-                f"Welcome to Spothole v{SOFTWARE_VERSION}.\r\nThis server is run by {SERVER_OWNER_CALLSIGN}.\r\n===\r\n"
+                BANNER
+                + "\r\n\r\n"
+                + "================================================================================"
+                + "\r\n"
+                + MOTD
+                + "\r\n"
+                + "================================================================================"
+                + "\r\n\r\n"
             )
-            writer.write(motd.encode("ascii"))
+            writer.write(text.encode("ascii"))
             await writer.drain()
         except Exception:
-            pass
+            logger.exception("Exception printing telnet motd")
 
         # Set up buffer for user input
         input_buffer = ""
@@ -73,26 +114,13 @@ class TelnetServer:
                 if not data:
                     break
 
-                text = data.decode("ascii", errors="ignore")
+                input_buffer, command = self._consume_input(input_buffer, data)
-
-                for char in text:
-                    if char in ("\r", "\n"):
-                        # User pressed Enter, evaluate the command
-                        command = input_buffer.strip().lower()
-                        input_buffer = ""
-
                 if command == "exit":
                     writer.write(b"Goodbye!\r\n")
                     await writer.drain()
                     # Exit the while read loop, this will disconnect the client.
                     return
 
-                    elif char in ("\b", "\x7f"):
-                        # Handle backspaces
-                        input_buffer = input_buffer[:-1]
-                    else:
-                        input_buffer += char
-
         except asyncio.CancelledError:
             pass
         except Exception:
@@ -106,9 +134,18 @@ class TelnetServer:
     def stop(self):
         """Stops the telnet server"""
 
+        self._running = False
         if self._loop and self._loop.is_running():
             logger.debug("Stopping telnet server...")
-            self._loop.call_soon_threadsafe(self._loop.stop)
+            self._loop.call_soon_threadsafe(self._shutdown_event.set)
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("Telnet server background thread did not exit on time and will be killed.")
+
+    @property
+    def client_count(self) -> int:
+        return len(self._clients)
 
     async def _stop_internal(self):
         """Stops the telnet server"""
@@ -133,9 +170,10 @@ class TelnetServer:
         # Ensure ASCII formatting for telnet clients
         encoded_line = self._format_dxspider_spot(spot).encode("ascii", errors="ignore")
 
-        # Try to write to all clients, and in the process find the ones that are disconnected
+        # Try to write to all clients, and in the process find the ones that are disconnected. Iterate over a copy
+        # since a new client can connect (mutating self._clients) while we're awaiting a write below.
         disconnected_clients = set()
-        for writer in self._clients:
+        for writer in list(self._clients):
             try:
                 writer.write(encoded_line)
                 await writer.drain()
@@ -146,15 +184,36 @@ class TelnetServer:
         for writer in disconnected_clients:
             self._clients.discard(writer)
 
+    @staticmethod
+    def _consume_input(input_buffer: str, data: bytes) -> tuple[str, str | None]:
+        """Handle any input the user gives us, keeping a rolling buffer that we keep passing back through and
+        adding to. Once we get a command, return that as well, so the caller can deal with it."""
+
+        command = None
+        for char in data.decode("ascii", errors="ignore"):
+            if char in ("\r", "\n"):
+                stripped = input_buffer.strip().lower()
+                if stripped:
+                    command = stripped
+                input_buffer = ""
+            elif char in ("\b", "\x7f"):
+                # Handle backspaces
+                input_buffer = input_buffer[:-1]
+            else:
+                input_buffer += char
+        return input_buffer, command
+
     @staticmethod
     def _format_dxspider_spot(spot: Spot) -> str:
         """Formats a spot into the format DXspider uses:
-        DX de CALLSIGN:  FREQUENCY  DX_CALLSIGN  COMMENTS   TIME_UTC"""
+        DX de CALLSIGN:  FREQUENCY  DX_CALLSIGN  COMMENTS   TIME_UTC.
+        Always use the base call for the spotter to save space. Everything must align properly to parse in clients,
+        and everything must be renderable in ASCII."""
 
-        de_call = f"{spot.de_call + ':' if spot.de_call else '???:'!s:<9}"
+        de_call = f"{callinfo.Callinfo.get_homecall(spot.de_call)[:6] + ':' if spot.de_call else '???:'!s:<7}"
-        frequency = f"{(spot.freq / 1000.0):8.1f}"
+        frequency = f"{(spot.freq / 1000.0):10.1f}"
         dx_call = f"{spot.dx_call!s:<12}"
-        comment = f"{spot.comment[:29]:<30}"
+        comment = f"{spot.comment.encode('ascii', errors='ignore').decode()[:29]:<30}"
         if spot.time:
             timestamp = datetime.fromtimestamp(spot.time, tz=pytz.utc).strftime("%H%M") + "Z"
         else:

templates/about.html

@@ -114,7 +114,7 @@
         Faros de España (FEA), Diploma Muesos de España (DMUE), Diploma Castillos de España (DCE), Diploma Monumentos y
         Vestigios de España (DMVE), Diploma Estaciones de Ferrocarril de España (DEFE), Diploma Teatri Musei e Belle
         Arti (DTMBA), British Inland Waterways on the Air (BIWOTA), Castles on the Air (COTA), Polish Gmina Award (PGA),
-        EME/Moonbounce, Amateur Satellite (AMSAT), and Toilets on the Air.</p>
+        Diplôme des Moulins de France (DMF), EME/Moonbounce, Amateur Satellite (AMSAT), and Toilets on the Air.</p>
     <p>As of the time of writing in August 2026, I think Spothole captures most outdoor radio programmes that have a
         defined, downloadable reference list, and almost certainly those that have a spotting/alerting API. If you know
         of one I've missed, please let me know!</p>

templates/add_spot.html

@@ -77,7 +77,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1789140190"></script>
+<script src="/static/js/add-spot.js?v=1789894090"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -83,7 +83,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1789140190"></script>
+<script src="/static/js/alerts.js?v=1789894090"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -76,8 +76,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1789140190"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1789894090"></script>
-<script src="/static/js/bands.js?v=1789140190"></script>
+<script src="/static/js/bands.js?v=1789894090"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1789140190" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1789894090" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -16,10 +16,10 @@
     window.fetchEventSource = fetchEventSource;
 </script>
 
-<script src="/static/js/utils.js?v=1789140190"></script>
+<script src="/static/js/utils.js?v=1789894090"></script>
-<script src="/static/js/ui-ham.js?v=1789140190"></script>
+<script src="/static/js/ui-ham.js?v=1789894090"></script>
-<script src="/static/js/geo.js?v=1789140190"></script>
+<script src="/static/js/geo.js?v=1789894090"></script>
-<script src="/static/js/common.js?v=1789140190"></script>
+<script src="/static/js/common.js?v=1789894090"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1789140190"></script>
+<script src="/static/js/conditions.js?v=1789894090"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -113,8 +113,8 @@
     const CARTODB_API_KEY = "{{ web_ui_options.get('cartodb_api_key', '') }}";
 </script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1789140190"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1789894090"></script>
-<script src="/static/js/map.js?v=1789140190"></script>
+<script src="/static/js/map.js?v=1789894090"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -125,8 +125,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1789140190"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1789894090"></script>
-<script src="/static/js/spots.js?v=1789140190"></script>
+<script src="/static/js/spots.js?v=1789894090"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -21,9 +21,19 @@
         <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
             <div class="col"><strong>Web Server</strong></div>
             <div class="col">Status: <span id="web-server-status"></span></div>
-            <div class="col">Last API call: <span id="web-server-last-api"></span></div>
+            <div class="col">SSE clients connected: <span id="web-server-sse-clients"></span></div>
-            <div class="col">Last page req: <span id="web-server-last-page"></span></div>
         </div>
+        <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
+            <div class="col"></div>
+            <div class="col">API request rate: <span id="web-server-api-rate"></span></div>
+            <div class="col">Page request rate: <span id="web-server-page-rate"></span></div>
+        </div>
+        {% if telnet_server_enabled %}
+        <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
+            <div class="col"><strong>Telnet Server</strong></div>
+            <div class="col">Clients connected: <span id="telnet-server-clients"></span></div>
+        </div>
+        {% end %}
         <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
             <div class="col"><strong>Cleanup Service</strong></div>
             <div class="col">Status: <span id="cleanup-status"></span></div>
@@ -86,7 +96,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1789140190"></script>
+<script src="/static/js/status.js?v=1789894090"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");

webserver/handlers/api/addspot.py

@@ -1,10 +1,8 @@
 import logging
 import re
 import threading
-from datetime import datetime
 from typing import Any
 
-import pytz
 import requests
 import tornado
 from tornado import httputil
@@ -12,7 +10,6 @@ from tornado.web import Application
 
 from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRET_KEY
 from core.constants import UNKNOWN_BAND
-from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import infer_band_from_freq, safe_json_dumps
 from data.spot import Spot
@@ -33,23 +30,15 @@ class APISpotHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._spots = None
-        self._web_server_metrics = None
         self._spot_providers = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, spots, web_server_metrics, spot_providers=None):
+    def initialize(self, spots, spot_providers=None):
         self._spots = spots
-        self._web_server_metrics = web_server_metrics
         self._spot_providers = spot_providers or []
 
     def post(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # Reject if not allowed
             if not ALLOW_SPOTTING:
                 self.set_status(401)

webserver/handlers/api/alerts.py

@@ -10,7 +10,6 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.enums import AlertType
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 
@@ -27,12 +26,10 @@ class APIAlertsHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._alerts = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, alerts, web_server_metrics):
+    def initialize(self, alerts):
         self._alerts = alerts
-        self._web_server_metrics = web_server_metrics
 
     @staticmethod
     def _enrich(alerts, credentials):
@@ -45,12 +42,6 @@ class APIAlertsHandler(tornado.web.RequestHandler):
 
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
@@ -82,15 +73,13 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
 
     def __init__(self, application, request, **kwargs: Any):
         self._sse_alert_broadcaster = None
-        self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
         self._fields = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, sse_alert_broadcaster, web_server_metrics):
+    def initialize(self, sse_alert_broadcaster):
         self._sse_alert_broadcaster = sse_alert_broadcaster
-        self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
         """Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data"""
@@ -99,12 +88,6 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
 
     def open(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}

webserver/handlers/api/dxstats.py

@@ -11,7 +11,6 @@ from tornado.web import Application
 
 from core.constants import BANDS
 from core.enums import Continent
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 
 logger = logging.getLogger(__name__)
@@ -30,20 +29,13 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._spots = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, spots, web_server_metrics):
+    def initialize(self, spots):
         self._spots = spots
-        self._web_server_metrics = web_server_metrics
 
     def get(self):
         try:
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
             counts = Counter()
 

webserver/handlers/api/lookups.py

@@ -1,9 +1,7 @@
 import logging
 import re
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
@@ -15,7 +13,6 @@ from core.geo_utils import (
     lat_lon_to_cq_zone,
     lat_lon_to_itu_zone,
 )
-from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import safe_json_dumps
@@ -34,20 +31,10 @@ class APILookupCallHandler(tornado.web.RequestHandler):
         request: httputil.HTTPServerRequest,
         **kwargs: Any,
     ):
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, web_server_metrics):
-        self._web_server_metrics = web_server_metrics
-
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
@@ -85,20 +72,10 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
         request: httputil.HTTPServerRequest,
         **kwargs: Any,
     ):
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, web_server_metrics):
-        self._web_server_metrics = web_server_metrics
-
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
@@ -143,20 +120,10 @@ class APILookupGridHandler(tornado.web.RequestHandler):
         request: httputil.HTTPServerRequest,
         **kwargs: Any,
     ):
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, web_server_metrics):
-        self._web_server_metrics = web_server_metrics
-
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}

webserver/handlers/api/options.py

@@ -1,8 +1,6 @@
 import logging
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
@@ -10,7 +8,6 @@ from tornado.web import Application
 from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
 from core.constants import BANDS, PROPAGATION_MODES, SIGS
 from core.enums import Continent, Mode, ModeType
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 
 logger = logging.getLogger(__name__)
@@ -26,23 +23,15 @@ class APIOptionsHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._status_data = None
-        self._web_server_metrics = None
         self._spot_providers = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, status_data, web_server_metrics, spot_providers=None):
+    def initialize(self, status_data, spot_providers=None):
         self._status_data = status_data
-        self._web_server_metrics = web_server_metrics
         self._spot_providers = spot_providers or []
 
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # Build a map of SIG name -> list of provider names that can submit spots for that SIG
             spot_submit_providers = {}
 

webserver/handlers/api/solar_conditions.py

@@ -1,13 +1,10 @@
 import logging
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
 
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 
 logger = logging.getLogger(__name__)
@@ -23,21 +20,13 @@ class APISolarConditionsHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._solar_conditions = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, solar_conditions, web_server_metrics):
+    def initialize(self, solar_conditions):
         self._solar_conditions = solar_conditions
-        self._web_server_metrics = web_server_metrics
 
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             self.write(self._solar_conditions.to_json())
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")

webserver/handlers/api/spots.py

@@ -9,7 +9,6 @@ import tornado_eventsource.handler
 from tornado import httputil
 from tornado.web import Application
 
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 
@@ -26,12 +25,10 @@ class APISpotsHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._spots = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, spots, web_server_metrics):
+    def initialize(self, spots):
         self._spots = spots
-        self._web_server_metrics = web_server_metrics
 
     @staticmethod
     def _enrich(spots, credentials):
@@ -44,12 +41,6 @@ class APISpotsHandler(tornado.web.RequestHandler):
 
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
@@ -81,15 +72,13 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
 
     def __init__(self, application, request, **kwargs: Any):
         self._sse_spot_broadcaster = None
-        self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
         self._fields = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, sse_spot_broadcaster, web_server_metrics):
+    def initialize(self, sse_spot_broadcaster):
         self._sse_spot_broadcaster = sse_spot_broadcaster
-        self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
         """Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data"""
@@ -100,12 +89,6 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
         """Called once on the client opening a connection, set things up"""
 
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}

webserver/handlers/api/status.py

@@ -1,13 +1,10 @@
 import logging
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
 
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 
 logger = logging.getLogger(__name__)
@@ -23,21 +20,13 @@ class APIStatusHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._status_data = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, status_data, web_server_metrics):
+    def initialize(self, status_data):
         self._status_data = status_data
-        self._web_server_metrics = web_server_metrics
 
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             self.write(safe_json_dumps(self._status_data))
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")

webserver/handlers/api/v1_addspot.py

@@ -1,16 +1,13 @@
 import logging
 import re
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
 
 from core.config import ALLOW_SPOTTING
 from core.constants import UNKNOWN_BAND
-from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import infer_band_from_freq, safe_json_dumps
 from data.spot import Spot
@@ -28,21 +25,13 @@ class V1APISpotHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._spots = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, spots, web_server_metrics):
+    def initialize(self, spots):
         self._spots = spots
-        self._web_server_metrics = web_server_metrics
 
     def post(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # Reject if not allowed
             if not ALLOW_SPOTTING:
                 self.set_status(401)

webserver/handlers/api/v1_compatability.py

@@ -34,7 +34,7 @@ class V1RedirectHandler(tornado.web.RequestHandler):
             response = await client.fetch(
                 new_url,
                 method=self.request.method,
-                headers=self.request.headers,
+                headers=headers,
                 body=None if self.request.method == "GET" else (self.request.body or b""),
                 raise_error=False,
                 follow_redirects=False,

webserver/handlers/pagetemplate.py

@@ -1,7 +1,5 @@
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
@@ -16,7 +14,6 @@ from core.config import (
     WEB_UI_OPTIONS,
 )
 from core.constants import SOFTWARE_VERSION
-from core.prometheus_metrics_handler import page_requests_counter
 
 
 class PageTemplateHandler(tornado.web.RequestHandler):
@@ -29,20 +26,12 @@ class PageTemplateHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._template_name = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, template_name, web_server_metrics):
+    def initialize(self, template_name):
         self._template_name = template_name
-        self._web_server_metrics = web_server_metrics
 
     def get(self):
-        # Metrics
-        self._web_server_metrics["last_page_access_time"] = datetime.now(pytz.UTC)
-        self._web_server_metrics["page_access_counter"] += 1
-        self._web_server_metrics["status"] = "OK"
-        page_requests_counter.inc()
-
         # Load named template, and provide variables used in templates
         self.render(
             f"{self._template_name}.html",

webserver/sse_broadcaster.py

@@ -26,6 +26,11 @@ class SSEBroadcaster:
         with self._lock:
             self._handlers.discard(handler)
 
+    @property
+    def client_count(self) -> int:
+        with self._lock:
+            return len(self._handlers)
+
     def publish(self, value):
         self._loop.add_callback(self._broadcast, value)
 

webserver/webserver.py

@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import os
+import threading
 
 import tornado
 from tornado.web import StaticFileHandler
@@ -33,6 +34,7 @@ from webserver.handlers.manifesthandler import ManifestHandler
 from webserver.handlers.metrics import PrometheusMetricsHandler
 from webserver.handlers.pagetemplate import PageTemplateHandler
 from webserver.sse_broadcaster import SSEBroadcaster
+from webserver.webserver_metrics import WebServerMetrics
 
 logger = logging.getLogger(__name__)
 
@@ -52,92 +54,107 @@ class WebServer:
         self._port = WEB_SERVER_PORT
         self._api_only_mode = API_ONLY_MODE
         self._shutdown_event = asyncio.Event()
-        self.web_server_metrics = {
+        self._loop = None
-            "last_page_access_time": None,
+        self._thread = None
-            "last_api_access_time": None,
+        self.web_server_metrics = WebServerMetrics()
-            "page_access_counter": 0,
-            "api_access_counter": 0,
-            "status": "Starting",
-        }
 
     def setup(self):
         # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
         DATA_STORE.spots.add_listener(self._spot_broadcaster.publish)
         DATA_STORE.alerts.add_listener(self._alert_broadcaster.publish)
 
+    @property
+    def sse_client_count(self) -> int:
+        """Number of connected SSE clients, across both the spots and alerts streams."""
+
+        return self._spot_broadcaster.client_count + self._alert_broadcaster.client_count
+
     def start(self):
         """Start the web server"""
 
-        asyncio.run(self._start_inner())
+        self._thread = threading.Thread(target=asyncio.run, args=(self._start_inner(),), name="WebServer", daemon=True)
+        self._thread.start()
 
     def stop(self):
         """Stop the web server"""
 
-        self._shutdown_event.set()
+        if self._loop and self._loop.is_running():
+            self._loop.call_soon_threadsafe(self._shutdown_event.set)
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("Web server background thread did not exit on time and will be killed.")
+
+    def _handle_loop_exception(self, loop, context):
+        """Ignore "cancelled" exceptions from the asyncio loop to avoid printing exceptions to the log on shutdown when
+        we cancel the SSE handler threads"""
+
+        exception = context.get("exception")
+        if isinstance(exception, asyncio.CancelledError):
+            return
+        loop.default_exception_handler(context)
 
     async def _start_inner(self):
         """Start method (async). Sets up the Tornado application."""
 
+        self._loop = asyncio.get_running_loop()
+        self._loop.set_exception_handler(self._handle_loop_exception)
+
         # Bind the SSE broadcasters to the web server's loop, so they fire correctly
         self._spot_broadcaster.bind_to_web_server_loop()
         self._alert_broadcaster.bind_to_web_server_loop()
 
-        # Prepare a list of common arguments that are passed in to every API & page handler. This is just a basic thing
-        # to avoid copy-pasting the same thing to every route declaration below.
-        handler_opts = {"web_server_metrics": self.web_server_metrics}
-
         # API endpoints are always enabled
         api_routes = [
             (
                 r"/api/v2/spots",
                 APISpotsHandler,
-                {"spots": self._data_store.spots, **handler_opts},
+                {"spots": self._data_store.spots},
             ),
             (
                 r"/api/v2/alerts",
                 APIAlertsHandler,
-                {"alerts": self._data_store.alerts, **handler_opts},
+                {"alerts": self._data_store.alerts},
             ),
             (
                 r"/api/v2/spots/stream",
                 APISpotsStreamHandler,
-                {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts},
+                {"sse_spot_broadcaster": self._spot_broadcaster},
             ),
             (
                 r"/api/v2/alerts/stream",
                 APIAlertsStreamHandler,
-                {"sse_alert_broadcaster": self._alert_broadcaster, **handler_opts},
+                {"sse_alert_broadcaster": self._alert_broadcaster},
             ),
             (
                 r"/api/v2/solar",
                 APISolarConditionsHandler,
-                {"solar_conditions": self._data_store.solar_conditions.get(), **handler_opts},
+                {"solar_conditions": self._data_store.solar_conditions.get()},
             ),
             (
                 r"/api/v2/dxstats",
                 APIDxStatsHandler,
-                {"spots": self._data_store.spots, **handler_opts},
+                {"spots": self._data_store.spots},
             ),
             (
                 r"/api/v2/options",
                 APIOptionsHandler,
-                {"status_data": self._data_store.status.get(), **handler_opts},
+                {"status_data": self._data_store.status.get()},
             ),
             (
                 r"/api/v2/status",
                 APIStatusHandler,
-                {"status_data": self._data_store.status.get(), **handler_opts},
+                {"status_data": self._data_store.status.get()},
             ),
-            (r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}),
+            (r"/api/v2/lookup/call", APILookupCallHandler),
-            (r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
+            (r"/api/v2/lookup/sigref", APILookupSIGRefHandler),
-            (r"/api/v2/lookup/grid", APILookupGridHandler, {**handler_opts}),
+            (r"/api/v2/lookup/grid", APILookupGridHandler),
             (
                 r"/api/v2/spot",
                 APISpotHandler,
                 {
                     "spots": self._data_store.spots,
                     "spot_providers": self._data_providers,
-                    **handler_opts,
                 },
             ),
         ]
@@ -148,19 +165,18 @@ class WebServer:
             (
                 r"/api/v1/spots",
                 V1APISpotsHandler,
-                {"spots": self._data_store.spots, **handler_opts},
+                {"spots": self._data_store.spots},
             ),
             (
                 r"/api/v1/spots/stream",
                 V1APISpotsStreamHandler,
-                {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts},
+                {"sse_spot_broadcaster": self._spot_broadcaster},
             ),
             (
                 r"/api/v1/spot",
                 V1APISpotHandler,
                 {
                     "spots": self._data_store.spots,
-                    **handler_opts,
                 },
             ),
             (r"/api/v1/(.*)", V1RedirectHandler),
@@ -173,41 +189,41 @@ class WebServer:
                 (
                     r"/",
                     PageTemplateHandler,
-                    {"template_name": "api_only_home", **handler_opts},
+                    {"template_name": "api_only_home"},
                 )
             ]
         else:
             ui_routes = [
-                (r"/", PageTemplateHandler, {"template_name": "spots", **handler_opts}),
+                (r"/", PageTemplateHandler, {"template_name": "spots"}),
                 (
                     r"/map",
                     PageTemplateHandler,
-                    {"template_name": "map", **handler_opts},
+                    {"template_name": "map"},
                 ),
                 (
                     r"/bands",
                     PageTemplateHandler,
-                    {"template_name": "bands", **handler_opts},
+                    {"template_name": "bands"},
                 ),
                 (
                     r"/alerts",
                     PageTemplateHandler,
-                    {"template_name": "alerts", **handler_opts},
+                    {"template_name": "alerts"},
                 ),
                 (
                     r"/conditions",
                     PageTemplateHandler,
-                    {"template_name": "conditions", **handler_opts},
+                    {"template_name": "conditions"},
                 ),
                 (
                     r"/status",
                     PageTemplateHandler,
-                    {"template_name": "status", **handler_opts},
+                    {"template_name": "status"},
                 ),
                 (
                     r"/about",
                     PageTemplateHandler,
-                    {"template_name": "about", **handler_opts},
+                    {"template_name": "about"},
                 ),
             ]
             # Only allow the Add Spot page if spotting is allowed
@@ -216,7 +232,7 @@ class WebServer:
                     (
                         r"/add-spot",
                         PageTemplateHandler,
-                        {"template_name": "add_spot", **handler_opts},
+                        {"template_name": "add_spot"},
                     )
                 ]
 
@@ -226,7 +242,7 @@ class WebServer:
             (
                 r"/apidocs",
                 PageTemplateHandler,
-                {"template_name": "apidocs", **handler_opts},
+                {"template_name": "apidocs"},
             ),
             (r"/metrics", PrometheusMetricsHandler),
             (r"/manifest.webmanifest", ManifestHandler),
@@ -254,7 +270,11 @@ class WebServer:
 
 def request_log(handler):
     """Custom log function to provide more data about requests when enabled, and to provide the ability to turn off
-    web request logging altogetether."""
+    web request logging altogetether. Also records the time of the request and status in the webserver metrics. Probably
+    not what this method is supposed to be used for but it's a convenient thing that gets called on every request, so
+    saves having to pass the metrics around each handler individually."""
+
+    WEB_SERVER.web_server_metrics.record(handler.request.path, handler.get_status())
 
     if LOG_WEB_REQUESTS:
         if handler.get_status() < 500:

webserver/webserver_metrics.py

@@ -0,0 +1,44 @@
+from collections import deque
+from datetime import datetime, timedelta
+
+import pytz
+
+from core.prometheus_metrics_handler import api_requests_counter, page_requests_counter
+
+
+class WebServerMetrics:
+    """Tracker for web server metrics. Stores the times pages and API endpoints were accessed for an hour, so we
+    can display the rate of requests per hour, and also updates the equivalent Prometheus counters."""
+
+    def __init__(self):
+        self.status = "Starting"
+        self._page_access_times = deque()
+        self._api_access_times = deque()
+
+    def record(self, path: str, status_code: int):
+        """Records data for a request, depending on whether it's a page, API, or other request, and making sure
+        the response code isn't 404. Also sets the status of the web server."""
+
+        if status_code == 404 or path.startswith(("/static/", "/metrics", "/manifest.webmanifest")):
+            return
+
+        self.status = "OK" if status_code < 500 else "Error"
+        if path.startswith("/api/"):
+            api_requests_counter.inc()
+            self._api_access_times.append(datetime.now(pytz.UTC))
+        else:
+            page_requests_counter.inc()
+            self._page_access_times.append(datetime.now(pytz.UTC))
+
+    def page_requests_per_hour(self) -> int:
+        return self._count_and_prune_last_hour(self._page_access_times)
+
+    def api_requests_per_hour(self) -> int:
+        return self._count_and_prune_last_hour(self._api_access_times)
+
+    @staticmethod
+    def _count_and_prune_last_hour(access_times: deque) -> int:
+        cutoff = datetime.now(pytz.UTC) - timedelta(hours=1)
+        while access_times and access_times[0] < cutoff:
+            access_times.popleft()
+        return len(access_times)