Release 2.1.1

Dockerfile

@@ -4,5 +4,6 @@ WORKDIR /app
 COPY . .
 RUN pip install --no-cache-dir -r requirements.txt
 EXPOSE 8080
+EXPOSE 7373
 
 CMD ["python3", "spothole.py"]

README.md

@@ -1,7 +1,7 @@
 # ![Spothole](/static/img/logo.png)
 
 Spothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open
-JSON API as well as a website to browse the data.
+JSON API as well as a website to browse the data, and its own telnet server for integration with desktop loggers.
 
 ![Screenshot](/images/screenshot.png)
 
@@ -17,10 +17,8 @@ Spothole itself is also open source, Public Domain licenced code that anyone can
 
 Supported data sources include DX Clusters, the Reverse Beacon Network (RBN), the APRS Internet Service (APRS-IS), POTA,
 SOTA, WWFF, GMA, WWBOTA, HEMA, Parks 'n' Peaks, ZLOTA, WOTA, BOTA, LLOTA, WWTOTA, Tiles on the Air, the UK Packet
-Repeater Network, NG3K, and any site based on the xOTA software by nischu.
+Repeater Network, NG3K, and any site based on the xOTA software by nischu. It also integrates with QRZ.com and HamQTH,
-
+retrieves solar data from various sources, provides information about upcoming contests, and more.
-Additional Special Interest Groups (SIGs) without their own specific data source include KRMNPA, SANPCPA, WAB, WAI and
-DME.
 
 ![Screenshot](/images/screenshot2.png)
 

config-example.yml

@@ -17,6 +17,15 @@ api_only_mode: false
 # The base URL at which the software runs.
 base_url: "http://localhost:8080"
 
+# Whether to run a telnet spot server as well as the web interface
+telnet_server_enabled: false
+
+# Telnet server address. This is not really needed by the software itself, just displayed in documentation.
+telnet_server_address: "localhost"
+
+# Port to run the telnet server on
+telnet_server_port: 7373
+
 # Spot providers to use. This is an example set, tailor it to your liking by commenting and uncommenting.
 # RBN and APRS-IS are supported but have such a high data rate, you probably don't want them enabled.
 # Each provider needs a class and an enabled/disabled state. Some require more config such as hostnames/IP
@@ -162,9 +171,15 @@ alert_providers:
   - class: "BOTA"
     enabled: true
 
+  - class: "Hamsat"
+    enabled: true
+
   - class: "NG3K"
     enabled: true
 
+  - class: "WA7BNM"
+    enabled: true
+
   - class: "RSGBHFContests"
     enabled: true
 
@@ -305,27 +320,27 @@ callsign_data_providers:
     priority: 2
     # No server-side credentials for HamQTH. Users must provide their own.
 
+  - class: "CountryFiles"
+    priority: 3
+    enabled: true
+
   - class: "ClublogAPI"
     # Querying the Clublog API directly doesn't provide any more data than the XML version, it just provides slightly
     # more up-to-date information in the rare case that the prefix data changes, at a significant cost of looking up
     # every callsign via an API call. Normally left disabled but it exists as an option.
     enabled: false
-    priority: 3
+    priority: 4
     # API key for Clublog to look up information. Required in order to enable this provider. Unlike QRZ and HamQTH,
     # Clublog uses an API key issued to Spothole, not to the end user.
     api_key: ""
 
   - class: "ClublogXML"
     enabled: true
-    priority: 4
+    priority: 5
     # API key for Clublog to look up information. Required in order to enable this provider. You will need to request
     # one via their helpdesk portal if you want to use callsign lookups from Clublog.
     api_key: ""
 
-  - class: "CountryFiles"
-    priority: 5
-    enabled: true
-
 
 # Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week
 # for alerts.

core/cleanup.py

@@ -34,6 +34,10 @@ class CleanupTimer:
         """Stop any threads and prepare for application shutdown"""
 
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("Cleanup worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while not self._stop_event.wait(timeout=self._cleanup_interval):

core/config.py

@@ -24,6 +24,9 @@ MAX_SPOT_AGE = config.get("max_spot_age_sec", 3600)
 MAX_ALERT_AGE = config.get("max_alert_age_sec", 604800)
 SERVER_OWNER_CALLSIGN = config.get("server_owner_callsign", "N0CALL")
 WEB_SERVER_PORT = config.get("web_server_port", 8080)
+TELNET_SERVER_ENABLED = config.get("telnet_server_enabled", False)
+TELNET_SERVER_ADDRESS = config.get("telnet_server_address", "localhost")
+TELNET_SERVER_PORT = config.get("telnet_server_port", 7373)
 ALLOW_SPOTTING = config.get("allow_spotting", True)
 ALLOW_UPSTREAM_SPOTTING = config.get("allow_upstream_spotting", True)
 WEB_UI_OPTIONS = config.get("web_ui_options", {})

core/constants.py

@@ -4,7 +4,7 @@ from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "2.1-pre"
+SOFTWARE_VERSION = "2.1.1"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"}
@@ -12,6 +12,22 @@ HAMQTH_PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}"
 
 # Special Interest Groups
 SIGS = [
+    SIG(
+        name="AMSAT",
+        comment_names=[],
+        description="Amateur Radio Satellites",
+        sig_type=SIGType.WORLDWIDE,
+        icon="fa-satellite",
+        refs_globally_unique=False,
+    ),
+    SIG(
+        name="EME",
+        comment_names=[],
+        description="Moonbounce",
+        sig_type=SIGType.WORLDWIDE,
+        icon="fa-moon",
+        refs_globally_unique=False,
+    ),
     SIG(
         name="POTA",
         comment_names=["POTA"],
@@ -247,6 +263,16 @@ SIGS = [
         region_flag="🇮🇪",
         refs_globally_unique=False,
     ),
+    SIG(
+        name="DMF",
+        comment_names=["DMF"],
+        description="Diplôme des Moulins de France",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.MILL,
+        icon="fa-fan",
+        region_flag="🇫🇷",
+        refs_globally_unique=False,
+    ),
     SIG(
         name="DME",
         comment_names=["DME"],

core/data_providers.py

@@ -1,5 +1,6 @@
 import logging
 import threading
+import time
 
 from core.config import config, create_provider_from_config
 
@@ -16,6 +17,7 @@ class DataProviders:
         self.static_data_providers = []
         self.sig_ref_data_providers = []
         self.callsign_data_providers = []
+        self._startup_timers = []
 
     def setup(self):
         for entry in config["spot_providers"]:
@@ -43,41 +45,58 @@ class DataProviders:
     def start(self):
         # Start data providers before spot/alert providers so the lookup data is there already for incoming spots.
         # Each category is fired off after a small delay to give the rest of Spothole chance to start up.
-        threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")).start()
+        self._startup_timers = [
-        threading.Timer(
+            threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")),
-            10.0,
+            threading.Timer(10.0, lambda: self.start_providers(self.callsign_data_providers, "callsign data")),
-            lambda: self.start_providers(self.callsign_data_providers, "callsign data"),
+            threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")),
-        ).start()
+            threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")),
-        threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")).start()
+            threading.Timer(
-        threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")).start()
+                25.0,
-        threading.Timer(
+                lambda: self.start_providers(self.solar_condition_providers, "solar condition"),
-            25.0,
+            ),
-            lambda: self.start_providers(self.solar_condition_providers, "solar condition"),
+            threading.Timer(30.0, lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data")),
-        ).start()
+        ]
-        threading.Timer(
+        for t in self._startup_timers:
-            30.0,
+            t.daemon = True
-            lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data"),
+            t.start()
-        ).start()
 
     def stop(self):
-        for sp in self.spot_providers:
+        # Cancel any startup timers that haven't fired yet
-            if sp.enabled:
+        for t in self._startup_timers:
-                sp.stop()
+            t.cancel()
-        for ap in self.alert_providers:
+
-            if ap.enabled:
+        # Stop all providers
-                ap.stop()
+        all_providers = [
-        for scp in self.solar_condition_providers:
+            p
-            if scp.enabled:
+            for p in (
-                scp.stop()
+                self.spot_providers
-        for srdp in self.sig_ref_data_providers:
+                + self.alert_providers
-            if srdp.enabled:
+                + self.solar_condition_providers
-                srdp.stop()
+                + self.sig_ref_data_providers
-        for sdp in self.static_data_providers:
+                + self.static_data_providers
-            if sdp.enabled:
+                + self.callsign_data_providers
-                sdp.stop()
+            )
-        for cdp in self.callsign_data_providers:
+            if p.enabled
-            if cdp.enabled:
+        ]
-                cdp.stop()
+        if not all_providers:
+            return
+
+        def stop_provider(p):
+            try:
+                p.stop()
+            except Exception:
+                logger.exception("Exception stopping provider")
+
+        threads = [threading.Thread(target=stop_provider, args=(p,), daemon=True) for p in all_providers]
+        for t in threads:
+            t.start()
+
+        deadline = time.monotonic() + 40
+        for t in threads:
+            t.join(timeout=max(0.0, deadline - time.monotonic()))
+        still_running = [t for t in threads if t.is_alive()]
+        if still_running:
+            logger.warning("Some threads did not stop in time!")
 
 
 # Global object

core/data_store.py

@@ -106,6 +106,7 @@ class DataStore:
         and testing the callsign every time is expensive. So instead we build a separate in-memory lookup of compiled
         regex against DXCC entity code, as a list of tuples we can iterate through."""
 
+        self.dxcc_lookup_by_call_regex = []
         for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]:
             self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
 

core/enums.py

@@ -71,7 +71,6 @@ class ModeSource(str, Enum):
     SPOT = "SPOT"
     COMMENT = "COMMENT"
     BANDPLAN = "BANDPLAN"
-    NONE = "NONE"
 
 
 class LocationSourceForSpot(str, Enum):
@@ -82,7 +81,6 @@ class LocationSourceForSpot(str, Enum):
     GRID = "GRID"
     HOME_QTH = "HOME QTH"
     DXCC = "DXCC"
-    NONE = "NONE"
 
 
 class LocationSourceForCallsign(str, Enum):
@@ -90,7 +88,6 @@ class LocationSourceForCallsign(str, Enum):
 
     HOME_QTH = "HOME QTH"
     DXCC = "DXCC"
-    NONE = "NONE"
 
 
 class SIGRefType(str, Enum):
@@ -119,6 +116,7 @@ class AlertType(str, Enum):
     """Type of an alert."""
 
     XOTA = "XOTA"
+    SATELLITE = "SATELLITE"
     DXPEDITION = "DXPEDITION"
     CONTEST = "CONTEST"
 

core/live_data_cache.py

@@ -22,6 +22,8 @@ class LiveDataCache:
         self._listeners_lock = threading.Lock()
         self._snapshot_dir = snapshot_dir
         self._disk_cache = diskcache.Cache(str(snapshot_dir))
+        self._stop_event = threading.Event()
+        self._snapshot_thread = None
         self._load_snapshot()
         self._start_periodic_snapshot(snapshot_interval_sec)
 
@@ -89,13 +91,19 @@ class LiveDataCache:
 
     def _start_periodic_snapshot(self, interval):
         def loop():
-            while True:
+            while not self._stop_event.wait(timeout=interval):
-                time.sleep(interval)
                 self.save_snapshot()
 
-        t = threading.Thread(target=loop, name=f"LiveDataCache-Snapshot-{self._snapshot_dir}")
+        self._snapshot_thread = threading.Thread(
-        t.start()
+            target=loop, name=f"LiveDataCache-Snapshot-{self._snapshot_dir}", daemon=True
+        )
+        self._snapshot_thread.start()
 
     def close(self):
+        self._stop_event.set()
+        if self._snapshot_thread:
+            self._snapshot_thread.join(timeout=15)
+            if self._snapshot_thread.is_alive():
+                logger.warning(f"LiveDataCache snapshot thread for {self._snapshot_dir} did not exit on time.")
         self.save_snapshot()
         self._disk_cache.close()

core/sig_lookup_helper.py

@@ -24,7 +24,7 @@ def get_sig_ref_info(sig_name, ref_id):
 
     # Sometimes we allow spaces instead of dashes in references due to common usage that way, but official reference
     # lists never do, so convert them here.
-    ref_id.replace(" ", "-")
+    ref_id = ref_id.replace(" ", "-")
 
     # Prepare the object to be returned
     sig_ref = SIGRef(sig=sig_name, id=ref_id)
@@ -43,8 +43,9 @@ def get_sig_ref_info(sig_name, ref_id):
         # database.
         if sig_name.upper() == "DME":
             match = re.match(r"DME[\- ](\d{3,5})", ref_id, re.IGNORECASE)
-            number = match.group(1)
+            if match:
-            ref_id = f"DME-{number.zfill(5)}"
+                number = match.group(1)
+                ref_id = f"DME-{number.zfill(5)}"
 
         # DTMBA spotters sometimes include spaces and dashes, our regex allows them but they must be removed here so we
         # can look up against the official list which doesn't have them
@@ -113,16 +114,23 @@ def get_sig_ref_info(sig_name, ref_id):
         # OK, this is something we have to look up. Now check to see if our data store contains reference data and if
         # so, copy the data into the sig_ref object
         key = f"{sig_name}:{ref_id}"
-        lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
+        try:
-        if lookup_data:
+            lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
-            for key, value in lookup_data.__dict__.items():
+            if lookup_data:
-                if value is not None and sig_ref.__dict__.get(key) is None:
+                for attr, value in lookup_data.__dict__.items():
-                    sig_ref.__dict__[key] = value
+                    if value is not None and sig_ref.__dict__.get(attr) is None:
+                        sig_ref.__dict__[attr] = value
 
-        else:
+            else:
-            # Maybe a super new reference we don't know about yet, but more likely a typo or a test reference,
+                # Maybe a super new reference we don't know about yet, but more likely a typo or a test reference,
-            # just silently ignore it.
+                # just silently ignore it.
-            logger.debug(f"{sig_name} database did not contain data for ref {ref_id}")
+                logger.debug(f"{sig_name} database did not contain data for ref {ref_id}")
+
+        except (ValueError, KeyError):
+            # Catch exceptions due to e.g. old versions of objects in the cache that are no longer compatible,
+            # and remove them from the cache.
+            del DATA_STORE.sigrefs[key]
+            return None
 
     except Exception:
         logger.exception(f"Exception when looking up sig_ref info for {sig_name} ref {ref_id}")

core/status_reporter.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from datetime import datetime
 from threading import Event, Thread
@@ -11,7 +12,10 @@ from core.constants import SOFTWARE_VERSION
 from core.data_providers import DATA_PROVIDERS
 from core.data_store import DATA_STORE
 from core.prometheus_metrics_handler import alerts_gauge, memory_use_gauge, spots_gauge
-from server.webserver import WEB_SERVER
+from telnetserver.telnetserver import TELNET_SERVER
+from webserver.webserver import WEB_SERVER
+
+logger = logging.getLogger(__name__)
 
 
 class StatusReporter:
@@ -32,13 +36,17 @@ class StatusReporter:
     def start(self):
         """Start the reporter thread"""
 
-        self._thread = Thread(target=self._run, name="StatusReporter")
+        self._thread = Thread(target=self._run, name="StatusReporter", daemon=True)
         self._thread.start()
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""
 
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("Status reporter worker thread did not exit on time and will be killed.")
 
     def _run(self):
         """Thread entry point: report immediately on startup, then on each interval until stopped"""
@@ -134,19 +142,13 @@ class StatusReporter:
             else 0,
         }
         DATA_STORE.status.get()["webserver"] = {
-            "status": WEB_SERVER.web_server_metrics["status"],
+            "status": WEB_SERVER.web_server_metrics.status,
-            "last_api_access": WEB_SERVER.web_server_metrics["last_api_access_time"]
+            "api_requests_per_hour": WEB_SERVER.web_server_metrics.api_requests_per_hour(),
-            .replace(tzinfo=pytz.UTC)
+            "page_requests_per_hour": WEB_SERVER.web_server_metrics.page_requests_per_hour(),
-            .timestamp()
+            "sse_client_count": WEB_SERVER.sse_client_count,
-            if WEB_SERVER.web_server_metrics["last_api_access_time"]
+        }
-            else 0,
+        DATA_STORE.status.get()["telnet"] = {
-            "api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"],
+            "client_count": TELNET_SERVER.client_count,
-            "last_page_access": WEB_SERVER.web_server_metrics["last_page_access_time"]
-            .replace(tzinfo=pytz.UTC)
-            .timestamp()
-            if WEB_SERVER.web_server_metrics["last_page_access_time"]
-            else 0,
-            "page_access_count": WEB_SERVER.web_server_metrics["page_access_counter"],
         }
         DATA_STORE.status.store()
 

core/url_data_cache.py

@@ -14,14 +14,13 @@ class URLDataCache(CachedSession):
     used across multiple threads, though note that URL lookups will block each other this way, so it is still better to
     create one of these objects per thread if possible."""
 
-    _lock = threading.Lock()
-
     def __init__(self, name):
         super().__init__(
             f"{CACHE_DIR}urls/{name}",
             expire_after=timedelta(days=1),
             allowable_codes=(200, 400, 401, 403, 404),
         )
+        self._lock = threading.Lock()
 
     def get(self, *args, **kwargs):
         with self._lock:

data/alert.py

@@ -1,7 +1,7 @@
 import hashlib
 import json
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 
 import pytz
@@ -64,7 +64,7 @@ class Alert:
     # Special Interest Group (SIG), e.g. outdoor activity programme such as POTA
     sig: str | None = None
     # SIG references. We allow multiple here for e.g. n-fer activations, unlike ADIF SIG_INFO
-    sig_refs: list | None = None
+    sig_refs: list = field(default_factory=list)
 
     # Timing info
 
@@ -128,13 +128,13 @@ class Alert:
             # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference
             # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs
             # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths
-            if self.sig_refs and len(self.sig_refs) > 0:
+            if self.sig_refs:
                 for sig_ref in self.sig_refs:
                     populate_missing_sig_ref_info(sig_ref)
 
             # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG
             # and apply it to the whole spot.
-            if self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0] and not self.sig:
+            if self.sig_refs and self.sig_refs[0] and not self.sig:
                 self.sig = self.sig_refs[0].sig
 
             # Create an ID based on the source and source ID if possible, as these guaranee uniqueness. If there is no
@@ -158,7 +158,9 @@ class Alert:
                 self.icon = "fa-globe-africa"
             elif self.alert_type == AlertType.CONTEST:
                 self.icon = "fa-trophy"
-            elif self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0].icon:
+            elif self.alert_type == AlertType.SATELLITE:
+                self.icon = "fa-satellite"
+            elif self.sig_refs and self.sig_refs[0].icon:
                 self.icon = self.sig_refs[0].icon
 
         except Exception:

data/callsign.py

@@ -38,7 +38,7 @@ class Callsign:
     # ITU zone in which the callsign indicates they are operating
     itu_zone: int | None = None
     # Location source
-    location_source: LocationSourceForCallsign = LocationSourceForCallsign.NONE
+    location_source: LocationSourceForCallsign | None = None
 
     def fully_populated(self):
         """Utility method to indicate that the callsign data is fully populated. Multiple providers can return data for

data/sig_ref.py

@@ -8,10 +8,10 @@ class SIGRef:
     """Data class that defines a Special Interest Group "info" or reference. As well as the basic reference ID we include a
     name and a lookup URL."""
 
-    # Reference ID, e.g. "GB-0001".
-    id: str
     # SIG that this reference is in, e.g. "POTA".
     sig: str
+    # Reference ID, e.g. "GB-0001".
+    id: str | None = None
     # Name of the reference, e.g. "Null Country Park", if known.
     name: str | None = None
     # Type of the reference, e.g. "Park", if known.

data/spot.py

@@ -2,7 +2,7 @@ import hashlib
 import json
 import logging
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 from math import isnan
 
@@ -12,6 +12,7 @@ from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 from core.call_lookup_helper import get_call_info
 from core.config import MAX_SPOT_AGE
 from core.constants import PROPAGATION_MODES, SIGS
+from core.data_store import DATA_STORE
 from core.enums import Continent, LocationSourceForSpot, Mode, ModeSource, ModeType
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.sig_lookup_helper import populate_missing_sig_ref_info
@@ -70,7 +71,7 @@ class Spot:
     dx_latitude: float | None = None
     dx_longitude: float | None = None
     # DX Location source. Indicates how accurate the location might be.
-    dx_location_source: LocationSourceForSpot = LocationSourceForSpot.NONE
+    dx_location_source: LocationSourceForSpot | None = None
     # DX Location good. Indicates that the software thinks the location data is good enough to plot on a map. This is
     # true if the location source is "SPOT", "SIG REF LOOKUP" or "GRID", or if the location source is "HOME QTH" and the
     # DX callsign doesn't have a suffix like /P.
@@ -107,7 +108,7 @@ class Spot:
     # Inferred mode "family".
     mode_type: ModeType | None = None
     # Source of the mode information.
-    mode_source: ModeSource = ModeSource.NONE
+    mode_source: ModeSource | None = None
     # Frequency, in Hz
     freq: float | None = None
     # Band, defined by the frequency, e.g. "40m" or "70cm"
@@ -124,7 +125,7 @@ class Spot:
     # Special Interest Group (SIG), e.g. outdoor activity programme such as POTA
     sig: str | None = None
     # SIG references. We allow multiple here for e.g. n-fer activations, unlike ADIF SIG_INFO
-    sig_refs: list | None = None
+    sig_refs: list = field(default_factory=list)
 
     # Timing info
 
@@ -262,12 +263,12 @@ class Spot:
                 self.dx_location_source = LocationSourceForSpot.SPOT
 
             # Set the top-level "SIG" if it is missing but we have at least one SIG ref.
-            if not self.sig and self.sig_refs and len(self.sig_refs) > 0:
+            if not self.sig and self.sig_refs:
                 self.sig = self.sig_refs[0].sig.upper()
 
             # See if we already have a SIG reference, but the comment looks like it contains more for the same SIG. This
             # should catch e.g. POTA comments like "2-fer: GB-0001 GB-0002".
-            if self.comment and self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0].sig:
+            if self.comment and self.sig_refs and self.sig_refs[0].sig:
                 sig = self.sig_refs[0].sig.upper()
                 regex = get_ref_regex_for_sig(sig)
                 if regex:
@@ -315,7 +316,7 @@ class Spot:
             # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference
             # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs
             # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths
-            if self.sig_refs and len(self.sig_refs) > 0:
+            if self.sig_refs:
                 for sig_ref in self.sig_refs:
                     sig_ref = populate_missing_sig_ref_info(sig_ref)
                     # If the spot itself doesn't have location yet, but the SIG ref does, extract it
@@ -331,7 +332,7 @@ class Spot:
 
             # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG
             # and apply it to the whole spot.
-            if self.sig_refs and len(self.sig_refs) > 0 and not self.sig:
+            if self.sig_refs and not self.sig:
                 self.sig = self.sig_refs[0].sig
 
             # Parse "de_grid<prop_mode>dx_grid" structures from the comment, e.g. "JN61ES(ES)JM56XT" or "JO02GQ<>KN17LG".
@@ -359,6 +360,18 @@ class Spot:
                             self.propagation_mode = mode_tag
                             logger.info(f"Seen a new propagation mode tag not yet in the system: {mode_tag}")
 
+            # Set SIGs based on propagation mode
+            if self.propagation_mode == "Satellite":
+                if not self.sig:
+                    self.sig = "AMSAT"
+                if not any(sig_ref.sig == "AMSAT" for sig_ref in self.sig_refs):
+                    self.sig_refs.append(SIGRef(sig="AMSAT"))
+            if self.propagation_mode == "Earth-Moon-Earth":
+                if not self.sig:
+                    self.sig = "EME"
+                if not any(sig_ref.sig == "EME" for sig_ref in self.sig_refs):
+                    self.sig_refs.append(SIGRef(sig="EME"))
+
             # Parse "de_grid -> dx_grid" structures from the comment
             if self.comment:
                 grid_mode_grid_match = re.search(
@@ -415,7 +428,7 @@ class Spot:
 
             # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable string,
             # otherwise see what they have set on an online lookup service.
-            if self.sig_refs and len(self.sig_refs) > 0:
+            if self.sig_refs:
                 qth = self.sig_refs[0].id
                 if self.sig_refs[0].name:
                     qth += f" {self.sig_refs[0].name}"
@@ -435,6 +448,15 @@ class Spot:
                 elif self.dx_call:
                     self.dx_itu_zone = dx_call_info.itu_zone
 
+            # DXCC lookup from callsign if nothing else has provided it
+            if self.dx_call and not self.dx_dxcc_id:
+                for regex, entity_code in DATA_STORE.dxcc_lookup_by_call_regex:
+                    if regex.pattern and regex.match(self.dx_call):
+                        self.dx_dxcc_id = entity_code
+                        break
+            if self.dx_dxcc_id and not self.dx_flag:
+                self.dx_flag = get_flag_for_dxcc(self.dx_dxcc_id)
+
             # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
             # is likely at home.
             self.dx_location_good = bool(
@@ -462,7 +484,7 @@ class Spot:
 
             # Icon for the spot should be the icon of the first SIG ref if present, otherwise a radio tower
             self.icon = "fa-tower-cell"
-            if self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0].icon:
+            if self.sig_refs and self.sig_refs[0].icon:
                 self.icon = self.sig_refs[0].icon
 
         except Exception:
@@ -476,16 +498,14 @@ class Spot:
     def _append_sig_ref_if_missing(self, new_sig_ref):
         """Append a sig_ref to the list, so long as it's not already there."""
 
-        sig_refs = self.sig_refs or []
-        self.sig_refs = sig_refs
         new_sig_ref.id = new_sig_ref.id.strip().upper()
         new_sig_ref.sig = new_sig_ref.sig.strip().upper()
         if new_sig_ref.id == "":
             return
-        for sig_ref in sig_refs:
+        for sig_ref in self.sig_refs:
             if sig_ref.id == new_sig_ref.id and sig_ref.sig == new_sig_ref.sig:
                 return
-        sig_refs.append(new_sig_ref)
+        self.sig_refs.append(new_sig_ref)
 
     def expired(self):
         """Decide if this spot has expired (in which case it should not be added to the system in the first place, and not

docs/docker.md

@@ -13,6 +13,7 @@ services:
     restart: unless-stopped
     ports:
       - "8080:8080"
+      - "7373:7373" # For telnet if required
     volumes:
       - ./config.yml:/app/config.yml
       - ./cache:/app/cache

docs/modifying.md

@@ -16,9 +16,12 @@ To navigate your way around the source code, this list may help.
 * `/providers/solarconditions` - Classes providing solar and propagation by accessing the APIs of other services
 * `/providers/staticdata` - Classes providing static lookup data by accessing bundled data files or the APIs of other
   services
+* `/providers/callsign` - Classes providing callsign lookup data by accessing bundled data files or the APIs of other
+  services
 * `/providers/sigrefdata` - Classes providing SIG reference lookup data by accessing bundled data files or the APIs of
   other services
-* `/server` - Classes for running Spothole's own web server
+* `/webserver` - Classes for running Spothole's own web server
+* `/telnetserver` - Classes for running Spothole's telnet server
 * `spothole.py` - Main application script
 
 *Templates*

providers/alert/hamsat.py

@@ -0,0 +1,56 @@
+from datetime import datetime
+
+import pytz
+
+from core.enums import AlertType
+from data.alert import Alert
+from data.sig_ref import SIGRef
+from providers.alert.http_alert_provider import HTTPAlertProvider
+
+
+class Hamsat(HTTPAlertProvider):
+    """Alert provider for Hamsat (hams.at)"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://hams.at/api/alerts"
+
+    def __init__(self, provider_config):
+        super().__init__("Hamsat", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json()["data"]:
+            # Convert to our alert format
+            freqs_modes = source_alert.get("mode", "")
+            if "mhz" in source_alert:
+                if "mhz_direction" in source_alert:
+                    freqs_modes = f"{source_alert['mhz']!s} {source_alert['mhz_direction']}, {freqs_modes}"
+                else:
+                    freqs_modes = f"{source_alert['mhz']!s}, {freqs_modes}"
+
+            alert = Alert(
+                source=self.name,
+                source_id=source_alert["id"],
+                dx_calls=[source_alert["callsign"].upper()],
+                freqs_modes=freqs_modes,
+                comment=source_alert["comment"],
+                # Fudge a SIG ref to provide the remaining bits of data we need: the satellite and the operator's grid
+                sig_refs=[
+                    SIGRef(
+                        sig="AMSAT",
+                        id=f"{source_alert['satellite']['name']} from {source_alert['grids'][0]}",
+                    )
+                ],
+                start_time=datetime.strptime(source_alert["aos_at"], "%Y-%m-%dT%H:%M:%SZ")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp(),
+                end_time=datetime.strptime(source_alert["los_at"], "%Y-%m-%dT%H:%M:%SZ")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp(),
+                alert_type=AlertType.SATELLITE,
+            )
+
+            # Add to our list
+            new_alerts.append(alert)
+        return new_alerts

providers/alert/http_alert_provider.py

@@ -27,11 +27,15 @@ class HTTPAlertProvider(AlertProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} alert API every {self._poll_interval!s} seconds.")
-        self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} alert worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/alert/ical_alert_provider.py

@@ -0,0 +1,47 @@
+from datetime import datetime, time
+from typing import cast
+
+import pytz
+from icalendar import Calendar, Event
+
+from data.alert import Alert
+from providers.alert.http_alert_provider import HTTPAlertProvider
+
+
+class ICALAlertProvider(HTTPAlertProvider):
+    """Generic alert provider for iCal calendars. Defines an abstract method event_to_alert(event) that subclasses must
+    implement, and use it to convert an iCal event to an Alert object based on whatever format their iCal events use."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        super().__init__(name, provider_config, url, poll_interval)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        cal = Calendar.from_ical(http_response.content)
+
+        # Iterate through events, passing each one in turn to the subclass' event_to_alert method to turn it into
+        # a Spothole alert object
+        for component in cal.walk():
+            if component.name != "VEVENT":
+                continue
+
+            event = cast(Event, component)
+            alert = self.event_to_alert(event)
+            new_alerts.append(alert)
+        return new_alerts
+
+    def event_to_alert(self, event: Event) -> Alert:
+        """Convert an ICal event to an Alert object. Subclasses must implement this method."""
+
+    @staticmethod
+    def _to_utc_timestamp(value):
+        """Convert a date or datetime value from an iCal field into a UTC UNIX timestamp."""
+
+        # Datetime object so we can treat it as-is, check if it has a non-UTC tz and convert it if necessary
+        if isinstance(value, datetime):
+            if value.tzinfo is None:
+                value = pytz.UTC.localize(value)
+            return value.astimezone(pytz.UTC).timestamp()
+
+        # Date object so this is an all day event
+        return pytz.UTC.localize(datetime.combine(value, time.min)).timestamp()

providers/alert/rsgb_ical_alert_provider.py

@@ -1,101 +1,75 @@
 import re
-from datetime import datetime, time
-from typing import cast
 
-import pytz
+from icalendar import Event
-from icalendar import Calendar, Event
 
 from core.enums import AlertType, Continent
 from data.alert import Alert
-from providers.alert.http_alert_provider import HTTPAlertProvider
+from providers.alert.ical_alert_provider import ICALAlertProvider
 
 
-class RSGBICALAlertProvider(HTTPAlertProvider):
+class RSGBICALAlertProvider(ICALAlertProvider):
-    """Generic alert provider for RSGB contest ICAL calendars."""
+    """Generic alert provider for RSGB contest iCal calendars. Builds on the generic iCal alert provider by adding
+    handling specific to how RSGB's iCal events are formatted. This is still effectively an abstract class itself;
+    RSGB has two contest calendars (HF & VHF) that each subclass this."""
 
     def __init__(self, name, provider_config, url, poll_interval):
         super().__init__(name, provider_config, url, poll_interval)
 
     FREQ_PATTERN = re.compile(r"([\d.]+(?:MHz|GHz))|SHF")
 
-    def _http_response_to_alerts(self, http_response):
+    def event_to_alert(self, event: Event) -> Alert:
-        new_alerts = []
+        """Convert an iCal event in RSGB's format to an Alert object."""
-        cal = Calendar.from_ical(http_response.content)
 
-        # Iterate through events
+        summary = str(event.get("summary", "")).strip()
-        for component in cal.walk():
-            if component.name != "VEVENT":
-                continue
 
-            event = cast(Event, component)
+        # Ensure summaries start with "RSGB" to avoid any confusion
-            summary = str(event.get("summary", "")).strip()
+        if not summary.startswith("RSGB "):
+            summary = "RSGB " + summary
 
-            # Ensure summaries start with "RSGB" to avoid any confusion
+        # Extract freq from summary. HF contests don't give frequencies, all VHF ones do
-            if not summary.startswith("RSGB "):
+        match = self.FREQ_PATTERN.search(summary)
-                summary = "RSGB " + summary
+        if match:
+            freqs_modes = match.group()
+        else:
+            freqs_modes = "HF bands"
+        freqs_modes = freqs_modes + ", "
 
-            # Extract freq from summary
+        # Extract mode from summary
-            freqs_modes = ""
+        if "FMAC" in summary:
+            freqs_modes = freqs_modes + "FM"
+        elif "CW" in summary:
+            freqs_modes = freqs_modes + "CW"
+        elif "SSB" in summary:
+            freqs_modes = freqs_modes + "SSB"
+        elif "FT8" in summary:
+            freqs_modes = freqs_modes + "FT8"
+        elif "FT4" in summary:
+            freqs_modes = freqs_modes + "FT4"
+        elif "DATA" in summary:
+            freqs_modes = freqs_modes + "Data modes"
+        else:
+            freqs_modes = freqs_modes + "All modes"
 
-            # HF contests don't give frequencies, all VHF ones do
+        dtstart = event.get("dtstart")
-            match = self.FREQ_PATTERN.search(summary)
+        dtend = event.get("dtend")
-            if match:
+        start_timestamp = self._to_utc_timestamp(dtstart.dt)
-                freqs_modes = match.group()
+        end_timestamp = self._to_utc_timestamp(dtend.dt) - 1 if dtend is not None else start_timestamp
-            else:
-                freqs_modes = "HF bands"
-            freqs_modes = freqs_modes + ", "
 
-            # Extract mode from summary
+        # Convert to our alert format
-            if "FMAC" in summary:
+        alert = Alert(
-                freqs_modes = freqs_modes + "FM"
+            source=self.name,
-            elif "CW" in summary:
+            dx_calls=[],
-                freqs_modes = freqs_modes + "CW"
+            dx_country="United Kingdom",
-            elif "SSB" in summary:
+            dx_dxcc_id=235,
-                freqs_modes = freqs_modes + "SSB"
+            dx_continent=Continent.EU,
-            elif "FT8" in summary:
+            dx_cq_zone=14,
-                freqs_modes = freqs_modes + "FT8"
+            dx_itu_zone=27,
-            elif "FT4" in summary:
+            dx_flag="🇬🇧",
-                freqs_modes = freqs_modes + "FT4"
+            freqs_modes=freqs_modes,
-            elif "DATA" in summary:
+            comment=summary,
-                freqs_modes = freqs_modes + "Data modes"
+            start_time=start_timestamp,
-            else:
+            end_time=end_timestamp,
-                freqs_modes = freqs_modes + "All modes"
+            alert_type=AlertType.CONTEST,
+        )
 
-            dtstart = event.get("dtstart")
+        return alert
-            dtend = event.get("dtend")
-            start_timestamp = self._to_utc_timestamp(dtstart.dt)
-            end_timestamp = self._to_utc_timestamp(dtend.dt) - 1 if dtend is not None else start_timestamp
-
-            # Convert to our alert format
-            alert = Alert(
-                source=self.name,
-                dx_calls=[],
-                dx_country="United Kingdom",
-                dx_dxcc_id=235,
-                dx_continent=Continent.EU,
-                dx_cq_zone=14,
-                dx_itu_zone=27,
-                dx_flag="🇬🇧",
-                freqs_modes=freqs_modes,
-                comment=summary,
-                start_time=start_timestamp,
-                end_time=end_timestamp,
-                alert_type=AlertType.CONTEST,
-            )
-
-            # Add to our list.
-            new_alerts.append(alert)
-        return new_alerts
-
-    @staticmethod
-    def _to_utc_timestamp(value):
-        """Convert a date or datetime value from an iCal field into a UTC UNIX timestamp."""
-
-        # Datetime object so we can treat it as-is, check if it has a non-UTC tz and convert it if necessary
-        if isinstance(value, datetime):
-            if value.tzinfo is None:
-                value = pytz.UTC.localize(value)
-            return value.astimezone(pytz.UTC).timestamp()
-
-        # Date object so this is an all day event
-        return pytz.UTC.localize(datetime.combine(value, time.min)).timestamp()

providers/alert/wa7bnm.py

@@ -0,0 +1,41 @@
+from icalendar import Event
+
+from core.enums import AlertType
+from data.alert import Alert
+from providers.alert.ical_alert_provider import ICALAlertProvider
+
+
+class WA7BNM(ICALAlertProvider):
+    """Alert provider for the WA7BNM contest calendar."""
+
+    POLL_INTERVAL_DAYS = 1
+    ALERTS_URL = "https://contestcalendar.com/weeklycontcustom.php"
+
+    def __init__(self, provider_config):
+        super().__init__(
+            "WA7BNM Contest Calendar", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60
+        )
+
+    def event_to_alert(self, event: Event) -> Alert:
+        """Convert an iCal event in WA7BNM's format to an Alert object."""
+
+        summary = str(event.get("summary", ""))
+        url = str(event.get("url", ""))
+
+        dtstart = event.get("dtstart")
+        dtend = event.get("dtend")
+        start_timestamp = self._to_utc_timestamp(dtstart.dt)
+        end_timestamp = self._to_utc_timestamp(dtend.dt) - 1 if dtend is not None else start_timestamp
+
+        # Convert to our alert format
+        alert = Alert(
+            source=self.name,
+            dx_calls=[],
+            comment=summary,
+            url=url,
+            start_time=start_timestamp,
+            end_time=end_timestamp,
+            alert_type=AlertType.CONTEST,
+        )
+
+        return alert

providers/callsigndata/callsign_data_provider.py

@@ -42,7 +42,13 @@ class CallsignDataProvider:
 
         if self.enabled:
             if callsign in self._storage:
-                return self._storage[callsign]
+                try:
+                    return self._storage[callsign]
+                except (ValueError, KeyError):
+                    # Catch exceptions due to e.g. old versions of objects in the cache that are no longer compatible,
+                    # and remove them from the cache.
+                    del self._storage[callsign]
+                    return None
             else:
                 c = self._perform_new_lookup(callsign, lookup_credentials)
                 if c:

providers/callsigndata/file_download_callsign_data_provider.py

@@ -32,11 +32,15 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
-        self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} callsign data worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/callsigndata/hamqth.py

@@ -124,8 +124,8 @@ class HamQTH(APIQueryCallsignDataProvider):
         lat = None
         lon = None
         if (
-            "latitude" in data
+            data.get("latitude") is not None
-            and "longitude" in data
+            and data.get("longitude") is not None
             and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
             and -89.9 < float(data["latitude"]) < 89.9
         ):
@@ -134,7 +134,7 @@ class HamQTH(APIQueryCallsignDataProvider):
 
         # Check for sensible grids
         grid = None
-        if "grid" in data and not data["grid"].startswith("AA00"):
+        if data.get("grid") and not data["grid"].startswith("AA00"):
             grid = data["grid"]
 
         return Callsign(
@@ -147,8 +147,8 @@ class HamQTH(APIQueryCallsignDataProvider):
             latitude=lat,
             longitude=lon,
             grid=grid,
-            dxcc_id=int(data["adif"]) if "adif" in data else None,
+            dxcc_id=int(data["adif"]) if data.get("adif") is not None else None,
-            cq_zone=int(data["cq"]) if "cq" in data else None,
+            cq_zone=int(data["cq"]) if data.get("cq") is not None else None,
-            itu_zone=int(data["itu"]) if "itu" in data else None,
+            itu_zone=int(data["itu"]) if data.get("itu") is not None else None,
             location_source=LocationSourceForCallsign.HOME_QTH,
         )

providers/callsigndata/qrz.py

@@ -150,8 +150,8 @@ class QRZ(APIQueryCallsignDataProvider):
         lat = None
         lon = None
         if (
-            "latitude" in data
+            data.get("latitude") is not None
-            and "longitude" in data
+            and data.get("longitude") is not None
             and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
             and -89.9 < float(data["latitude"]) < 89.9
         ):
@@ -160,7 +160,7 @@ class QRZ(APIQueryCallsignDataProvider):
 
         # Check for sensible grids
         grid = None
-        if "grid" in data and not data["grid"].startswith("AA00"):
+        if data.get("grid") and not data["grid"].startswith("AA00"):
             grid = data["grid"]
 
         return Callsign(
@@ -173,8 +173,8 @@ class QRZ(APIQueryCallsignDataProvider):
             latitude=lat,
             longitude=lon,
             grid=grid,
-            dxcc_id=int(data["adif"]) if "adif" in data else None,
+            dxcc_id=int(data["adif"]) if data.get("adif") is not None else None,
-            cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
+            cq_zone=int(data["cqzone"]) if data.get("cqzone") is not None else None,
-            itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
+            itu_zone=int(data["ituzone"]) if data.get("ituzone") is not None else None,
             location_source=LocationSourceForCallsign.HOME_QTH,
         )

providers/sigrefdata/dme.py

@@ -53,7 +53,7 @@ class DME(LocalFileSIGRefDataProvider):
 
                 # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
                 # of the data in this case
-                if self._stop:
+                if self._stop_event.is_set():
                     break
 
                 # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time

providers/sigrefdata/file_download_sig_ref_data_provider.py

@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Event, Thread
+from threading import Thread
 
 import pytz
 from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
@@ -21,19 +21,21 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         self._url = url
         self._poll_interval = poll_interval
         self._thread = None
-        self._stop_event = Event()
         self._url_data_cache = URLDataCache(f"sigrefdata_{sig_name}")
 
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
-        self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}")
+        self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         super().stop()
-        self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.sig_name} SIG ref data worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/sigrefdata/sig_ref_data_provider.py

@@ -1,5 +1,6 @@
 import logging
 from datetime import datetime
+from threading import Event
 
 import pytz
 
@@ -19,7 +20,7 @@ class SIGRefDataProvider:
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
         self.reference_count = 0
-        self._stop = False
+        self._stop_event = Event()
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -30,20 +31,22 @@ class SIGRefDataProvider:
         """Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
         super()."""
 
-        self._stop = True
+        self._stop_event.set()
 
     def _add_data(self, new_data):
         """Add all the provided reference data objects to the data store."""
 
-        # with transact() batches all writes together to save making thousands of individual sqlite writes
+        # with transact() batches all writes together to save making thousands of individual sqlite writes. However,
-        with DATA_STORE.sigrefs.transact():
+        # that means that each provider holds the lock while it writes, and the default behaviour for other attempted
+        # transact()s is to fail if they can't get the lock (?!). This behaviour is fixed by retry=True.
+        with DATA_STORE.sigrefs.transact(retry=True):
             for d in new_data:
                 DATA_STORE.sigrefs.set(f"{self.sig_name}:{d.id}", d)
 
                 # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
                 # within the first few minutes of startup, we need a way to abort this expensive process of filling up the
                 # disk cache.
-                if self._stop:
+                if self._stop_event.is_set():
                     break
 
         self.reference_count = len(new_data)

providers/sigrefdata/toilets.py

@@ -35,7 +35,7 @@ class Toilets(LocalFileSIGRefDataProvider):
 
                 # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
                 # of the data in this case
-                if self._stop:
+                if self._stop_event.is_set():
                     break
 
         return new_data

providers/solarconditions/giroionosonde.py

@@ -67,11 +67,15 @@ class GIROIonosonde(SolarConditionsProvider):
 
     def start(self):
         logger.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
-        self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider")
+        self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning("GIRO ionosonde worker thread did not exit on time and will be killed.")
 
     def _run(self):
         # Real interval at which we poll is the "once per hour" divided by the number of stations, so each one gets

providers/solarconditions/http_solar_conditions_provider.py

@@ -25,11 +25,15 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
 
     def start(self):
         logger.info(f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
-        self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} solar conditions worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/solarconditions/kc2gprop.py

@@ -32,11 +32,15 @@ class KC2GProp(SolarConditionsProvider):
 
     def start(self):
         logger.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
-        self._thread = Thread(target=self._run, name="KC2GPropProvider")
+        self._thread = Thread(target=self._run, name="KC2GPropProvider", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning("KC2G ionosonde worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/spot/aprsis.py

@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Thread
+from threading import Event, Thread
 
 import aprslib
 import pytz
@@ -17,49 +17,69 @@ class APRSIS(SpotProvider):
 
     def __init__(self, provider_config):
         super().__init__("APRS-IS", provider_config)
-        self._thread = Thread(target=self._connect, name="APRSISSpotProvider")
+        self._thread = None
-        self._thread.daemon = True
         self._aprsis = None
+        self._stop_event = Event()
 
     def start(self):
+        self._thread = Thread(target=self._run, name="APRSISSpotProvider", daemon=True)
         self._thread.start()
 
-    def _connect(self):
+    def _run(self):
-        self._aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN)
+        while not self._stop_event.is_set():
-        self.status = "Connecting"
+            try:
-        logger.info("APRS-IS connecting...")
+                self._aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN)
-        self._aprsis.connect()
+                self.status = "Connecting"
-        self._aprsis.consumer(self._handle)
+                logger.info("APRS-IS connecting...")
-        logger.info("APRS-IS connected.")
+                self._aprsis.connect()
+                logger.info("APRS-IS connected.")
+                self._aprsis.consumer(self._handle, immortal=True)
+
+            except Exception:
+                if not self._stop_event.is_set():
+                    self.status = "Error"
+                    logger.exception("Exception in APRS-IS provider")
+
+            if not self._stop_event.is_set():
+                self._stop_event.wait(timeout=5)
 
     def stop(self):
         self.status = "Shutting down"
-        self._aprsis.close()
+        self._stop_event.set()
-        self._thread.join()
+        if self._aprsis:
+            self._aprsis.close()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("APRS-IS worker thread did not exit on time and will be killed.")
 
     def _handle(self, data):
-        # Split SSID in "from" call and store separately
+        try:
-        from_parts = str(data["from"]).split("-")
+            # Split SSID in "from" call and store separately
-        dx_call = from_parts[0].upper()
+            from_parts = str(data["from"]).split("-")
-        dx_ssid = from_parts[1].upper() if len(from_parts) > 1 else None
+            dx_call = from_parts[0].upper()
-        via_parts = str(data["via"]).split("-")
+            dx_ssid = from_parts[1].upper() if len(from_parts) > 1 else None
-        de_call = via_parts[0].upper()
+            via_parts = str(data["via"]).split("-")
-        de_ssid = via_parts[1].upper() if len(via_parts) > 1 else None
+            de_call = via_parts[0].upper()
-        spot = Spot(
+            de_ssid = via_parts[1].upper() if len(via_parts) > 1 else None
-            source="APRS-IS",
+            spot = Spot(
-            dx_call=dx_call,
+                source="APRS-IS",
-            dx_ssid=dx_ssid,
+                dx_call=dx_call,
-            de_call=de_call,
+                dx_ssid=dx_ssid,
-            de_ssid=de_ssid,
+                de_call=de_call,
-            comment=str(data["comment"]) if "comment" in data else None,
+                de_ssid=de_ssid,
-            dx_latitude=float(data["latitude"]) if "latitude" in data else None,
+                comment=str(data["comment"]) if "comment" in data else None,
-            dx_longitude=float(data["longitude"]) if "longitude" in data else None,
+                dx_latitude=float(data["latitude"]) if data.get("latitude") is not None else None,
-            time=datetime.now(pytz.UTC).timestamp(),
+                dx_longitude=float(data["longitude"]) if data.get("longitude") is not None else None,
-        )  # APRS-IS spots are live so we can assume spot time is "now"
+                time=datetime.now(pytz.UTC).timestamp(),
+            )  # APRS-IS spots are live so we can assume spot time is "now"
 
-        # Add to our list
+            # Add to our list
-        self._submit(spot)
+            self._submit(spot)
 
-        self.status = "OK"
+            self.status = "OK"
-        self.last_update_time = datetime.now(pytz.UTC)
+            self.last_update_time = datetime.now(pytz.UTC)
-        logger.debug("Data received from APRS-IS.")
+            logger.debug("Data received from APRS-IS.")
+
+        except Exception:
+            logger.exception("Exception handling APRS-IS packet")

providers/spot/dxcluster.py

@@ -1,8 +1,7 @@
 import logging
 import re
 from datetime import datetime
-from threading import Thread
+from threading import Event, Thread
-from time import sleep
 
 import pytz
 import telnetlib3
@@ -41,23 +40,26 @@ class DXCluster(SpotProvider):
             self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
         )
         self._telnet = None
-        self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}")
+        self._thread = None
-        self._thread.daemon = True
+        self._stop_event = Event()
-        self._running = True
 
     def start(self):
+        self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
-        self._running = False
+        self._stop_event.set()
         if self._telnet:
             self._telnet.close()
-        self._thread.join()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning(f"DX Cluster {self._hostname} worker thread did not exit on time and will be killed.")
 
     def _handle(self):
-        while self._running:
+        while not self._stop_event.is_set():
             connected = False
-            while not connected and self._running:
+            while not connected and not self._stop_event.is_set():
                 try:
                     self.status = "Connecting"
                     logger.info(f"DX Cluster {self._hostname} connecting...")
@@ -69,14 +71,14 @@ class DXCluster(SpotProvider):
                 except ConnectionRefusedError:
                     self.status = "Error"
                     logger.warning(f"Connection refused to DX cluster {self._hostname}")
-                    sleep(300)
+                    self._stop_event.wait(timeout=300)
                 except Exception:
                     self.status = "Error"
                     logger.exception(f"Exception while connecting to DX Cluster Provider ({self._hostname}).")
-                    sleep(5)
+                    self._stop_event.wait(timeout=5)
 
             self.status = "Waiting for Data"
-            while connected and self._running:
+            while connected and not self._stop_event.is_set():
                 try:
                     # Check new telnet info against regular expression
                     telnet_output = self._telnet.read_until("\n".encode("latin-1"))
@@ -106,19 +108,19 @@ class DXCluster(SpotProvider):
 
                 except EOFError:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Restarting"
                         logger.warning(f"Disconnected from DX Cluster {self._hostname}. Reconnecting...")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"DX Cluster {self._hostname} shutting down...")
                         self.status = "Shutting down"
                 except Exception:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Error"
                         logger.exception(f"Exception in DX Cluster Provider ({self._hostname})")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"DX Cluster {self._hostname} shutting down...")
                         self.status = "Shutting down"

providers/spot/http_spot_provider.py

@@ -28,12 +28,16 @@ class HTTPSpotProvider(SpotProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} spot API every {self._poll_interval!s} seconds.")
-        self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
         self._wakeup_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} spot worker thread did not exit on time and will be killed.")
 
     def force_poll(self):
         """Trigger an immediate poll without waiting for the normal interval."""

providers/spot/rbn.py

@@ -1,8 +1,7 @@
 import logging
 import re
 from datetime import datetime
-from threading import Thread
+from threading import Event, Thread
-from time import sleep
 
 import pytz
 import telnetlib3
@@ -30,23 +29,26 @@ class RBN(SpotProvider):
         super().__init__(name, provider_config)
         self._port = provider_config["port"]
         self._telnet = None
-        self._thread = Thread(target=self._handle, name=f"RBNSpotProvider-{self.name}")
+        self._thread = None
-        self._thread.daemon = True
+        self._stop_event = Event()
-        self._running = True
 
     def start(self):
+        self._thread = Thread(target=self._handle, name=f"RBNSpotProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
-        self._running = False
+        self._stop_event.set()
         if self._telnet:
             self._telnet.close()
-        self._thread.join()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning(f"RBN (port {self._port!s}) worker thread did not exit on time and will be killed.")
 
     def _handle(self):
-        while self._running:
+        while not self._stop_event.is_set():
             connected = False
-            while not connected and self._running:
+            while not connected and not self._stop_event.is_set():
                 try:
                     self.status = "Connecting"
                     logger.info(f"RBN port {self._port!s} connecting...")
@@ -58,10 +60,10 @@ class RBN(SpotProvider):
                 except Exception:
                     self.status = "Error"
                     logger.exception(f"Exception while connecting to RBN (port {self._port!s}).")
-                    sleep(5)
+                    self._stop_event.wait(timeout=5)
 
             self.status = "Waiting for Data"
-            while connected and self._running:
+            while connected and not self._stop_event.is_set():
                 try:
                     # Check new telnet info against regular expression
                     telnet_output = self._telnet.read_until("\n".encode("latin-1"))
@@ -91,19 +93,19 @@ class RBN(SpotProvider):
 
                 except EOFError:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Restarting"
                         logger.warning(f"Disconnected from RBN provider (port {self._port!s}). Reconnecting...")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"RBN provider (port {self._port!s}) shutting down...")
                         self.status = "Shutting down"
                 except Exception:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Error"
                         logger.exception(f"Exception in RBN provider (port {self._port!s})")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"RBN provider (port {self._port!s}) shutting down...")
                         self.status = "Shutting down"

providers/spot/sse_spot_provider.py

@@ -65,7 +65,7 @@ class SSESpotProvider(SpotProvider):
                     self._url,
                     headers=HTTP_HEADERS,
                     latest_event_id=self._last_event_id,
-                    timeout=10,
+                    timeout=30,
                     on_open=self._on_open,
                     on_error=self._on_error,
                 ) as event_source:

providers/spot/ukpacketnet.py

@@ -42,7 +42,7 @@ class UKPacketNet(HTTPSpotProvider):
                                 )
                                 comment = (
                                     f"{comment} {listed_port['baud']!s} baud"
-                                    if "baud" in listed_port and listed_port["baud"] > 0
+                                    if listed_port.get("baud") and listed_port["baud"] > 0
                                     else comment
                                 )
 
@@ -50,7 +50,7 @@ class UKPacketNet(HTTPSpotProvider):
                                 # very hacky but a lot of node comments contain their frequency as the first or second
                                 # word of their comment, but not in the proper data structure field.
                                 freq = (
-                                    listed_port["freq"] if "freq" in listed_port and listed_port["freq"] > 0 else None
+                                    listed_port["freq"] if listed_port.get("freq") and listed_port["freq"] > 0 else None
                                 )
                                 if not freq and comment:
                                     possible_freq = comment.split(" ")[0].upper().replace("MHZ", "")

providers/spot/websocket_spot_provider.py

@@ -1,7 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Thread
+from threading import Event, Thread
-from time import sleep
 
 import pytz
 from websocket import create_connection
@@ -20,22 +19,24 @@ class WebsocketSpotProvider(SpotProvider):
         self._url = url
         self._ws = None
         self._thread = None
-        self._stopped = False
+        self._stop_event = Event()
         self._last_event_id = None
 
     def start(self):
         logger.info(f"Set up websocket connection to {self.name} spot API.")
-        self._stopped = False
+        self._stop_event.clear()
         self._thread = Thread(target=self._run, name=f"WebsocketSpotProvider-{self.name}")
         self._thread.daemon = True
         self._thread.start()
 
     def stop(self):
-        self._stopped = True
+        self._stop_event.set()
         if self._ws:
             self._ws.close()
         if self._thread:
-            self._thread.join()
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} websocket worker thread did not exit on time and will be killed.")
 
     def _on_open(self):
         self.status = "Waiting for Data"
@@ -44,14 +45,19 @@ class WebsocketSpotProvider(SpotProvider):
         self.status = "Connecting"
 
     def _run(self):
-        while not self._stopped:
+        while not self._stop_event.is_set():
             try:
                 logger.debug(f"Connecting to {self.name} spot API...")
                 self.status = "Connecting"
                 self._ws = create_connection(self._url, header=HTTP_HEADERS)
                 self.status = "Connected"
-                data = self._ws.recv()
+
-                if data:
+                # Keep reading from this same connection until it drops or we're asked to stop, rather than
+                # reconnecting for every message.
+                while not self._stop_event.is_set():
+                    data = self._ws.recv()
+                    if not data:
+                        break
                     try:
                         new_spot = self._ws_message_to_spot(data)
                         if new_spot:
@@ -69,7 +75,16 @@ class WebsocketSpotProvider(SpotProvider):
                 logger.exception(f"Exception in Websocket Spot Provider ({self.name})")
             else:
                 self.status = "Disconnected"
-            sleep(5)  # Wait before trying to reconnect
+            finally:
+                if self._ws:
+                    try:
+                        self._ws.close()
+                    except Exception:
+                        # No problem, we were getting rid of this object anyway.
+                        pass
+                    self._ws = None
+            if not self._stop_event.is_set():
+                self._stop_event.wait(timeout=5)  # Wait before trying to reconnect
 
     def _ws_message_to_spot(self, b):
         """Convert a WS message received from the API into a spot. The exact message data (in bytes) is provided here so the

providers/spot/wwbota.py

@@ -36,7 +36,7 @@ class WWBOTA(SSESpotProvider):
             dx_call=source_spot["call"].upper(),
             de_call=source_spot["spotter"].upper(),
             freq=float(source_spot["freq"]) * 1000000,
-            mode=Mode.from_name(source_spot["mode"].upper()) if "mode" in source_spot else None,
+            mode=Mode.from_name(source_spot["mode"].upper()) if source_spot.get("mode") else None,
             comment=source_spot["comment"],
             sig="WWBOTA",
             sig_refs=refs,

providers/staticdata/file_download_static_data_provider.py

@@ -29,11 +29,15 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.")
-        self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} static data worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "spothole"
-version = "2.1-pre"
+version = "2.1.1"
 authors = [
     { name = "Ian Renton", email = "ian@ianrenton.com" },
 ]

requirements.txt

@@ -1,6 +1,6 @@
 pyyaml~=6.0.3
 requests-cache~=1.2.1
-pyhamtools~=0.12.0
+pyhamtools~=0.13.2
 telnetlib3~=2.0.8
 pytz~=2025.2
 requests~=2.32.4

spothole.py

@@ -5,25 +5,35 @@ import signal
 import sys
 
 from core.cleanup import CLEANUP_TIMER
-from core.config import LOG_LEVEL, SERVER_OWNER_CALLSIGN
+from core.config import LOG_LEVEL, SERVER_OWNER_CALLSIGN, TELNET_SERVER_ENABLED, TELNET_SERVER_PORT
 from core.constants import SOFTWARE_VERSION
 from core.data_providers import DATA_PROVIDERS
 from core.data_store import DATA_STORE
 from core.status_reporter import StatusReporter
-from server.webserver import WEB_SERVER
+from telnetserver.telnetserver import TELNET_SERVER
+from webserver.webserver import WEB_SERVER
 
 logger = logging.getLogger(__name__)
 
+_shutdown_in_progress = False
+
 
 def shutdown(_signum=None, _frame=None):
     """Shutdown function"""
 
+    # Check if this is the second time a shutdown was asked for, if so immediately kill the program.
+    global _shutdown_in_progress
+    if _shutdown_in_progress:
+        os._exit(1)
+    _shutdown_in_progress = True
+
     logger.info("Stopping program...")
     WEB_SERVER.stop()
+    TELNET_SERVER.stop()
     DATA_PROVIDERS.stop()
     CLEANUP_TIMER.stop()
     DATA_STORE.close()
-    os._exit(0)
+    logger.info("Stopped.")
 
 
 # Main function
@@ -41,8 +51,9 @@ if __name__ == "__main__":
     logger.info("Starting...")
     logger.info(f"This is Spothole version {SOFTWARE_VERSION}. This instance is run by {SERVER_OWNER_CALLSIGN}.")
 
-    # Shut down gracefully on SIGINT
+    # Shut down gracefully on SIGINT or SIGTERM
     signal.signal(signal.SIGINT, shutdown)
+    signal.signal(signal.SIGTERM, shutdown)
 
     # Set up data store
     DATA_STORE.setup()
@@ -57,10 +68,16 @@ if __name__ == "__main__":
     status_reporter = StatusReporter(run_interval=5)
     status_reporter.start()
 
+    # Run the telnet server
+    if TELNET_SERVER_ENABLED:
+        TELNET_SERVER.start(port=TELNET_SERVER_PORT)
+
     # Set up the web server
     WEB_SERVER.setup()
 
-    # Run the web server. This is the blocking call that keeps the application running in the main thread, so this must
+    # Run the web server
-    # be the last thing we do. web_server.stop() triggers an await condition in the web server which finishes the main
-    # thread.
     WEB_SERVER.start()
+
+    # Block the main thread until a termination signal arrives and shutdown() is running.
+    while not _shutdown_in_progress:
+        signal.pause()

static/apidocs/openapi.yml

@@ -17,14 +17,17 @@ info:
     
     ### 2.1
     
-    * Added DTMBA, FEA, BIWOTA, COTA & PGA SIGs
+    * Added AMSAT, EME, DTMBA, FEA, BIWOTA, COTA & PGA SIGs
     * Removed the distinction between LSB & USB (both will now show as SSB) and between the various digital voice modes, which will now show as DV.
     * Added `sig_type`, `icon`, `region_flag` and `refs_globally_unique` to SIG data
     * Added `icon` to spot and alert data
     * Added `alert_type` and `url` to alert data
     * Added `contests_skip_max_duration_check` to alert query parameters
     * SIG reference types (e.g. "Park") are now capitalised to match other enums
-    
+    * Added the ability to get only certain fields of spots and alerts from the API by using the `fields` query parameter.
+    * Replace `last_page_access` with `page_requests_per_hour` and `last_api_access` with `api_requests_per_hour` in the web server stats.
+    * Added `sse_client_count` to the `webserver` stats, and a new `telnet` object with `client_count`.
+
     ### 2.0
 
     * **Breaking change:** The "add spot" API has changed to enable future support for upstream submission to the spotting services associated with various SIGs. Instead of just posting the spot object itself as the JSON content of the POST, this has moved into a `spot` object within the structure. A new `handling` object alongside it contains the `submit_upstream`, `upstream_provider`, `upstream_credentials`, and `captcha_token` fields which control the server handling of the spot.
@@ -140,6 +143,7 @@ paths:
         - $ref: '#/components/parameters/SpotTextIncludes'
         - $ref: '#/components/parameters/SpotNeedsGoodLocation'
         - $ref: '#/components/parameters/SpotAllowQrt'
+        - $ref: '#/components/parameters/SpotFields'
         - $ref: '#/components/parameters/QrzUsername'
         - $ref: '#/components/parameters/QrzPassword'
         - $ref: '#/components/parameters/QrzSessionKey'
@@ -180,6 +184,7 @@ paths:
         - $ref: '#/components/parameters/SpotTextIncludes'
         - $ref: '#/components/parameters/SpotNeedsGoodLocation'
         - $ref: '#/components/parameters/SpotAllowQrt'
+        - $ref: '#/components/parameters/SpotFields'
         - $ref: '#/components/parameters/QrzUsername'
         - $ref: '#/components/parameters/QrzPassword'
         - $ref: '#/components/parameters/QrzSessionKey'
@@ -217,6 +222,7 @@ paths:
         - $ref: '#/components/parameters/AlertDxContinent'
         - $ref: '#/components/parameters/AlertDxCallIncludes'
         - $ref: '#/components/parameters/AlertTextIncludes'
+        - $ref: '#/components/parameters/AlertFields'
         - $ref: '#/components/parameters/QrzUsername'
         - $ref: '#/components/parameters/QrzPassword'
         - $ref: '#/components/parameters/QrzSessionKey'
@@ -252,6 +258,7 @@ paths:
         - $ref: '#/components/parameters/AlertDxContinent'
         - $ref: '#/components/parameters/AlertDxCallIncludes'
         - $ref: '#/components/parameters/AlertTextIncludes'
+        - $ref: '#/components/parameters/AlertFields'
         - $ref: '#/components/parameters/QrzUsername'
         - $ref: '#/components/parameters/QrzPassword'
         - $ref: '#/components/parameters/QrzSessionKey'
@@ -657,6 +664,16 @@ components:
       schema:
         type: boolean
         default: true
+    SpotFields:
+      name: fields
+      in: query
+      description: >
+        Filter the fields you receive in each spot, to conserve data bandwidth for constrained applications.
+        Supply a comma-separated list of the fields you want to receive, using the names of the fields returned by the
+        `/spots` call, e.g. `id,dx_call,freq,mode,time`. If the "fields" parameter is not supplied, all fields will be
+        included in the spot data.
+      schema:
+        type: string
     AlertMaxDuration:
       name: max_duration
       in: query
@@ -795,6 +812,16 @@ components:
         you will get all the more recent alerts back, without duplicating the previous latest spot.
       schema:
         type: number
+    AlertFields:
+      name: fields
+      in: query
+      description: >
+        Filter the fields you receive in each alert, to conserve data bandwidth for constrained applications.
+        Supply a comma-separated list of the fields you want to receive, using the names of the fields returned by the
+        `/alerts` call, e.g. `id,dx_calls,freqs_modes,start_time`. If the "fields" parameter is not supplied, all fields
+        will be included in the alert data.
+      schema:
+        type: string
     CallParam:
       name: call
       in: query
@@ -874,7 +901,12 @@ components:
         - WAB
         - WAI
         - DME
+        - DMF
         - FEA
+        - DMUE
+        - DMVE
+        - DCE
+        - DEFE
         - DTMBA
         - BIWOTA
         - COTA
@@ -2075,14 +2107,25 @@ components:
               type: string
               description: The status of the web server
               example: OK
-            last_page_access:
+            page_requests_per_hour:
-              type: number
+              type: integer
-              description: The last time a page was accessed on the web server, UTC seconds since UNIX epoch.
+              description: The number of page requests handled by the web server in the last hour
-              example: 1759579508
+              example: 123
-            last_api_access:
+            api_requests_per_hour:
-              type: number
+              type: integer
-              description: The last time an API endpoint was accessed on the web server, UTC seconds since UNIX epoch.
+              description: The number of API requests handled by the web server in the last hour
-              example: 1759579508
+              example: 123
+            sse_client_count:
+              type: integer
+              description: The number of clients currently connected to SSE streams
+              example: 5
+        "telnet":
+          type: object
+          properties:
+            client_count:
+              type: integer
+              description: The number of clients currently connected to the telnet server
+              example: 2
         spot_providers:
           type: array
           description: An array of all the spot providers.

static/js/alerts.js

@@ -54,7 +54,7 @@ function updateTable() {
     const showDX = $("#tableShowDX")[0].checked;
     const showFreqsModes = $("#tableShowFreqsModes")[0].checked;
     const showComment = $("#tableShowComment")[0].checked;
-    const showSource = $("#tableShowSource")[0].checked;
+    const showType = $("#tableShowType")[0].checked;
     const showRef = $("#tableShowRef")[0].checked;
 
     // Populate table with headers
@@ -75,8 +75,8 @@ function updateTable() {
     if (showComment) {
         table.find('thead tr').append(`<th class='bg-primary-subtle hideonmobile'>Comment</th>`);
     }
-    if (showSource) {
+    if (showType) {
-        table.find('thead tr').append(`<th class='bg-primary-subtle hideonmobile'>Source</th>`);
+        table.find('thead tr').append(`<th class='bg-primary-subtle hideonmobile'>Type</th>`);
     }
     if (showRef) {
         table.find('thead tr').append(`<th class='bg-primary-subtle hideonmobile'>Ref.</th>`);
@@ -151,7 +151,7 @@ function addAlertRowsToTable(tbody, alerts) {
         const showDX = $("#tableShowDX")[0].checked;
         const showFreqsModes = $("#tableShowFreqsModes")[0].checked;
         const showComment = $("#tableShowComment")[0].checked;
-        const showSource = $("#tableShowSource")[0].checked;
+        const showType = $("#tableShowType")[0].checked;
         const showRef = $("#tableShowRef")[0].checked;
 
         // Get times for the alert, and convert to local time if necessary.
@@ -232,14 +232,38 @@ function addAlertRowsToTable(tbody, alerts) {
         if (a["comment"] != null) {
             commentText = escapeHtml(a["comment"]);
         }
-        if (a["url"] != null) {
+
-            commentText = `<a href="${escapeHtml(a['url'])}" target="_new">${commentText}</a>`;
+        // Format extra text, like URL and attribution
+        let subComment = a["url"] != null || a["source"] === "NG3K" || a["source"] === "WA7BNM Contest Calendar";
+        if (subComment) {
+            let subCommentText = ""
+            if (a["url"] != null) {
+                subCommentText += `<a href="${escapeHtml(a['url'])}" target="_new" style="text-decoration: none">More info</a>`;
+            }
+            if ((a["source"] === "NG3K" || a["source"] === "WA7BNM Contest Calendar")) {
+                if (subCommentText !== "") {
+                    subCommentText += " | ";
+                }
+                subCommentText += `From ${a["source"]}`;
+            }
+            commentText += `<div class="mt-2 small text-secondary">${subCommentText}</div>`;
         }
 
-        // Sig or fallback to source
+
-        let sigSourceText = a["source"];
+        // Type, SIG or fallback to source
-        if (a["sig"]) {
+        let sigTypeText = a["source"];
-            sigSourceText = a["sig"];
+        if (a["alert_type"] === "CONTEST") {
+            sigTypeText = "Contest";
+        } else if (a["alert_type"] === "DXPEDITION") {
+            sigTypeText = "DXpedition";
+        } else if (a["alert_type"] === "SATELLITE") {
+            sigTypeText = "Satellite";
+        } else if (a["alert_type"] === "XOTA") {
+            if (a["sig"]) {
+                sigTypeText = a["sig"];
+            } else {
+                sigTypeText = "xOTA";
+            }
         }
 
         // Format sig_refs
@@ -272,8 +296,8 @@ function addAlertRowsToTable(tbody, alerts) {
         if (showComment) {
             $tr.append(`<td class='hideonmobile'>${commentText}</td>`);
         }
-        if (showSource) {
+        if (showType) {
-            $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid ${a["icon"]}'></i></span> ${sigSourceText}</td>`);
+            $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid ${a["icon"]}'></i></span> ${sigTypeText}</td>`);
         }
         if (showRef) {
             $tr.append(`<td class='hideonmobile'>${sig_refs}</td>`);
@@ -290,7 +314,7 @@ function addAlertRowsToTable(tbody, alerts) {
         }
 
         const $td2 = $("<td colspan='100'>");
-        if (showSource) {
+        if (showType) {
             $td2.append(`<span class='icon-wrapper'><i class='fa-solid ${a["icon"]}'></i></span> `);
         }
         if (showRef) {

static/js/spots.js

@@ -538,6 +538,15 @@ function displayIntroBox() {
     $("#intro-box-dismiss").click(function () {
         localStorage.setItem("intro-box-dismissed", true);
     });
+
+    // Do the same with the "telnet" intro box, but only show it if the user has dismissed the normal intro box once,
+    // to avoid two boxes on first page load.
+    if (localStorage.getItem("intro-box-telnet-dismissed") == null && localStorage.getItem("intro-box-dismissed") != null) {
+        $("#intro-box-telnet").show();
+    }
+    $("#intro-box-telnet-dismiss").click(function () {
+        localStorage.setItem("intro-box-telnet-dismissed", true);
+    });
 }
 
 // Mark a callsign-band-mode combination as worked (or unmark it). Persist this to localStorage.

static/js/status.js

@@ -9,8 +9,10 @@ function loadStatus() {
         $("#total-alerts").text(jsonData["num_alerts"]);
 
         $("#web-server-status").text(jsonData["webserver"]["status"]);
-        $("#web-server-last-api").text(moment.unix(jsonData["webserver"]["last_api_access"]).utc().fromNow());
+        $("#web-server-api-rate").text(jsonData["webserver"]["api_requests_per_hour"] + " / hour");
-        $("#web-server-last-page").text(moment.unix(jsonData["webserver"]["last_page_access"]).utc().fromNow());
+        $("#web-server-page-rate").text(jsonData["webserver"]["page_requests_per_hour"] + " / hour");
+        $("#web-server-sse-clients").text(jsonData["webserver"]["sse_client_count"]);
+        $("#telnet-server-clients").text(jsonData["telnet"]["client_count"]);
 
         $("#cleanup-status").text(jsonData["cleanup"]["status"]);
         $("#cleanup-last-ran").text((jsonData["cleanup"]["last_ran"] > 0) ? moment.unix(jsonData["cleanup"]["last_ran"]).utc().fromNow() : "N/A");

telnetserver/telnetserver.py

@@ -0,0 +1,227 @@
+import asyncio
+import logging
+import threading
+from datetime import datetime
+
+import pytz
+from pyhamtools import callinfo
+
+from core.config import SERVER_OWNER_CALLSIGN
+from core.constants import SOFTWARE_VERSION
+from core.data_store import DATA_STORE
+from data.spot import Spot
+
+logger = logging.getLogger(__name__)
+
+BANNER = (
+    """
+           ==                                                                  
+          ====                      #####    #####               ####          
+         ==  ==                    ..###    ..###               ..###          
+        == ###==########   ######  #######   .#######    ######  .###   ###### 
+       ==###.. ..###..### ###..###...###.    .###..###  ###..### .###  ###..###
+      ==###.### .### .###.### .###  .###     .### .### .### .### .### .####### 
+     == .###.###.### .###.### .###  .### ### .### .### .### .### .### .###...  
+    ==   ...### .####### ..######   ..#####  #### #####..######  #####..###### 
+   ==    .###.  .###=..   ......     .....  .... .....  ......  .....  ......  
+  == %%  ... %%%%###==                                                         
+ ==  %%%%%%%%%%%#####==                                                        \r\n"""
+    + "==             .....  =="
+    + f"Welcome to Spothole v{SOFTWARE_VERSION}".rjust(56)
+    + "\r\n"
+    + "========================"
+    + f"This server is run by {SERVER_OWNER_CALLSIGN}".rjust(56)
+)
+
+MOTD = (
+    "Spothole's telnet server is a new feature and may not work properly in all\r\n"
+    + "loggers. Please give it a try in your logger of choice and let me know if it\r\n"
+    + "(or doesn't!) Please note that DXSpider-like commands are not yet supported\r\n"
+    + "so you are not yet able to filter spots server-side or log in at all."
+)
+
+
+class TelnetServer:
+    """A telnet server designed to provide spots in the same format as DXSpider, for compatibility with desktop loggers."""
+
+    def __init__(self):
+        self._port = None
+        self._running = False
+        self._clients = set()
+        self._loop = None
+        self._thread = None
+        self._shutdown_event = asyncio.Event()
+
+    def start(self, port=7373):
+        """Starts the telnet server"""
+
+        self._port = port
+
+        # Start the telnet server. asyncio.run() needs a coroutine, and threading.Thread needs a plain callable, so
+        # hand Thread the bridge between the two directly rather than writing a one-line wrapper method for it.
+        self._thread = threading.Thread(
+            target=asyncio.run, args=(self._start_internal(),), name="TelnetServer", daemon=True
+        )
+        self._thread.start()
+        logger.debug("Telnet server background thread spawned")
+
+        # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
+        DATA_STORE.spots.add_listener(self.publish)
+        self._running = True
+
+    async def _start_internal(self):
+        """Start method (async). Sets up the telnet server and waits for shutdown."""
+
+        self._loop = asyncio.get_running_loop()
+
+        server = await asyncio.start_server(self._handle_client, "0.0.0.0", self._port)
+        logger.info(f"Telnet server listening on port {self._port}")
+        async with server:
+            await self._shutdown_event.wait()
+
+        await self._stop_internal()
+
+    async def _handle_client(self, reader, writer):
+        """Handles a new client connection"""
+
+        logger.debug("Telnet client connected")
+        self._clients.add(writer)
+
+        # Print banner and MOTD
+        try:
+            text = (
+                BANNER
+                + "\r\n\r\n"
+                + "================================================================================"
+                + "\r\n"
+                + MOTD
+                + "\r\n"
+                + "================================================================================"
+                + "\r\n\r\n"
+            )
+            writer.write(text.encode("ascii"))
+            await writer.drain()
+        except Exception:
+            logger.exception("Exception printing telnet motd")
+
+        # Set up buffer for user input
+        input_buffer = ""
+
+        try:
+            # Read forever, picking out any commands. Currently we just support "exit"
+            while True:
+                data = await reader.read(1024)
+                if not data:
+                    break
+
+                input_buffer, command = self._consume_input(input_buffer, data)
+                if command == "exit":
+                    writer.write(b"Goodbye!\r\n")
+                    await writer.drain()
+                    # Exit the while read loop, this will disconnect the client.
+                    return
+
+        except asyncio.CancelledError:
+            pass
+        except Exception:
+            logger.exception("Exception handling telnet client")
+        finally:
+            logger.debug("Telnet client disconnected")
+            self._clients.remove(writer)
+            writer.close()
+            await writer.wait_closed()
+
+    def stop(self):
+        """Stops the telnet server"""
+
+        self._running = False
+        if self._loop and self._loop.is_running():
+            logger.debug("Stopping telnet server...")
+            self._loop.call_soon_threadsafe(self._shutdown_event.set)
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("Telnet server background thread did not exit on time and will be killed.")
+
+    @property
+    def client_count(self) -> int:
+        return len(self._clients)
+
+    async def _stop_internal(self):
+        """Stops the telnet server"""
+
+        for writer in list(self._clients):
+            try:
+                writer.close()
+                await writer.wait_closed()
+            except Exception:
+                pass
+        self._clients.clear()
+
+    def publish(self, spot: Spot):
+        """Callback from the data store when a spot is added"""
+
+        if self._running and self._clients and self._loop and self._loop.is_running():
+            asyncio.run_coroutine_threadsafe(self._broadcast_spot_internal(spot), self._loop)
+
+    async def _broadcast_spot_internal(self, spot: Spot):
+        """Internal version, run on async loop for thread safety?"""
+
+        # Ensure ASCII formatting for telnet clients
+        encoded_line = self._format_dxspider_spot(spot).encode("ascii", errors="ignore")
+
+        # Try to write to all clients, and in the process find the ones that are disconnected. Iterate over a copy
+        # since a new client can connect (mutating self._clients) while we're awaiting a write below.
+        disconnected_clients = set()
+        for writer in list(self._clients):
+            try:
+                writer.write(encoded_line)
+                await writer.drain()
+            except Exception:
+                disconnected_clients.add(writer)
+
+        # Clean up any disconnected connections caught during writing
+        for writer in disconnected_clients:
+            self._clients.discard(writer)
+
+    @staticmethod
+    def _consume_input(input_buffer: str, data: bytes) -> tuple[str, str | None]:
+        """Handle any input the user gives us, keeping a rolling buffer that we keep passing back through and
+        adding to. Once we get a command, return that as well, so the caller can deal with it."""
+
+        command = None
+        for char in data.decode("ascii", errors="ignore"):
+            if char in ("\r", "\n"):
+                stripped = input_buffer.strip().lower()
+                if stripped:
+                    command = stripped
+                input_buffer = ""
+            elif char in ("\b", "\x7f"):
+                # Handle backspaces
+                input_buffer = input_buffer[:-1]
+            else:
+                input_buffer += char
+        return input_buffer, command
+
+    @staticmethod
+    def _format_dxspider_spot(spot: Spot) -> str:
+        """Formats a spot into the format DXspider uses:
+        DX de CALLSIGN:  FREQUENCY  DX_CALLSIGN  COMMENTS   TIME_UTC.
+        Always use the base call for the spotter to save space. Everything must align properly to parse in clients,
+        and everything must be renderable in ASCII."""
+
+        de_call = f"{callinfo.Callinfo.get_homecall(spot.de_call)[:6] + ':' if spot.de_call else '???:'!s:<7}"
+        frequency = f"{(spot.freq / 1000.0):10.1f}"
+        dx_call = f"{spot.dx_call!s:<12}"
+        comment = f"{spot.comment.encode('ascii', errors='ignore').decode()[:29]:<30}"
+        if spot.time:
+            timestamp = datetime.fromtimestamp(spot.time, tz=pytz.utc).strftime("%H%M") + "Z"
+        else:
+            timestamp = datetime.now(tz=pytz.utc).strftime("%H%M") + "Z"
+
+        # Combine into classic DXSpider output string followed by network line breaks
+        return f"DX de {de_call} {frequency}  {dx_call} {comment} {timestamp}\r\n"
+
+
+# Global object
+TELNET_SERVER = TelnetServer()

templates/about.html

@@ -34,6 +34,13 @@
             like. The usage is explained in more detail in the <a
                     href="https://git.ianrenton.com/ian/spothole/src/branch/main/README.md">README file</a>.
         </li>
+        {% if telnet_server_enabled %}
+        <li>You can use it as a traditional telnet-based source of spots, similar to DXSpider and other software, <b>in
+            your desktop logging application</b>. To do this, set up your logger with the server address
+            <code>{{ telnet_server_address }}</code> and port <code>{{ telnet_server_port }}</code>. You can also access
+            it from a terminal with <code>telnet {{ telnet_server_address }} {{ telnet_server_port }}</code>.
+        </li>
+        {% end %}
         <li>You can <b>write your own client using the Spothole API</b>, using the main Spothole instance to provide
             data, and do whatever you like with it. The README contains guidance on how to do this, and the full API
             docs are linked above. You can also find reference implementations in the form of Spothole's own web-based
@@ -85,7 +92,8 @@
     <p>Spothole can retrieve alerts from: <a href="https://www.ng3k.com/">NG3K</a>, <a href="https://pota.app">POTA</a>,
         <a href="https://www.sota.org.uk/">SOTA</a>, <a href="https://wwff.co/">WWFF</a>, <a
                 href="https://www.parksnpeaks.org/">Parks 'n' Peaks</a>, <a href="https://www.wota.org.uk/">WOTA</a> and
-        <a href="https://www.beachesontheair.com/">BOTA</a>.</p>
+        <a href="https://www.beachesontheair.com/">BOTA</a>. It also fetches contest dates from
+        <a href="https://contestcalendar.com/">WA7BNM Contest Calendar</a> and RSGB contest calendars.</p>
     <p>Spothole can retrieve solar and propagation condition data from <a href="https://www.hamqsl.com">HamQSL</a>, the
         <a href="https://www.swpc.noaa.gov/">NOAA Space Weather Prediction Center</a>, the <a
                 href="https://giro.uml.edu/">Lowell GIRO Data Center</a> and <a href="https://prop.kc2g.com/">prop.kc2g.com</a>
@@ -106,7 +114,7 @@
         Faros de España (FEA), Diploma Muesos de España (DMUE), Diploma Castillos de España (DCE), Diploma Monumentos y
         Vestigios de España (DMVE), Diploma Estaciones de Ferrocarril de España (DEFE), Diploma Teatri Musei e Belle
         Arti (DTMBA), British Inland Waterways on the Air (BIWOTA), Castles on the Air (COTA), Polish Gmina Award (PGA),
-        and Toilets on the Air.</p>
+        Diplôme des Moulins de France (DMF), EME/Moonbounce, Amateur Satellite (AMSAT), and Toilets on the Air.</p>
     <p>As of the time of writing in August 2026, I think Spothole captures most outdoor radio programmes that have a
         defined, downloadable reference list, and almost certainly those that have a spotting/alerting API. If you know
         of one I've missed, please let me know!</p>
@@ -162,6 +170,10 @@
         modify it however you like, you can claim you wrote it and charge people £1000 for a copy, I don't really mind.
         (Please don't do the last one. But if you're using my code for something cool, it would be nice to hear from
         you!)</p>
+    <h4 class="mt-4">What commands are supported in the telnet server?</h4>
+    <p>Currently, <code>exit</code>, and nothing else. Support for some DXSpider-like commands may be added to Spothole
+        in due course, but at the moment if you want to add Spothole as a telnet cluster data source to your desktop
+        logging application, that application must handle filtering itself.</p>
     <h2 id="accuracy" class="mt-4">Data Accuracy</h2>
     <p>Please note that the data coming out of Spothole is only as good as the data going in. People mis-hear and make
         typos when spotting callsigns all the time. There are also plenty of cases where Spothole's data, particularly

templates/add_spot.html

@@ -77,7 +77,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1788600259"></script>
+<script src="/static/js/add-spot.js?v=1789763826"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -83,7 +83,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1788600259"></script>
+<script src="/static/js/alerts.js?v=1789763826"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -76,8 +76,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1788600259"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1789763826"></script>
-<script src="/static/js/bands.js?v=1788600259"></script>
+<script src="/static/js/bands.js?v=1789763826"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1788600258" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1789763825" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -16,10 +16,10 @@
     window.fetchEventSource = fetchEventSource;
 </script>
 
-<script src="/static/js/utils.js?v=1788600258"></script>
+<script src="/static/js/utils.js?v=1789763825"></script>
-<script src="/static/js/ui-ham.js?v=1788600258"></script>
+<script src="/static/js/ui-ham.js?v=1789763825"></script>
-<script src="/static/js/geo.js?v=1788600258"></script>
+<script src="/static/js/geo.js?v=1789763825"></script>
-<script src="/static/js/common.js?v=1788600258"></script>
+<script src="/static/js/common.js?v=1789763825"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/cards/table_columns_alerts.html

@@ -39,9 +39,9 @@
             </div>
             <div class="col">
                 <div class="form-check">
-                    <input class="form-check-input storeable-checkbox" type="checkbox" id="tableShowSource"
+                    <input class="form-check-input storeable-checkbox" type="checkbox" id="tableShowType"
-                           value="tableShowSource" oninput="columnsUpdated();" checked>
+                           value="tableShowType" oninput="columnsUpdated();" checked>
-                    <label class="form-check-label" for="tableShowSource">Source</label>
+                    <label class="form-check-label" for="tableShowType">Source</label>
                 </div>
             </div>
             <div class="col">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1788600259"></script>
+<script src="/static/js/conditions.js?v=1789763825"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -113,8 +113,8 @@
     const CARTODB_API_KEY = "{{ web_ui_options.get('cartodb_api_key', '') }}";
 </script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1788600259"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1789763825"></script>
-<script src="/static/js/map.js?v=1788600259"></script>
+<script src="/static/js/map.js?v=1789763825"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -14,6 +14,18 @@
     </div>
 </div>
 
+{% if telnet_server_enabled %}
+<div id="intro-box-telnet" class="permanently-dismissible-box mt-3">
+    <div class="alert alert-primary alert-dismissible fade show" role="alert">
+        <i class="fa-solid fa-circle-info"></i> <strong>Spothole now has a telnet server!</strong><br/>If you'd like to
+        add Spothole as a source of data to your desktop logging application, now you can. Use server address
+        <code>{{ telnet_server_address }}</code> and port <code>{{ telnet_server_port }}</code>.
+        <button type="button" id="intro-box-telnet-dismiss" class="btn-close" data-bs-dismiss="alert"
+                aria-label="Close"></button>
+    </div>
+</div>
+{% end %}
+
 <div class="mt-3">
     <div id="settingsButtonRow" class="row mb-3">
         <div class="col-md-4 mb-3 mb-md-0">
@@ -113,8 +125,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1788600258"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1789763825"></script>
-<script src="/static/js/spots.js?v=1788600258"></script>
+<script src="/static/js/spots.js?v=1789763825"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -21,9 +21,19 @@
         <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
             <div class="col"><strong>Web Server</strong></div>
             <div class="col">Status: <span id="web-server-status"></span></div>
-            <div class="col">Last API call: <span id="web-server-last-api"></span></div>
+            <div class="col">SSE clients connected: <span id="web-server-sse-clients"></span></div>
-            <div class="col">Last page req: <span id="web-server-last-page"></span></div>
         </div>
+        <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
+            <div class="col"></div>
+            <div class="col">API request rate: <span id="web-server-api-rate"></span></div>
+            <div class="col">Page request rate: <span id="web-server-page-rate"></span></div>
+        </div>
+        {% if telnet_server_enabled %}
+        <div class="row row-cols-1 row-cols-md-4 g-4 mb-4 mb-md-2">
+            <div class="col"><strong>Telnet Server</strong></div>
+            <div class="col">Clients connected: <span id="telnet-server-clients"></span></div>
+        </div>
+        {% end %}
         <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
             <div class="col"><strong>Cleanup Service</strong></div>
             <div class="col">Status: <span id="cleanup-status"></span></div>
@@ -86,7 +96,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1788600259"></script>
+<script src="/static/js/status.js?v=1789763826"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");

webserver/handlers/api/addspot.py

@@ -1,10 +1,8 @@
 import logging
 import re
 import threading
-from datetime import datetime
 from typing import Any
 
-import pytz
 import requests
 import tornado
 from tornado import httputil
@@ -12,7 +10,6 @@ from tornado.web import Application
 
 from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRET_KEY
 from core.constants import UNKNOWN_BAND
-from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import infer_band_from_freq, safe_json_dumps
 from data.spot import Spot
@@ -33,23 +30,15 @@ class APISpotHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._spots = None
-        self._web_server_metrics = None
         self._spot_providers = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, spots, web_server_metrics, spot_providers=None):
+    def initialize(self, spots, spot_providers=None):
         self._spots = spots
-        self._web_server_metrics = web_server_metrics
         self._spot_providers = spot_providers or []
 
     def post(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # Reject if not allowed
             if not ALLOW_SPOTTING:
                 self.set_status(401)

webserver/handlers/api/alerts.py

@@ -10,7 +10,6 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.enums import AlertType
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 
@@ -27,12 +26,10 @@ class APIAlertsHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._alerts = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, alerts, web_server_metrics):
+    def initialize(self, alerts):
         self._alerts = alerts
-        self._web_server_metrics = web_server_metrics
 
     @staticmethod
     def _enrich(alerts, credentials):
@@ -45,12 +42,6 @@ class APIAlertsHandler(tornado.web.RequestHandler):
 
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
@@ -58,8 +49,12 @@ class APIAlertsHandler(tornado.web.RequestHandler):
             # Fetch all alerts matching the query, then optionally enrich with online data
             credentials = extract_credentials(self.request.headers)
             data = get_alert_list_with_filters(self._alerts, query_params)
+            fields = [f.strip() for f in query_params["fields"].split(",")] if "fields" in query_params else []
             if credentials:
                 data = self._enrich(data, credentials)
+            # Filter for only the required fields, if necessary
+            if fields:
+                data = filter_fields(data, fields)
             self.write(safe_json_dumps(data))
             self.set_status(200)
         except ValueError as e:
@@ -78,14 +73,13 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
 
     def __init__(self, application, request, **kwargs: Any):
         self._sse_alert_broadcaster = None
-        self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
+        self._fields = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, sse_alert_broadcaster, web_server_metrics):
+    def initialize(self, sse_alert_broadcaster):
         self._sse_alert_broadcaster = sse_alert_broadcaster
-        self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
         """Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data"""
@@ -94,16 +88,13 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
 
     def open(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
             self._credentials = extract_credentials(self.request.headers)
+            self._fields = (
+                [f.strip() for f in self._query_params["fields"].split(",")] if "fields" in self._query_params else []
+            )
 
             # Flush headers immediately so nginx doesn't time out waiting for a response
             self.write_message("keepalive", "")
@@ -126,10 +117,15 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
         """Callback when a new alert arrives"""
 
         try:
+            # If the new alert matches our param filters, send it to the client. If not, ignore it.
             if alert_allowed_by_query(alert, self._query_params):
+                # Add lookup data if we have credentials
                 if self._credentials:
                     alert = copy.deepcopy(alert)
                     alert.infer_missing(self._credentials)
+                # Filter fields returned if necessary
+                if self._fields:
+                    alert = filter_fields([alert], self._fields)[0]
                 self.write_message(msg=safe_json_dumps(alert))
         except Exception:
             logger.exception("Exception in SSE callback, connection will be closed")
@@ -216,3 +212,9 @@ def alert_allowed_by_query(alert, query):
                 ):
                     return False
     return True
+
+
+def filter_fields(alerts, fields):
+    """Given a list of alert objects, return copies containing only the named fields."""
+
+    return [{k: v for k, v in alert.__dict__.items() if k in fields} for alert in alerts]

webserver/handlers/api/dxstats.py

@@ -11,7 +11,6 @@ from tornado.web import Application
 
 from core.constants import BANDS
 from core.enums import Continent
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 
 logger = logging.getLogger(__name__)
@@ -30,20 +29,13 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._spots = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, spots, web_server_metrics):
+    def initialize(self, spots):
         self._spots = spots
-        self._web_server_metrics = web_server_metrics
 
     def get(self):
         try:
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
             counts = Counter()
 

webserver/handlers/api/lookups.py

@@ -1,9 +1,7 @@
 import logging
 import re
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
@@ -15,7 +13,6 @@ from core.geo_utils import (
     lat_lon_to_cq_zone,
     lat_lon_to_itu_zone,
 )
-from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import safe_json_dumps
@@ -34,20 +31,10 @@ class APILookupCallHandler(tornado.web.RequestHandler):
         request: httputil.HTTPServerRequest,
         **kwargs: Any,
     ):
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, web_server_metrics):
-        self._web_server_metrics = web_server_metrics
-
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
@@ -85,20 +72,10 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
         request: httputil.HTTPServerRequest,
         **kwargs: Any,
     ):
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, web_server_metrics):
-        self._web_server_metrics = web_server_metrics
-
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
@@ -143,20 +120,10 @@ class APILookupGridHandler(tornado.web.RequestHandler):
         request: httputil.HTTPServerRequest,
         **kwargs: Any,
     ):
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, web_server_metrics):
-        self._web_server_metrics = web_server_metrics
-
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}

webserver/handlers/api/options.py

@@ -1,8 +1,6 @@
 import logging
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
@@ -10,7 +8,6 @@ from tornado.web import Application
 from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
 from core.constants import BANDS, PROPAGATION_MODES, SIGS
 from core.enums import Continent, Mode, ModeType
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 
 logger = logging.getLogger(__name__)
@@ -26,23 +23,15 @@ class APIOptionsHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._status_data = None
-        self._web_server_metrics = None
         self._spot_providers = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, status_data, web_server_metrics, spot_providers=None):
+    def initialize(self, status_data, spot_providers=None):
         self._status_data = status_data
-        self._web_server_metrics = web_server_metrics
         self._spot_providers = spot_providers or []
 
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # Build a map of SIG name -> list of provider names that can submit spots for that SIG
             spot_submit_providers = {}
 

webserver/handlers/api/solar_conditions.py

@@ -1,13 +1,10 @@
 import logging
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
 
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 
 logger = logging.getLogger(__name__)
@@ -23,21 +20,13 @@ class APISolarConditionsHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._solar_conditions = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, solar_conditions, web_server_metrics):
+    def initialize(self, solar_conditions):
         self._solar_conditions = solar_conditions
-        self._web_server_metrics = web_server_metrics
 
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             self.write(self._solar_conditions.to_json())
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")

webserver/handlers/api/spots.py

@@ -9,7 +9,6 @@ import tornado_eventsource.handler
 from tornado import httputil
 from tornado.web import Application
 
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 
@@ -26,12 +25,10 @@ class APISpotsHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._spots = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, spots, web_server_metrics):
+    def initialize(self, spots):
         self._spots = spots
-        self._web_server_metrics = web_server_metrics
 
     @staticmethod
     def _enrich(spots, credentials):
@@ -44,21 +41,19 @@ class APISpotsHandler(tornado.web.RequestHandler):
 
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
 
             # Fetch all spots matching the query, then optionally enrich with online data
             credentials = extract_credentials(self.request.headers)
+            fields = [f.strip() for f in query_params["fields"].split(",")] if "fields" in query_params else []
             data = get_spot_list_with_filters(self._spots, query_params)
             if credentials:
                 data = self._enrich(data, credentials)
+            # Filter for only the required fields, if necessary
+            if fields:
+                data = filter_fields(data, fields)
             self.write(safe_json_dumps(data))
             self.set_status(200)
         except ValueError as e:
@@ -77,14 +72,13 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
 
     def __init__(self, application, request, **kwargs: Any):
         self._sse_spot_broadcaster = None
-        self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
+        self._fields = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, sse_spot_broadcaster, web_server_metrics):
+    def initialize(self, sse_spot_broadcaster):
         self._sse_spot_broadcaster = sse_spot_broadcaster
-        self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
         """Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data"""
@@ -95,16 +89,13 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
         """Called once on the client opening a connection, set things up"""
 
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
             self._credentials = extract_credentials(self.request.headers)
+            self._fields = (
+                [f.strip() for f in self._query_params["fields"].split(",")] if "fields" in self._query_params else []
+            )
 
             # Flush headers immediately so nginx doesn't time out waiting for a response
             self.write_message("keepalive", "")
@@ -129,9 +120,13 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
         try:
             # If the new spot matches our param filters, send it to the client. If not, ignore it.
             if spot_allowed_by_query(spot, self._query_params):
+                # Add lookup data if we have credentials
                 if self._credentials:
                     spot = copy.deepcopy(spot)
                     spot.infer_missing(self._credentials)
+                # Filter fields returned if necessary
+                if self._fields:
+                    spot = filter_fields([spot], self._fields)[0]
                 self.write_message(msg=safe_json_dumps(spot))
         except Exception:
             logger.exception("Exception in SSE callback, connection will be closed")
@@ -266,3 +261,9 @@ def spot_allowed_by_query(spot, query):
                 if needs_good_location and not spot.dx_location_good:
                     return False
     return True
+
+
+def filter_fields(spots, fields):
+    """Given a list of spot objects, return copies containing only the named fields."""
+
+    return [{k: v for k, v in spot.__dict__.items() if k in fields} for spot in spots]

webserver/handlers/api/status.py

@@ -1,13 +1,10 @@
 import logging
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
 
-from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import safe_json_dumps
 
 logger = logging.getLogger(__name__)
@@ -23,21 +20,13 @@ class APIStatusHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._status_data = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, status_data, web_server_metrics):
+    def initialize(self, status_data):
         self._status_data = status_data
-        self._web_server_metrics = web_server_metrics
 
     def get(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             self.write(safe_json_dumps(self._status_data))
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")

webserver/handlers/api/v1_addspot.py

@@ -1,16 +1,13 @@
 import logging
 import re
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
 
 from core.config import ALLOW_SPOTTING
 from core.constants import UNKNOWN_BAND
-from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import infer_band_from_freq, safe_json_dumps
 from data.spot import Spot
@@ -28,21 +25,13 @@ class V1APISpotHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._spots = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, spots, web_server_metrics):
+    def initialize(self, spots):
         self._spots = spots
-        self._web_server_metrics = web_server_metrics
 
     def post(self):
         try:
-            # Metrics
-            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-            self._web_server_metrics["api_access_counter"] += 1
-            self._web_server_metrics["status"] = "OK"
-            api_requests_counter.inc()
-
             # Reject if not allowed
             if not ALLOW_SPOTTING:
                 self.set_status(401)

webserver/handlers/api/v1_compatability.py

@@ -34,7 +34,7 @@ class V1RedirectHandler(tornado.web.RequestHandler):
             response = await client.fetch(
                 new_url,
                 method=self.request.method,
-                headers=self.request.headers,
+                headers=headers,
                 body=None if self.request.method == "GET" else (self.request.body or b""),
                 raise_error=False,
                 follow_redirects=False,

webserver/handlers/api/v1_spots.py

@@ -1,6 +1,6 @@
 import re
 
-from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
+from webserver.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
 
 _GRID_SOURCE_RE = re.compile(r'"dx_location_source":\s*"GRID"')
 _LEGACY_PARAM_TO_HEADER_MAP = {

webserver/handlers/pagetemplate.py

@@ -1,14 +1,19 @@
-from datetime import datetime
 from typing import Any
 
-import pytz
 import tornado
 from tornado import httputil
 from tornado.web import Application
 
-from core.config import ALLOW_SPOTTING, BASE_URL, SERVER_OWNER_CALLSIGN, WEB_UI_OPTIONS
+from core.config import (
+    ALLOW_SPOTTING,
+    BASE_URL,
+    SERVER_OWNER_CALLSIGN,
+    TELNET_SERVER_ADDRESS,
+    TELNET_SERVER_ENABLED,
+    TELNET_SERVER_PORT,
+    WEB_UI_OPTIONS,
+)
 from core.constants import SOFTWARE_VERSION
-from core.prometheus_metrics_handler import page_requests_counter
 
 
 class PageTemplateHandler(tornado.web.RequestHandler):
@@ -21,20 +26,12 @@ class PageTemplateHandler(tornado.web.RequestHandler):
         **kwargs: Any,
     ):
         self._template_name = None
-        self._web_server_metrics = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, template_name, web_server_metrics):
+    def initialize(self, template_name):
         self._template_name = template_name
-        self._web_server_metrics = web_server_metrics
 
     def get(self):
-        # Metrics
-        self._web_server_metrics["last_page_access_time"] = datetime.now(pytz.UTC)
-        self._web_server_metrics["page_access_counter"] += 1
-        self._web_server_metrics["status"] = "OK"
-        page_requests_counter.inc()
-
         # Load named template, and provide variables used in templates
         self.render(
             f"{self._template_name}.html",
@@ -43,5 +40,8 @@ class PageTemplateHandler(tornado.web.RequestHandler):
             allow_spotting=ALLOW_SPOTTING,
             web_ui_options=WEB_UI_OPTIONS,
             baseurl=BASE_URL,
+            telnet_server_enabled=TELNET_SERVER_ENABLED,
+            telnet_server_address=TELNET_SERVER_ADDRESS,
+            telnet_server_port=TELNET_SERVER_PORT,
             current_path=self.request.path,
         )

webserver/sse_broadcaster.py

@@ -26,6 +26,11 @@ class SSEBroadcaster:
         with self._lock:
             self._handlers.discard(handler)
 
+    @property
+    def client_count(self) -> int:
+        with self._lock:
+            return len(self._handlers)
+
     def publish(self, value):
         self._loop.add_callback(self._broadcast, value)
 

webserver/webserver.py

@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import os
+import threading
 
 import tornado
 from tornado.web import StaticFileHandler
@@ -14,25 +15,26 @@ from core.config import (
 )
 from core.data_providers import DATA_PROVIDERS
 from core.data_store import DATA_STORE
-from server.handlers.api.addspot import APISpotHandler
+from webserver.handlers.api.addspot import APISpotHandler
-from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
+from webserver.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
-from server.handlers.api.dxstats import APIDxStatsHandler
+from webserver.handlers.api.dxstats import APIDxStatsHandler
-from server.handlers.api.lookups import (
+from webserver.handlers.api.lookups import (
     APILookupCallHandler,
     APILookupGridHandler,
     APILookupSIGRefHandler,
 )
-from server.handlers.api.options import APIOptionsHandler
+from webserver.handlers.api.options import APIOptionsHandler
-from server.handlers.api.solar_conditions import APISolarConditionsHandler
+from webserver.handlers.api.solar_conditions import APISolarConditionsHandler
-from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
+from webserver.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
-from server.handlers.api.status import APIStatusHandler
+from webserver.handlers.api.status import APIStatusHandler
-from server.handlers.api.v1_addspot import V1APISpotHandler
+from webserver.handlers.api.v1_addspot import V1APISpotHandler
-from server.handlers.api.v1_compatability import V1RedirectHandler
+from webserver.handlers.api.v1_compatability import V1RedirectHandler
-from server.handlers.api.v1_spots import V1APISpotsHandler, V1APISpotsStreamHandler
+from webserver.handlers.api.v1_spots import V1APISpotsHandler, V1APISpotsStreamHandler
-from server.handlers.manifesthandler import ManifestHandler
+from webserver.handlers.manifesthandler import ManifestHandler
-from server.handlers.metrics import PrometheusMetricsHandler
+from webserver.handlers.metrics import PrometheusMetricsHandler
-from server.handlers.pagetemplate import PageTemplateHandler
+from webserver.handlers.pagetemplate import PageTemplateHandler
-from server.sse_broadcaster import SSEBroadcaster
+from webserver.sse_broadcaster import SSEBroadcaster
+from webserver.webserver_metrics import WebServerMetrics
 
 logger = logging.getLogger(__name__)
 
@@ -52,92 +54,107 @@ class WebServer:
         self._port = WEB_SERVER_PORT
         self._api_only_mode = API_ONLY_MODE
         self._shutdown_event = asyncio.Event()
-        self.web_server_metrics = {
+        self._loop = None
-            "last_page_access_time": None,
+        self._thread = None
-            "last_api_access_time": None,
+        self.web_server_metrics = WebServerMetrics()
-            "page_access_counter": 0,
-            "api_access_counter": 0,
-            "status": "Starting",
-        }
 
     def setup(self):
         # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
         DATA_STORE.spots.add_listener(self._spot_broadcaster.publish)
         DATA_STORE.alerts.add_listener(self._alert_broadcaster.publish)
 
+    @property
+    def sse_client_count(self) -> int:
+        """Number of connected SSE clients, across both the spots and alerts streams."""
+
+        return self._spot_broadcaster.client_count + self._alert_broadcaster.client_count
+
     def start(self):
         """Start the web server"""
 
-        asyncio.run(self._start_inner())
+        self._thread = threading.Thread(target=asyncio.run, args=(self._start_inner(),), name="WebServer", daemon=True)
+        self._thread.start()
 
     def stop(self):
         """Stop the web server"""
 
-        self._shutdown_event.set()
+        if self._loop and self._loop.is_running():
+            self._loop.call_soon_threadsafe(self._shutdown_event.set)
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("Web server background thread did not exit on time and will be killed.")
+
+    def _handle_loop_exception(self, loop, context):
+        """Ignore "cancelled" exceptions from the asyncio loop to avoid printing exceptions to the log on shutdown when
+        we cancel the SSE handler threads"""
+
+        exception = context.get("exception")
+        if isinstance(exception, asyncio.CancelledError):
+            return
+        loop.default_exception_handler(context)
 
     async def _start_inner(self):
         """Start method (async). Sets up the Tornado application."""
 
+        self._loop = asyncio.get_running_loop()
+        self._loop.set_exception_handler(self._handle_loop_exception)
+
         # Bind the SSE broadcasters to the web server's loop, so they fire correctly
         self._spot_broadcaster.bind_to_web_server_loop()
         self._alert_broadcaster.bind_to_web_server_loop()
 
-        # Prepare a list of common arguments that are passed in to every API & page handler. This is just a basic thing
-        # to avoid copy-pasting the same thing to every route declaration below.
-        handler_opts = {"web_server_metrics": self.web_server_metrics}
-
         # API endpoints are always enabled
         api_routes = [
             (
                 r"/api/v2/spots",
                 APISpotsHandler,
-                {"spots": self._data_store.spots, **handler_opts},
+                {"spots": self._data_store.spots},
             ),
             (
                 r"/api/v2/alerts",
                 APIAlertsHandler,
-                {"alerts": self._data_store.alerts, **handler_opts},
+                {"alerts": self._data_store.alerts},
             ),
             (
                 r"/api/v2/spots/stream",
                 APISpotsStreamHandler,
-                {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts},
+                {"sse_spot_broadcaster": self._spot_broadcaster},
             ),
             (
                 r"/api/v2/alerts/stream",
                 APIAlertsStreamHandler,
-                {"sse_alert_broadcaster": self._alert_broadcaster, **handler_opts},
+                {"sse_alert_broadcaster": self._alert_broadcaster},
             ),
             (
                 r"/api/v2/solar",
                 APISolarConditionsHandler,
-                {"solar_conditions": self._data_store.solar_conditions.get(), **handler_opts},
+                {"solar_conditions": self._data_store.solar_conditions.get()},
             ),
             (
                 r"/api/v2/dxstats",
                 APIDxStatsHandler,
-                {"spots": self._data_store.spots, **handler_opts},
+                {"spots": self._data_store.spots},
             ),
             (
                 r"/api/v2/options",
                 APIOptionsHandler,
-                {"status_data": self._data_store.status.get(), **handler_opts},
+                {"status_data": self._data_store.status.get()},
             ),
             (
                 r"/api/v2/status",
                 APIStatusHandler,
-                {"status_data": self._data_store.status.get(), **handler_opts},
+                {"status_data": self._data_store.status.get()},
             ),
-            (r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}),
+            (r"/api/v2/lookup/call", APILookupCallHandler),
-            (r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
+            (r"/api/v2/lookup/sigref", APILookupSIGRefHandler),
-            (r"/api/v2/lookup/grid", APILookupGridHandler, {**handler_opts}),
+            (r"/api/v2/lookup/grid", APILookupGridHandler),
             (
                 r"/api/v2/spot",
                 APISpotHandler,
                 {
                     "spots": self._data_store.spots,
                     "spot_providers": self._data_providers,
-                    **handler_opts,
                 },
             ),
         ]
@@ -148,19 +165,18 @@ class WebServer:
             (
                 r"/api/v1/spots",
                 V1APISpotsHandler,
-                {"spots": self._data_store.spots, **handler_opts},
+                {"spots": self._data_store.spots},
             ),
             (
                 r"/api/v1/spots/stream",
                 V1APISpotsStreamHandler,
-                {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts},
+                {"sse_spot_broadcaster": self._spot_broadcaster},
             ),
             (
                 r"/api/v1/spot",
                 V1APISpotHandler,
                 {
                     "spots": self._data_store.spots,
-                    **handler_opts,
                 },
             ),
             (r"/api/v1/(.*)", V1RedirectHandler),
@@ -173,41 +189,41 @@ class WebServer:
                 (
                     r"/",
                     PageTemplateHandler,
-                    {"template_name": "api_only_home", **handler_opts},
+                    {"template_name": "api_only_home"},
                 )
             ]
         else:
             ui_routes = [
-                (r"/", PageTemplateHandler, {"template_name": "spots", **handler_opts}),
+                (r"/", PageTemplateHandler, {"template_name": "spots"}),
                 (
                     r"/map",
                     PageTemplateHandler,
-                    {"template_name": "map", **handler_opts},
+                    {"template_name": "map"},
                 ),
                 (
                     r"/bands",
                     PageTemplateHandler,
-                    {"template_name": "bands", **handler_opts},
+                    {"template_name": "bands"},
                 ),
                 (
                     r"/alerts",
                     PageTemplateHandler,
-                    {"template_name": "alerts", **handler_opts},
+                    {"template_name": "alerts"},
                 ),
                 (
                     r"/conditions",
                     PageTemplateHandler,
-                    {"template_name": "conditions", **handler_opts},
+                    {"template_name": "conditions"},
                 ),
                 (
                     r"/status",
                     PageTemplateHandler,
-                    {"template_name": "status", **handler_opts},
+                    {"template_name": "status"},
                 ),
                 (
                     r"/about",
                     PageTemplateHandler,
-                    {"template_name": "about", **handler_opts},
+                    {"template_name": "about"},
                 ),
             ]
             # Only allow the Add Spot page if spotting is allowed
@@ -216,7 +232,7 @@ class WebServer:
                     (
                         r"/add-spot",
                         PageTemplateHandler,
-                        {"template_name": "add_spot", **handler_opts},
+                        {"template_name": "add_spot"},
                     )
                 ]
 
@@ -226,7 +242,7 @@ class WebServer:
             (
                 r"/apidocs",
                 PageTemplateHandler,
-                {"template_name": "apidocs", **handler_opts},
+                {"template_name": "apidocs"},
             ),
             (r"/metrics", PrometheusMetricsHandler),
             (r"/manifest.webmanifest", ManifestHandler),
@@ -254,7 +270,11 @@ class WebServer:
 
 def request_log(handler):
     """Custom log function to provide more data about requests when enabled, and to provide the ability to turn off
-    web request logging altogetether."""
+    web request logging altogetether. Also records the time of the request and status in the webserver metrics. Probably
+    not what this method is supposed to be used for but it's a convenient thing that gets called on every request, so
+    saves having to pass the metrics around each handler individually."""
+
+    WEB_SERVER.web_server_metrics.record(handler.request.path, handler.get_status())
 
     if LOG_WEB_REQUESTS:
         if handler.get_status() < 500:

webserver/webserver_metrics.py

@@ -0,0 +1,44 @@
+from collections import deque
+from datetime import datetime, timedelta
+
+import pytz
+
+from core.prometheus_metrics_handler import api_requests_counter, page_requests_counter
+
+
+class WebServerMetrics:
+    """Tracker for web server metrics. Stores the times pages and API endpoints were accessed for an hour, so we
+    can display the rate of requests per hour, and also updates the equivalent Prometheus counters."""
+
+    def __init__(self):
+        self.status = "Starting"
+        self._page_access_times = deque()
+        self._api_access_times = deque()
+
+    def record(self, path: str, status_code: int):
+        """Records data for a request, depending on whether it's a page, API, or other request, and making sure
+        the response code isn't 404. Also sets the status of the web server."""
+
+        if status_code == 404 or path.startswith(("/static/", "/metrics", "/manifest.webmanifest")):
+            return
+
+        self.status = "OK" if status_code < 500 else "Error"
+        if path.startswith("/api/"):
+            api_requests_counter.inc()
+            self._api_access_times.append(datetime.now(pytz.UTC))
+        else:
+            page_requests_counter.inc()
+            self._page_access_times.append(datetime.now(pytz.UTC))
+
+    def page_requests_per_hour(self) -> int:
+        return self._count_and_prune_last_hour(self._page_access_times)
+
+    def api_requests_per_hour(self) -> int:
+        return self._count_and_prune_last_hour(self._api_access_times)
+
+    @staticmethod
+    def _count_and_prune_last_hour(access_times: deque) -> int:
+        cutoff = datetime.now(pytz.UTC) - timedelta(hours=1)
+        while access_times and access_times[0] < cutoff:
+            access_times.popleft()
+        return len(access_times)