v2.1 release

.idea/inspectionProfiles/Project_Default.xml

@@ -65,6 +65,13 @@
         </list>
       </option>
     </inspection_tool>
+    <inspection_tool class="PyStubPackagesAdvertiser" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <list>
+          <option value="pandas" />
+        </list>
+      </option>
+    </inspection_tool>
     <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
       <option name="processCode" value="true" />
       <option name="processLiterals" value="true" />

Dockerfile

@@ -4,5 +4,6 @@ WORKDIR /app
 COPY . .
 RUN pip install --no-cache-dir -r requirements.txt
 EXPOSE 8080
+EXPOSE 7373
 
 CMD ["python3", "spothole.py"]

README.md

@@ -1,7 +1,7 @@
 # ![Spothole](/static/img/logo.png)
 
 Spothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open
-JSON API as well as a website to browse the data.
+JSON API as well as a website to browse the data, and its own telnet server for integration with desktop loggers.
 
 ![Screenshot](/images/screenshot.png)
 
@@ -17,10 +17,8 @@ Spothole itself is also open source, Public Domain licenced code that anyone can
 
 Supported data sources include DX Clusters, the Reverse Beacon Network (RBN), the APRS Internet Service (APRS-IS), POTA,
 SOTA, WWFF, GMA, WWBOTA, HEMA, Parks 'n' Peaks, ZLOTA, WOTA, BOTA, LLOTA, WWTOTA, Tiles on the Air, the UK Packet
-Repeater Network, NG3K, and any site based on the xOTA software by nischu.
+Repeater Network, NG3K, and any site based on the xOTA software by nischu. It also integrates with QRZ.com and HamQTH,
-
+retrieves solar data from various sources, provides information about upcoming contests, and more.
-Additional Special Interest Groups (SIGs) without their own specific data source include KRMNPA, SANPCPA, WAB, WAI and
-DME.
 
 ![Screenshot](/images/screenshot2.png)
 

config-example.yml

@@ -17,6 +17,15 @@ api_only_mode: false
 # The base URL at which the software runs.
 base_url: "http://localhost:8080"
 
+# Whether to run a telnet spot server as well as the web interface
+telnet_server_enabled: false
+
+# Telnet server address. This is not really needed by the software itself, just displayed in documentation.
+telnet_server_address: "localhost"
+
+# Port to run the telnet server on
+telnet_server_port: 7373
+
 # Spot providers to use. This is an example set, tailor it to your liking by commenting and uncommenting.
 # RBN and APRS-IS are supported but have such a high data rate, you probably don't want them enabled.
 # Each provider needs a class and an enabled/disabled state. Some require more config such as hostnames/IP
@@ -162,9 +171,21 @@ alert_providers:
   - class: "BOTA"
     enabled: true
 
+  - class: "Hamsat"
+    enabled: true
+
   - class: "NG3K"
     enabled: true
 
+  - class: "WA7BNM"
+    enabled: true
+
+  - class: "RSGBHFContests"
+    enabled: true
+
+  - class: "RSGBVHFContests"
+    enabled: true
+
 
 # Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band
 # conditions, etc.) and make it available via the /api/v2/solar endpoint.
@@ -246,12 +267,36 @@ sig_ref_data_providers:
   - class: "DME"
     enabled: true
 
+  - class: "FEA"
+    enabled: true
+
+  - class: "DMVE"
+    enabled: true
+
+  - class: "DMUE"
+    enabled: true
+
+  - class: "DCE"
+    enabled: true
+
+  - class: "DEFE"
+    enabled: true
+
   - class: "KRMNPA"
     enabled: true
 
   - class: "SANPCPA"
     enabled: true
 
+  - class: "DTMBA"
+    enabled: true
+
+  - class: "COTA"
+    enabled: true
+
+  - class: "PGA"
+    enabled: true
+
   - class: "Toilets"
     enabled: true
 
@@ -275,27 +320,27 @@ callsign_data_providers:
     priority: 2
     # No server-side credentials for HamQTH. Users must provide their own.
 
+  - class: "CountryFiles"
+    priority: 3
+    enabled: true
+
   - class: "ClublogAPI"
     # Querying the Clublog API directly doesn't provide any more data than the XML version, it just provides slightly
     # more up-to-date information in the rare case that the prefix data changes, at a significant cost of looking up
     # every callsign via an API call. Normally left disabled but it exists as an option.
     enabled: false
-    priority: 3
+    priority: 4
     # API key for Clublog to look up information. Required in order to enable this provider. Unlike QRZ and HamQTH,
     # Clublog uses an API key issued to Spothole, not to the end user.
     api_key: ""
 
   - class: "ClublogXML"
     enabled: true
-    priority: 4
+    priority: 5
     # API key for Clublog to look up information. Required in order to enable this provider. You will need to request
     # one via their helpdesk portal if you want to use callsign lookups from Clublog.
     api_key: ""
 
-  - class: "CountryFiles"
-    priority: 5
-    enabled: true
-
 
 # Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week
 # for alerts.

core/cleanup.py

@@ -34,6 +34,10 @@ class CleanupTimer:
         """Stop any threads and prepare for application shutdown"""
 
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("Cleanup worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while not self._stop_event.wait(timeout=self._cleanup_interval):

core/config.py

@@ -24,6 +24,9 @@ MAX_SPOT_AGE = config.get("max_spot_age_sec", 3600)
 MAX_ALERT_AGE = config.get("max_alert_age_sec", 604800)
 SERVER_OWNER_CALLSIGN = config.get("server_owner_callsign", "N0CALL")
 WEB_SERVER_PORT = config.get("web_server_port", 8080)
+TELNET_SERVER_ENABLED = config.get("telnet_server_enabled", False)
+TELNET_SERVER_ADDRESS = config.get("telnet_server_address", "localhost")
+TELNET_SERVER_PORT = config.get("telnet_server_port", 7373)
 ALLOW_SPOTTING = config.get("allow_spotting", True)
 ALLOW_UPSTREAM_SPOTTING = config.get("allow_upstream_spotting", True)
 WEB_UI_OPTIONS = config.get("web_ui_options", {})

core/constants.py

@@ -1,9 +1,10 @@
 from core.config import SERVER_OWNER_CALLSIGN
+from core.enums import SIGRefType, SIGType
 from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "2.0.5"
+SOFTWARE_VERSION = "2.1"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"}
@@ -11,203 +12,392 @@ HAMQTH_PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}"
 
 # Special Interest Groups
 SIGS = [
+    SIG(
+        name="AMSAT",
+        comment_names=[],
+        description="Amateur Radio Satellites",
+        sig_type=SIGType.WORLDWIDE,
+        icon="fa-satellite",
+        refs_globally_unique=False,
+    ),
+    SIG(
+        name="EME",
+        comment_names=[],
+        description="Moonbounce",
+        sig_type=SIGType.WORLDWIDE,
+        icon="fa-moon",
+        refs_globally_unique=False,
+    ),
     SIG(
         name="POTA",
         comment_names=["POTA"],
         description="Parks on the Air",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.PARK,
         ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST",
+        icon="fa-tree",
+        refs_globally_unique=False,
     ),
     SIG(
         name="SOTA",
         comment_names=["SOTA"],
         description="Summits on the Air",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.SUMMIT,
         ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
+        icon="fa-mountain-sun",
+        refs_globally_unique=False,
     ),
     SIG(
         name="WWFF",
         comment_names=["WWFF"],
         description="World Wide Flora & Fauna",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.PARK,
         ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}",
+        icon="fa-seedling",
+        refs_globally_unique=True,
     ),
     SIG(
         name="GMA",
         comment_names=["GMA"],
         description="Global Mountain Activity",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.SUMMIT,
         ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
+        icon="fa-person-hiking",
+        refs_globally_unique=False,
     ),
     SIG(
         name="WWBOTA",
         comment_names=["WWBOTA", "BOTA"],
         description="Worldwide Bunkers on the Air",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.BUNKER,
         ref_regex=r"B\/[A-Z0-9]{1,3}\-\d{3,4}",
+        icon="fa-radiation",
+        refs_globally_unique=True,
     ),
     SIG(
         name="HEMA",
         comment_names=["HEMA"],
         description="HuMPs Excluding Marilyns Award",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.SUMMIT,
         ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}",
+        icon="fa-mound",
+        refs_globally_unique=False,
     ),
     SIG(
         name="IOTA",
         comment_names=["IOTA"],
         description="Islands on the Air",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.ISLAND,
         ref_regex=r"[A-Z]{2}\-\d{3}",
+        icon="fa-book-atlas",
+        refs_globally_unique=False,
     ),
     SIG(
         name="GMA Islands",
         comment_names=[],
         description="Global Mountain Activity - Islands",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.ISLAND,
         ref_regex=r"(([A-Z]{2}\-\d{3})|([A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}))",
-    ),
+        icon="fa-person-hiking",
-    SIG(
+        refs_globally_unique=False,
-        name="MOTA",
-        comment_names=["MOTA"],
-        description="Mills on the Air",
-        ref_regex=r"X\d{4,6}",
     ),
     SIG(
         name="ARLHS",
         comment_names=["ARLHS"],
         description="Amateur Radio Lighthouse Society",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.LIGHTHOUSE,
         ref_regex=r"[A-Z]{3}[\- ]\d{3,4}",
+        icon="fa-house-flood-water",
+        refs_globally_unique=False,
     ),
     SIG(
         name="ILLW",
         comment_names=["ILLW"],
         description="International Lighthouse & Lightship Weekend",
+        sig_type=SIGType.EVENT,
+        ref_type=SIGRefType.LIGHTHOUSE,
         ref_regex=r"[A-Z]{2}\d{4}",
+        icon="fa-house-flood-water",
+        refs_globally_unique=False,
+    ),
+    SIG(
+        name="MOTA",
+        comment_names=["MOTA"],
+        description="Mills on the Air",
+        sig_type=SIGType.EVENT,
+        ref_type=SIGRefType.MILL,
+        ref_regex=r"X\d{4,6}",
+        icon="fa-fan",
+        refs_globally_unique=True,
     ),
     SIG(
         name="SIOTA",
         comment_names=["SIOTA"],
         description="Silos on the Air",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.SILO,
         ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d",
+        icon="fa-wheat-awn",
+        refs_globally_unique=False,
     ),
     SIG(
         name="WCA",
         comment_names=["WCA"],
         description="World Castles Award",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.CASTLE,
         ref_regex=r"[A-Z0-9]{1,3}\-\d{5}",
+        icon="fa-chess-rook",
+        refs_globally_unique=False,
     ),
     SIG(
         name="ZLOTA",
         comment_names=["ZLOTA"],
         description="New Zealand on the Air",
+        sig_type=SIGType.REGIONAL,
+        ref_type=None,
         ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}",
+        icon="fa-kiwi-bird",
+        region_flag="🇳🇿",
+        refs_globally_unique=True,
     ),
     SIG(
         name="WOTA",
         comment_names=["WOTA"],
         description="Wainwrights on the Air",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.SUMMIT,
         ref_regex=r"[A-Z]{3}-[0-9]{2}",
+        icon="fa-w",
+        region_flag="🇬🇧",
+        refs_globally_unique=False,
+    ),
+    SIG(
+        name="BOTA",
+        comment_names=[],
+        description="Beaches on the Air",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.BEACH,
+        icon="fa-umbrella-beach",
+        refs_globally_unique=False,
     ),
-    SIG(name="BOTA", comment_names=[], description="Beaches on the Air"),
     SIG(
         name="KRMNPA",
         comment_names=["KRMNPA"],
         description="Keith Roget Memorial National Parks Award",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.PARK,
         ref_regex=r"VKFF\-\d{4}",
+        icon="fa-earth-oceania",
+        region_flag="🇦🇺",
+        refs_globally_unique=False,
     ),
     SIG(
         name="SANPCPA",
         comment_names=["SANPCPA"],
         description="South Australian National Parks and Conservation Parks Award",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.PARK,
         ref_regex=r"VKFF\-\d{4}",
+        icon="fa-earth-oceania",
+        region_flag="🇦🇺",
+        refs_globally_unique=False,
     ),
     SIG(
         name="LLOTA",
         comment_names=["LLOTA"],
         description="Lagos y Lagunas on the Air",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.LAKE,
         ref_regex=r"LL[A-Z]{2}\-\d{4}",
+        icon="fa-water",
+        refs_globally_unique=True,
     ),
     SIG(
         name="Towers",
         comment_names=["TOTA"],
         description="Towers on the Air",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.TOWER,
         ref_regex=r"[A-Z]{2,3}R\-\d{4}",
+        icon="fa-tower-observation",
+        refs_globally_unique=False,
     ),
     SIG(
         name="Tiles",
         comment_names=[],
         description="Tiles on the Air",
+        sig_type=SIGType.WORLDWIDE,
+        ref_type=SIGRefType.GRID,
         ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}",
+        icon="fa-square",
+        refs_globally_unique=False,
     ),
     SIG(
         name="WAB",
         comment_names=["WAB"],
         description="Worked All Britain",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.GRID,
         ref_regex=r"[A-Z]{1,2}[0-9]{2}",
+        icon="fa-table-cells-large",
+        region_flag="🇬🇧",
+        refs_globally_unique=False,
     ),
     SIG(
         name="WAI",
         comment_names=["WAI"],
         description="Worked All Ireland",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.GRID,
         ref_regex=r"[A-Z][0-9]{2}",
+        icon="fa-table-cells-large",
+        region_flag="🇮🇪",
+        refs_globally_unique=False,
+    ),
+    SIG(
+        name="DMF",
+        comment_names=["DMF"],
+        description="Diplôme des Moulins de France",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.MILL,
+        icon="fa-fan",
+        region_flag="🇫🇷",
+        refs_globally_unique=False,
     ),
     SIG(
         name="DME",
         comment_names=["DME"],
-        description="Diplomas de Municipios Españoles",
+        description="Diploma Municipios de España",
-        ref_regex=r"\d{4,5}",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.TOWN,
+        ref_regex=r"DME[\- ]\d{3,5}",
+        icon="fa-building",
+        region_flag="🇪🇸",
+        refs_globally_unique=True,
+    ),
+    SIG(
+        name="FEA",
+        comment_names=["FEA"],
+        description="Diploma Faros de España",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.LIGHTHOUSE,
+        # FEA references are technically [DE]\-\d{4}(\.\d)? but spotters always seem to miss out the D- or E-
+        # prefix and just use FEA-1234 or FEA 1234, so allow for that. The FEA sigref data provider adds both
+        # forms to the database.
+        ref_regex=r"([DE]|FEA)[\- ]\d{4}(\.\d)?",
+        icon="fa-house-flood-water",
+        region_flag="🇪🇸",
+        refs_globally_unique=True,
+    ),
+    SIG(
+        name="DMUE",
+        comment_names=["DMUE"],
+        description="Diploma Museos de España",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.BUILDING,
+        ref_regex=r"MUE[A-Z]{2}-\d{3}",
+        icon="fa-landmark",
+        region_flag="🇪🇸",
+        refs_globally_unique=True,
+    ),
+    SIG(
+        name="DMVE",
+        comment_names=["DMVE"],
+        description="Diploma Monumentos y Vestigios de España",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.BUILDING,
+        ref_regex=r"MV[A-Z]{1,2}-\d{4}",
+        icon="fa-monument",
+        region_flag="🇪🇸",
+        refs_globally_unique=True,
+    ),
+    SIG(
+        name="DCE",
+        comment_names=["DCE"],
+        description="Diploma Castillos de España",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.CASTLE,
+        ref_regex=r"C[A-Z]{1,2}-\d{3}",
+        icon="fa-chess-rook",
+        region_flag="🇪🇸",
+        refs_globally_unique=False,
+    ),
+    SIG(
+        name="DEFE",
+        comment_names=["DEFE"],
+        description="Diploma Estaciones de Ferrocarril de España",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.BUILDING,
+        ref_regex=r"EF[A-Z]{1,2}-\d{3}",
+        icon="fa-train",
+        region_flag="🇪🇸",
+        refs_globally_unique=True,
+    ),
+    SIG(
+        name="DTMBA",
+        comment_names=["DTMBA"],
+        description="Diploma Teatri Musei e Belle Arti",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.BUILDING,
+        ref_regex=r"I-?[0-9]{3,4}\s?[A-Z]{2}",
+        icon="fa-landmark",
+        region_flag="🇮🇹",
+        refs_globally_unique=True,
+    ),
+    SIG(
+        name="BIWOTA",
+        comment_names=["BIWOTA"],
+        description="British Inland Waterways on the Air",
+        sig_type=SIGType.EVENT,
+        ref_type=SIGRefType.WATERWAY,
+        icon="fa-ship",
+        region_flag="🇬🇧",
+        refs_globally_unique=False,
+    ),
+    SIG(
+        name="COTA",
+        comment_names=["COTA"],
+        description="Castles on the Air",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.CASTLE,
+        ref_regex=r"[A-Z]{3}\-[0-9]{3,5}",
+        icon="fa-chess-rook",
+        region_flag="🇩🇪",
+        refs_globally_unique=False,
+    ),
+    SIG(
+        name="PGA",
+        comment_names=["PGA"],
+        description="Polish Gmina Award",
+        sig_type=SIGType.REGIONAL,
+        ref_type=SIGRefType.REGION,
+        ref_regex=r"[A-Z]{2}[0-9]{2}",
+        icon="fa-g",
+        region_flag="🇵🇱",
+        refs_globally_unique=False,
     ),
     SIG(
         name="Toilets",
         comment_names=[],
         description="Toilets on the Air",
+        sig_type=SIGType.EVENT,
+        ref_type=SIGRefType.TOILET,
         ref_regex=r"T\-[0-9]{2}",
+        icon="fa-toilet",
+        region_flag="🏴‍☠️",
+        refs_globally_unique=True,
     ),
 ]
 
-# Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
-CW_MODES = ["CW"]
-PHONE_MODES = [
-    "PHONE",
-    "SSB",
-    "USB",
-    "LSB",
-    "AM",
-    "FM",
-    "DV",
-    "DMR",
-    "DSTAR",
-    "C4FM",
-    "FUSION",
-    "M17",
-]
-DATA_MODES = [
-    "DATA",
-    "FT8",
-    "FT4",
-    "RTTY",
-    "SSTV",
-    "JS8",
-    "HELL",
-    "PSK",
-    "FSK",
-    "OLIVIA",
-    "PKT",
-    "MSK144",
-]
-ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
-MODE_TYPES = ["CW", "PHONE", "DATA"]
-SSB_SUB_MODES = ["USB", "LSB"]
-DV_SUB_MODES = ["DMR", "DSTAR", "C4FM", "FUSION", "M17"]
-
-# Mode aliases. Sometimes we get spots with a mode described in a different way that is effectively the same as a mode
-# we already know, or we want to normalise things for consistency. The lookup table for this is here. Incoming spots
-# that match a key in this table will be converted to the corresponding value, so only the modes above will actually be
-# present in the spots.
-MODE_ALIASES = {
-    "RTT": "RTTY",
-    "BPSK": "PSK",
-    "PSK31": "PSK",
-    "BPSK31": "PSK",
-    "MFSK": "FSK",
-    "MFSK32": "FSK",
-    "DIGI": "DATA",
-    "DIGITAL": "DATA",
-}
-
 # Band definitions
 BANDS = [
     Band(name="2200m", start_freq=135700, end_freq=137800),
@@ -223,7 +413,7 @@ BANDS = [
     Band(name="12m", start_freq=24890000, end_freq=24990000, is_ham_hf=True),
     Band(name="11m", start_freq=26965000, end_freq=27405000),
     Band(name="10m", start_freq=28000000, end_freq=29700000, is_ham_hf=True),
-    Band(name="6m", start_freq=50000000, end_freq=54000000),
+    Band(name="6m", start_freq=50000000, end_freq=54000000, is_ham_hf=True),
     Band(name="5m", start_freq=56000000, end_freq=60500000),
     Band(name="4m", start_freq=70000000, end_freq=70500000),
     Band(name="2m", start_freq=144000000, end_freq=148000000),
@@ -239,9 +429,6 @@ BANDS = [
 ]
 UNKNOWN_BAND = Band(name="Unknown", start_freq=0, end_freq=0)
 
-# Continents
-CONTINENTS = ["EU", "NA", "SA", "AS", "AF", "OC", "AN"]
-
 # Propagation modes used in VHF/UHF DX cluster comments, e.g. "JN61ES<ES>JM56XT". I don't think there's an official list
 # of these anywhere, but here are some I've seen or seen reference to
 PROPAGATION_MODES = {
@@ -254,5 +441,6 @@ PROPAGATION_MODES = {
     "MS": "Meteor scatter",
     "RS": "Rain scatter",
     "AS": "Aircraft scatter",
+    "ACS": "Aircraft scatter",
     "SAT": "Satellite",
 }

core/data_providers.py

@@ -1,5 +1,6 @@
 import logging
 import threading
+import time
 
 from core.config import config, create_provider_from_config
 
@@ -16,6 +17,7 @@ class DataProviders:
         self.static_data_providers = []
         self.sig_ref_data_providers = []
         self.callsign_data_providers = []
+        self._startup_timers = []
 
     def setup(self):
         for entry in config["spot_providers"]:
@@ -43,41 +45,58 @@ class DataProviders:
     def start(self):
         # Start data providers before spot/alert providers so the lookup data is there already for incoming spots.
         # Each category is fired off after a small delay to give the rest of Spothole chance to start up.
-        threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")).start()
+        self._startup_timers = [
-        threading.Timer(
+            threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")),
-            10.0,
+            threading.Timer(10.0, lambda: self.start_providers(self.callsign_data_providers, "callsign data")),
-            lambda: self.start_providers(self.callsign_data_providers, "callsign data"),
+            threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")),
-        ).start()
+            threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")),
-        threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")).start()
-        threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")).start()
             threading.Timer(
                 25.0,
                 lambda: self.start_providers(self.solar_condition_providers, "solar condition"),
-        ).start()
+            ),
-        threading.Timer(
+            threading.Timer(30.0, lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data")),
-            30.0,
+        ]
-            lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data"),
+        for t in self._startup_timers:
-        ).start()
+            t.daemon = True
+            t.start()
 
     def stop(self):
-        for sp in self.spot_providers:
+        # Cancel any startup timers that haven't fired yet
-            if sp.enabled:
+        for t in self._startup_timers:
-                sp.stop()
+            t.cancel()
-        for ap in self.alert_providers:
+
-            if ap.enabled:
+        # Stop all providers
-                ap.stop()
+        all_providers = [
-        for scp in self.solar_condition_providers:
+            p
-            if scp.enabled:
+            for p in (
-                scp.stop()
+                self.spot_providers
-        for srdp in self.sig_ref_data_providers:
+                + self.alert_providers
-            if srdp.enabled:
+                + self.solar_condition_providers
-                srdp.stop()
+                + self.sig_ref_data_providers
-        for sdp in self.static_data_providers:
+                + self.static_data_providers
-            if sdp.enabled:
+                + self.callsign_data_providers
-                sdp.stop()
+            )
-        for cdp in self.callsign_data_providers:
+            if p.enabled
-            if cdp.enabled:
+        ]
-                cdp.stop()
+        if not all_providers:
+            return
+
+        def stop_provider(p):
+            try:
+                p.stop()
+            except Exception:
+                logger.exception("Exception stopping provider")
+
+        threads = [threading.Thread(target=stop_provider, args=(p,), daemon=True) for p in all_providers]
+        for t in threads:
+            t.start()
+
+        deadline = time.monotonic() + 40
+        for t in threads:
+            t.join(timeout=max(0.0, deadline - time.monotonic()))
+        still_running = [t for t in threads if t.is_alive()]
+        if still_running:
+            logger.warning("Some threads did not stop in time!")
 
 
 # Global object

core/data_store.py

@@ -106,13 +106,14 @@ class DataStore:
         and testing the callsign every time is expensive. So instead we build a separate in-memory lookup of compiled
         regex against DXCC entity code, as a list of tuples we can iterate through."""
 
+        self.dxcc_lookup_by_call_regex = []
         for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]:
             self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
 
     def close(self):
         self.spots.close()
         self.alerts.close()
-        self.solar.close()
+        self.solar_conditions.close()
         self.status.close()
         self.dxcc_data.close()
         self.sigrefs.close()

core/enums.py

@@ -0,0 +1,154 @@
+from enum import Enum
+
+
+class Continent(str, Enum):
+    EU = "EU"
+    NA = "NA"
+    SA = "SA"
+    AS = "AS"
+    AF = "AF"
+    OC = "OC"
+    AN = "AN"
+
+
+class Mode(str, Enum):
+    CW = "CW"
+    PHONE = "PHONE"
+    SSB = "SSB"
+    AM = "AM"
+    FM = "FM"
+    DV = "DV"
+    DATA = "DATA"
+    FT8 = "FT8"
+    FT4 = "FT4"
+    RTTY = "RTTY"
+    SSTV = "SSTV"
+    JS8 = "JS8"
+    HELL = "HELL"
+    PSK = "PSK"
+    FSK = "FSK"
+    OLIVIA = "OLIVIA"
+    PKT = "PKT"
+    MSK144 = "MSK144"
+
+    @property
+    def is_cw(self) -> bool:
+        return self == Mode.CW
+
+    @property
+    def is_phone(self) -> bool:
+        return self in {Mode.PHONE, Mode.SSB, Mode.AM, Mode.FM, Mode.DV}
+
+    @property
+    def is_data(self) -> bool:
+        return not (self.is_cw or self.is_phone)
+
+    @staticmethod
+    def from_name(name):
+        """Convert a string to an enum mode using the alias table."""
+
+        if not name:
+            return None
+
+        try:
+            return Mode(name.upper())
+        except (KeyError, ValueError):
+            try:
+                return Mode(MODE_ALIASES[name.upper()])
+            except (KeyError, ValueError):
+                return None
+
+
+class ModeType(str, Enum):
+    PHONE = "PHONE"
+    CW = "CW"
+    DATA = "DATA"
+
+
+class ModeSource(str, Enum):
+    """Where the mode data came from in a spot."""
+
+    SPOT = "SPOT"
+    COMMENT = "COMMENT"
+    BANDPLAN = "BANDPLAN"
+
+
+class LocationSourceForSpot(str, Enum):
+    """Where the location data came from in a spot."""
+
+    SPOT = "SPOT"
+    SIG_REF_LOOKUP = "SIG REF LOOKUP"
+    GRID = "GRID"
+    HOME_QTH = "HOME QTH"
+    DXCC = "DXCC"
+
+
+class LocationSourceForCallsign(str, Enum):
+    """Where the location data came from in a callsign data object."""
+
+    HOME_QTH = "HOME QTH"
+    DXCC = "DXCC"
+
+
+class SIGRefType(str, Enum):
+    """Type of a Special Interest Group reference."""
+
+    PARK = "PARK"
+    SUMMIT = "SUMMIT"
+    CASTLE = "CASTLE"
+    LIGHTHOUSE = "LIGHTHOUSE"
+    ISLAND = "ISLAND"
+    BUNKER = "BUNKER"
+    MILL = "MILL"
+    SILO = "SILO"
+    BEACH = "BEACH"
+    LAKE = "LAKE"
+    TOWER = "TOWER"
+    WATERWAY = "WATERWAY"
+    TOWN = "TOWN"
+    BUILDING = "BUILDING"
+    REGION = "REGION"
+    GRID = "GRID"
+    TOILET = "TOILET"
+
+
+class AlertType(str, Enum):
+    """Type of an alert."""
+
+    XOTA = "XOTA"
+    SATELLITE = "SATELLITE"
+    DXPEDITION = "DXPEDITION"
+    CONTEST = "CONTEST"
+
+
+class SIGType(str, Enum):
+    """Type of a Special Interest Group. Used to group them in the web UI."""
+
+    WORLDWIDE = "WORLDWIDE"
+    REGIONAL = "REGIONAL"
+    EVENT = "EVENT"
+
+
+# Mode aliases. Sometimes we get spots with a mode described in a different way that is effectively the same as a mode
+# we already know, or we want to normalise things for consistency. The lookup table for this is here. Incoming spots
+# that match a key in this table will be converted to the corresponding value, so only the modes above will actually be
+# present in the spots.
+MODE_ALIASES = {
+    "USB": "SSB",
+    "LSB": "SSB",
+    "DIGITALVOICE": "DV",
+    "DIGITALVOI": "DV",
+    "DMR": "DV",
+    "DSTAR": "DV",
+    "C4FM": "DV",
+    "FUSION": "DV",
+    "M17": "DV",
+    "RTT": "RTTY",
+    "BPSK": "PSK",
+    "PSK31": "PSK",
+    "BPSK31": "PSK",
+    "MFSK": "FSK",
+    "MFSK32": "FSK",
+    "DIGI": "DATA",
+    "DIGITAL": "DATA",
+}

core/live_data_cache.py

@@ -22,6 +22,8 @@ class LiveDataCache:
         self._listeners_lock = threading.Lock()
         self._snapshot_dir = snapshot_dir
         self._disk_cache = diskcache.Cache(str(snapshot_dir))
+        self._stop_event = threading.Event()
+        self._snapshot_thread = None
         self._load_snapshot()
         self._start_periodic_snapshot(snapshot_interval_sec)
 
@@ -89,13 +91,19 @@ class LiveDataCache:
 
     def _start_periodic_snapshot(self, interval):
         def loop():
-            while True:
+            while not self._stop_event.wait(timeout=interval):
-                time.sleep(interval)
                 self.save_snapshot()
 
-        t = threading.Thread(target=loop, name=f"LiveDataCache-Snapshot-{self._snapshot_dir}")
+        self._snapshot_thread = threading.Thread(
-        t.start()
+            target=loop, name=f"LiveDataCache-Snapshot-{self._snapshot_dir}", daemon=True
+        )
+        self._snapshot_thread.start()
 
     def close(self):
+        self._stop_event.set()
+        if self._snapshot_thread:
+            self._snapshot_thread.join(timeout=15)
+            if self._snapshot_thread.is_alive():
+                logger.warning(f"LiveDataCache snapshot thread for {self._snapshot_dir} did not exit on time.")
         self.save_snapshot()
         self._disk_cache.close()

core/sig_lookup_helper.py

@@ -1,46 +1,70 @@
 import logging
+import re
 
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 
+from core.constants import SIGS
 from core.data_store import DATA_STORE
+from core.enums import SIGRefType
 from core.geo_utils import wab_wai_square_to_lat_lon
 from data.sig_ref import SIGRef
 
 logger = logging.getLogger(__name__)
 
 
-def get_sig_ref_info(sig, ref_id):
+def get_sig_ref_info(sig_name, ref_id):
     """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig name and
     a reference ID (both strings) and returns a SigRef object populated with as much data as we can find. This makes
     use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
     SIG we are getting data for."""
 
-    if sig is None or sig == "" or ref_id is None or ref_id == "":
+    if sig_name is None or sig_name == "" or ref_id is None or ref_id == "":
         logger.debug("Failed to look up sig_ref info, sig or ref were not set.")
         return None
 
-    sig_ref = SIGRef(sig=sig, id=ref_id)
+    # Sometimes we allow spaces instead of dashes in references due to common usage that way, but official reference
+    # lists never do, so convert them here.
+    ref_id = ref_id.replace(" ", "-")
+
+    # Prepare the object to be returned
+    sig_ref = SIGRef(sig=sig_name, id=ref_id)
+
+    # We can always get the reference type and the icon from the SIG itself
+    for sig in SIGS:
+        if sig.name.upper() == sig_name.upper():
+            sig_ref.ref_type = sig.ref_type
+            sig_ref.icon = sig.icon
 
     try:
         ### FUDGES ###
         #
         # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all
-        # activators add leading zeros.
+        # activators add leading zeros. We also need to normalise "DME 01234" to "DME-01234" to match what's in our
-        if sig.upper() == "DME":
+        # database.
-            ref_id = ref_id.zfill(5)
+        if sig_name.upper() == "DME":
+            match = re.match(r"DME[\- ](\d{3,5})", ref_id, re.IGNORECASE)
+            if match:
+                number = match.group(1)
+                ref_id = f"DME-{number.zfill(5)}"
 
-        # If the SIG is HEMA, we have no current lookup for this so just skip the lookup here.
+        # DTMBA spotters sometimes include spaces and dashes, our regex allows them but they must be removed here so we
-        if sig.upper() == "HEMA":
+        # can look up against the official list which doesn't have them
-            sig_ref.ref_type = "Summit"
+        if sig_name.upper() == "DTMBA":
+            ref_id = ref_id.replace("-", "").replace(" ", "")
+
+        ### NO DATA SIGS ###
+        #
+        # If the SIG is HEMA or BIWOTA, we have no way to either generate useful data or look it up on a reference list,
+        # so just skip the lookup here.
+        if sig_name.upper() == "HEMA" or sig_name.upper() == "BIWOTA":
             return sig_ref
 
         ### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ###
         #
         # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
         # calculate all the information we are going to get directly.
-        if sig.upper() == "TILES":
+        if sig_name.upper() == "TILES":
             # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
-            sig_ref.ref_type = "Grid"
             if not sig_ref.name:
                 sig_ref.name = sig_ref.id
             if not sig_ref.grid:
@@ -51,8 +75,7 @@ def get_sig_ref_info(sig, ref_id):
                 sig_ref.longitude = ll[1]
             return sig_ref
 
-        elif sig.upper() == "WAB" or sig.upper() == "WAI":
+        elif sig_name.upper() == "WAB" or sig_name.upper() == "WAI":
-            sig_ref.ref_type = "Grid"
             ll = wab_wai_square_to_lat_lon(ref_id)
             if ll:
                 sig_ref.name = ref_id
@@ -64,16 +87,15 @@ def get_sig_ref_info(sig, ref_id):
                     logger.warning("Invalid lat/lon received for WAB/WAI reference")
             return sig_ref
 
-        elif sig.upper() == "BOTA":
+        elif sig_name.upper() == "BOTA":
             # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
-            sig_ref.ref_type = "Beach"
             if not sig_ref.name:
                 sig_ref.name = sig_ref.id
             if sig_ref.name:
                 sig_ref.url = f"https://www.beachesontheair.com/beaches/{sig_ref.name.lower().replace(' ', '-')}"
             return sig_ref
 
-        elif sig.upper() == "GMA Islands":
+        elif sig_name.upper() == "GMA Islands":
             # GMA Islands is a bit of a mess of GMA and IOTA references. Try looking them both up and see what returns
             # the best result.
             iota_lookup = get_sig_ref_info("IOTA", ref_id)
@@ -84,25 +106,34 @@ def get_sig_ref_info(sig, ref_id):
             for key, value in gma_lookup.__dict__.items():
                 if value is not None and sig_ref.__dict__.get(key) is None:
                     sig_ref.__dict__[key] = value
-            sig_ref.ref_type = "Island"
+            sig_ref.ref_type = SIGRefType.ISLAND
             return sig_ref
 
         ### ACTUAL LOOKUP ###
         #
-        # OK, this is something we have to look up. Now check to see if our data store contains reference data and use
+        # OK, this is something we have to look up. Now check to see if our data store contains reference data and if
-        # that.
+        # so, copy the data into the sig_ref object
-        key = f"{sig}:{ref_id}"
+        key = f"{sig_name}:{ref_id}"
+        try:
             lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
             if lookup_data:
-            return lookup_data
+                for attr, value in lookup_data.__dict__.items():
+                    if value is not None and sig_ref.__dict__.get(attr) is None:
+                        sig_ref.__dict__[attr] = value
 
             else:
                 # Maybe a super new reference we don't know about yet, but more likely a typo or a test reference,
                 # just silently ignore it.
-            logger.debug(f"{sig} database did not contain data for ref {ref_id}")
+                logger.debug(f"{sig_name} database did not contain data for ref {ref_id}")
+
+        except (ValueError, KeyError):
+            # Catch exceptions due to e.g. old versions of objects in the cache that are no longer compatible,
+            # and remove them from the cache.
+            del DATA_STORE.sigrefs[key]
+            return None
 
     except Exception:
-        logger.exception(f"Exception when looking up sig_ref info for {sig} ref {ref_id}")
+        logger.exception(f"Exception when looking up sig_ref info for {sig_name} ref {ref_id}")
     return sig_ref
 
 

core/status_reporter.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from datetime import datetime
 from threading import Event, Thread
@@ -11,7 +12,10 @@ from core.constants import SOFTWARE_VERSION
 from core.data_providers import DATA_PROVIDERS
 from core.data_store import DATA_STORE
 from core.prometheus_metrics_handler import alerts_gauge, memory_use_gauge, spots_gauge
-from server.webserver import WEB_SERVER
+from telnetserver.telnetserver import TELNET_SERVER
+from webserver.webserver import WEB_SERVER
+
+logger = logging.getLogger(__name__)
 
 
 class StatusReporter:
@@ -32,13 +36,17 @@ class StatusReporter:
     def start(self):
         """Start the reporter thread"""
 
-        self._thread = Thread(target=self._run, name="StatusReporter")
+        self._thread = Thread(target=self._run, name="StatusReporter", daemon=True)
         self._thread.start()
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""
 
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("Status reporter worker thread did not exit on time and will be killed.")
 
     def _run(self):
         """Thread entry point: report immediately on startup, then on each interval until stopped"""
@@ -134,19 +142,13 @@ class StatusReporter:
             else 0,
         }
         DATA_STORE.status.get()["webserver"] = {
-            "status": WEB_SERVER.web_server_metrics["status"],
+            "status": WEB_SERVER.web_server_metrics.status,
-            "last_api_access": WEB_SERVER.web_server_metrics["last_api_access_time"]
+            "api_requests_per_hour": WEB_SERVER.web_server_metrics.api_requests_per_hour(),
-            .replace(tzinfo=pytz.UTC)
+            "page_requests_per_hour": WEB_SERVER.web_server_metrics.page_requests_per_hour(),
-            .timestamp()
+            "sse_client_count": WEB_SERVER.sse_client_count,
-            if WEB_SERVER.web_server_metrics["last_api_access_time"]
+        }
-            else 0,
+        DATA_STORE.status.get()["telnet"] = {
-            "api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"],
+            "client_count": TELNET_SERVER.client_count,
-            "last_page_access": WEB_SERVER.web_server_metrics["last_page_access_time"]
-            .replace(tzinfo=pytz.UTC)
-            .timestamp()
-            if WEB_SERVER.web_server_metrics["last_page_access_time"]
-            else 0,
-            "page_access_count": WEB_SERVER.web_server_metrics["page_access_counter"],
         }
         DATA_STORE.status.store()
 

core/url_data_cache.py

@@ -14,14 +14,13 @@ class URLDataCache(CachedSession):
     used across multiple threads, though note that URL lookups will block each other this way, so it is still better to
     create one of these objects per thread if possible."""
 
-    _lock = threading.Lock()
-
     def __init__(self, name):
         super().__init__(
             f"{CACHE_DIR}urls/{name}",
             expire_after=timedelta(days=1),
             allowable_codes=(200, 400, 401, 403, 404),
         )
+        self._lock = threading.Lock()
 
     def get(self, *args, **kwargs):
         with self._lock:

core/utils.py

@@ -1,20 +1,14 @@
 import logging
+import re
 
 import simplejson
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
 
-from core.constants import (
+from core.constants import BANDS, UNKNOWN_BAND
-    ALL_MODES,
-    BANDS,
-    CW_MODES,
-    DATA_MODES,
-    MODE_ALIASES,
-    PHONE_MODES,
-    UNKNOWN_BAND,
-)
 from core.data_store import DATA_STORE
-from data.callsign import Callsign
+from core.enums import MODE_ALIASES, Continent, Mode, ModeType
+from data.callsign import Callsign, LocationSourceForCallsign
 
 logger = logging.getLogger(__name__)
 
@@ -26,28 +20,40 @@ def safe_json_dumps(obj):
     return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
 
 
-def infer_mode_from_comment(comment):
+def infer_mode_from_comment(comment: str) -> Mode | None:
     """Infer a mode from the comment"""
 
-    for mode in ALL_MODES:
+    if not comment:
-        if mode in comment.upper():
+        return None
+
+    for mode in Mode:
+        if re.search(r"(^|\W)" + mode + r"($|\W)", comment, re.IGNORECASE):
             return mode
-    for mode in MODE_ALIASES:
+    for alias in MODE_ALIASES:
-        if mode in comment.upper():
+        if re.search(r"(^|\W)" + alias + r"($|\W)", comment, re.IGNORECASE):
-            return MODE_ALIASES[mode]
+            return Mode(MODE_ALIASES[alias])
+
     return None
 
 
-def infer_mode_type_from_mode(mode):
+def infer_mode_type_from_mode(mode: str) -> ModeType | None:
     """Infer a "mode family" from a mode ."""
 
-    if mode.upper() in CW_MODES:
+    if not mode:
-        return "CW"
+        return None
-    elif mode.upper() in PHONE_MODES:
+
-        return "PHONE"
+    if mode in MODE_ALIASES:
-    elif mode.upper() in DATA_MODES:
+        mode = MODE_ALIASES[mode]
-        return "DATA"
+
-    else:
+    try:
+        mode = Mode(mode.upper())
+        if mode.is_cw:
+            return ModeType.CW
+        if mode.is_phone:
+            return ModeType.PHONE
+        return ModeType.DATA
+
+    except ValueError:
         if mode.upper() != "OTHER" and mode != "?":
             logger.warning(f"Found an unrecognised mode: {mode}. Developer should categorise this.")
         return None
@@ -70,7 +76,7 @@ def infer_mode_from_frequency(freq):
         mode = freq_to_band(khz)["mode"]
         # Some additional common digimode ranges in addition to what the 3rd-party freq_to_band function returns.
         # This is mostly here just because freq_to_band is very specific about things like FT8 frequencies, and e.g.
-        # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
+        # a spot at 7074.5 kHz will be indicated as SSB, even though it's clearly in the FT8 range. Future updates
         # might include other common digimode centres of activity here, but this achieves the main goal of keeping
         # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
         if (
@@ -93,7 +99,16 @@ def infer_mode_from_frequency(freq):
             or (28180 <= khz < 28183)
         ):
             mode = "FT4"
-        return mode
+
+        if not mode:
+            return None
+
+        mode = mode.upper()
+        if mode in MODE_ALIASES:
+            mode = MODE_ALIASES[mode]
+
+        return Mode(mode)
+
     except KeyError:
         return None
 
@@ -115,7 +130,7 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
 
         country = data.get("country", None)
         dxcc_id = data.get("adif", None)
-        continent = data.get("continent", None)
+        continent = Continent(data["continent"]) if "continent" in data else None
         cq_zone = data.get("cqz", None)
         itu_zone = data.get("ituz", None)
         lat = float(data["latitude"]) if "latitude" in data else None
@@ -135,7 +150,7 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
             latitude=lat,
             longitude=lon,
             grid=grid,
-            location_source="DXCC",
+            location_source=LocationSourceForCallsign.DXCC,
         )
 
     except (KeyError, ValueError):

data/alert.py

@@ -1,12 +1,13 @@
 import hashlib
 import json
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 
 import pytz
 
 from core.call_lookup_helper import get_call_info
+from core.enums import AlertType, Continent
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.utils import get_flag_for_dxcc
 
@@ -19,6 +20,9 @@ class Alert:
 
     # Unique identifier for the alert
     id: str | None = None
+
+    # DX (alerting) operator info
+
     # Callsigns of the operators that has been alerted
     dx_calls: list | None = None
     # Names of the operators that has been alerted
@@ -28,13 +32,16 @@ class Alert:
     # Country flag of the DX operator
     dx_flag: str | None = None
     # Continent of the DX operator
-    dx_continent: str | None = None
+    dx_continent: Continent | None = None
     # DXCC ID of the DX operator
     dx_dxcc_id: int | None = None
     # CQ zone of the DX operator
     dx_cq_zone: int | None = None
     # ITU zone of the DX operator
     dx_itu_zone: int | None = None
+
+    # General alert info
+
     # Intended frequencies & modes of operation. Essentially just a different kind of comment field.
     freqs_modes: str | None = None
     # Start time of the activation, UTC seconds since UNIX epoch
@@ -45,25 +52,41 @@ class Alert:
     end_time: float | None = None
     # End time of the activation of the alert, ISO 8601
     end_time_iso: str | None = None
+    # Comment made by the alerter, if any
+    comment: str | None = None
+    # The type of alert this is: xOTA, DXpedition, or Contest.
+    alert_type: AlertType | None = None
+    # A URL link to more information, if any
+    url: str | None = None
+
+    # Special Interest Group info
+
+    # Special Interest Group (SIG), e.g. outdoor activity programme such as POTA
+    sig: str | None = None
+    # SIG references. We allow multiple here for e.g. n-fer activations, unlike ADIF SIG_INFO
+    sig_refs: list = field(default_factory=list)
+
+    # Timing info
+
     # Time that this software received the alert, UTC seconds since UNIX epoch. This is used with the "since_received"
     # call to our API to receive all data that is new to us, even if by a quirk of the API it might be older than the
     # list time the client polled the API.
     received_time: float | None = None
     # Time that this software received the alert, ISO 8601
     received_time_iso: str | None = None
-    # Comment made by the alerter, if any
+
-    comment: str | None = None
+    # Source info
-    # Special Interest Group (SIG), e.g. outdoor activity programme such as POTA
+
-    sig: str | None = None
-    # SIG references. We allow multiple here for e.g. n-fer activations, unlike ADIF SIG_INFO
-    sig_refs: list | None = None
-    # Whether this alert is for a DXpedition, as opposed to e.g. an xOTA programme.
-    is_dxpedition: bool = False
     # Where we got the alert from, e.g. "POTA", "SOTA"...
     source: str | None = None
     # The ID the source gave it, if any.
     source_id: str | None = None
 
+    # Display info
+
+    # Icon to use when displaying this alert in the web UI. Chosen from the Font Awesome set.
+    icon: str | None = None
+
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
 
@@ -91,8 +114,8 @@ class Alert:
                 call_info = get_call_info(self.dx_calls[0], credentials)
                 if self.dx_calls and self.dx_calls[0] and not self.dx_country:
                     self.dx_country = call_info.country
-                if self.dx_calls and self.dx_calls[0] and not self.dx_continent:
+                if self.dx_calls and self.dx_calls[0] and call_info.continent and not self.dx_continent:
-                    self.dx_continent = call_info.continent
+                    self.dx_continent = Continent(call_info.continent)
                 if self.dx_calls and self.dx_calls[0] and not self.dx_cq_zone:
                     self.dx_cq_zone = call_info.cq_zone
                 if self.dx_calls and self.dx_calls[0] and not self.dx_itu_zone:
@@ -105,13 +128,13 @@ class Alert:
             # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference
             # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs
             # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths
-            if self.sig_refs and len(self.sig_refs) > 0:
+            if self.sig_refs:
                 for sig_ref in self.sig_refs:
                     populate_missing_sig_ref_info(sig_ref)
 
             # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG
             # and apply it to the whole spot.
-            if self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0] and not self.sig:
+            if self.sig_refs and self.sig_refs[0] and not self.sig:
                 self.sig = self.sig_refs[0].sig
 
             # Create an ID based on the source and source ID if possible, as these guaranee uniqueness. If there is no
@@ -129,6 +152,17 @@ class Alert:
             if self.dx_calls and not self.dx_names:
                 self.dx_names = [get_call_info(c, credentials).name for c in self.dx_calls]
 
+            # Icon for the spot should be the icon of the first SIG ref if present, otherwise a radio tower
+            self.icon = "fa-tower-cell"
+            if self.alert_type == AlertType.DXPEDITION:
+                self.icon = "fa-globe-africa"
+            elif self.alert_type == AlertType.CONTEST:
+                self.icon = "fa-trophy"
+            elif self.alert_type == AlertType.SATELLITE:
+                self.icon = "fa-satellite"
+            elif self.sig_refs and self.sig_refs[0].icon:
+                self.icon = self.sig_refs[0].icon
+
         except Exception:
             logger.exception("Exception while inferring missing data from spot")
 

data/callsign.py

@@ -1,5 +1,7 @@
 from dataclasses import dataclass
 
+from core.enums import Continent, LocationSourceForCallsign
+
 
 @dataclass
 class Callsign:
@@ -28,15 +30,15 @@ class Callsign:
     # Country in which the callsign indicates they are operating
     country: str | None = None
     # Continent in which the callsign indicates they are operating
-    continent: str | None = None
+    continent: Continent | None = None
     # DXCC ID in which the callsign indicates they are operating
     dxcc_id: int | None = None
     # CQ zone in which the callsign indicates they are operating
     cq_zone: int | None = None
     # ITU zone in which the callsign indicates they are operating
     itu_zone: int | None = None
-    # Location source. This can be "HOME QTH" or "DXCC" depending on which provider gave us a location
+    # Location source
-    location_source: str | None = None
+    location_source: LocationSourceForCallsign | None = None
 
     def fully_populated(self):
         """Utility method to indicate that the callsign data is fully populated. Multiple providers can return data for

data/sig.py

@@ -1,5 +1,7 @@
 from dataclasses import dataclass, field
 
+from core.enums import SIGRefType, SIGType
+
 
 @dataclass
 class SIG:
@@ -15,7 +17,23 @@ class SIG:
     name: str
     # Description, e.g. "Towers on the Air"
     description: str
+    # Type, either Worldwide, Regional or Event. Used for sorting in the web UI.
+    sig_type: SIGType
+    # Identifies that the SIG's reference ID structure defined by its regex is unique across all programmes and
+    # anything else we expect a user to put in a spot comment, and therefore we can pull references out of
+    # spot comments without also needing to see the SIG name first. For example, "OHFF-1234" or "B/G-1234" are
+    # obviously WWFF and WWBOTA, nothing else looks like those. But "SZ09" could be WAB or Tiles, "GB1234" could
+    # conceivably be POTA or ILLW, etc.
+    refs_globally_unique: bool
     # SIG names as they might appear in cluster spot comments, e.g. ["TOTA"]
     comment_names: list[str] = field(default_factory=list)
+    # Reference type, what gets activated e.g. Park, Summit. May be None if the SIG is for multiple types of things, in
+    # which case the spot data will have to provide this instead.
+    ref_type: SIGRefType | None = None
     # Regex matcher for references, e.g. for POTA r"[A-Z]{2}\-\d+".
     ref_regex: str | None = None
+    # Icon to use in the UI when referencing this SIG. Chosen from the Font Awesome set.
+    icon: str | None = None
+    # Emoji flag for the country or region where this SIG is relevant, if any. If None, this implies the SIG is in
+    # worldwide usage.
+    region_flag: str | None = None

data/sig_ref.py

@@ -1,21 +1,25 @@
 from dataclasses import dataclass
 
+from core.enums import SIGRefType
+
 
 @dataclass
 class SIGRef:
     """Data class that defines a Special Interest Group "info" or reference. As well as the basic reference ID we include a
     name and a lookup URL."""
 
-    # Reference ID, e.g. "GB-0001".
-    id: str
     # SIG that this reference is in, e.g. "POTA".
     sig: str
+    # Reference ID, e.g. "GB-0001".
+    id: str | None = None
     # Name of the reference, e.g. "Null Country Park", if known.
     name: str | None = None
     # Type of the reference, e.g. "Park", if known.
-    ref_type: str | None = None
+    ref_type: SIGRefType | None = None
     # URL to look up more information about the reference, if known.
     url: str | None = None
+    # Icon to use for the reference, derived from the SIG. Chosen from the Font Awesome set.
+    icon: str | None = None
     # Latitude of the reference, in degrees, if known.
     latitude: float | None = None
     # Longitude of the reference, in degrees, if known.

data/spot.py

@@ -2,7 +2,7 @@ import hashlib
 import json
 import logging
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 from math import isnan
 
@@ -11,7 +11,9 @@ from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 
 from core.call_lookup_helper import get_call_info
 from core.config import MAX_SPOT_AGE
-from core.constants import MODE_ALIASES, PROPAGATION_MODES
+from core.constants import PROPAGATION_MODES, SIGS
+from core.data_store import DATA_STORE
+from core.enums import Continent, LocationSourceForSpot, Mode, ModeSource, ModeType
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.sig_utils import (
@@ -52,7 +54,7 @@ class Spot:
     # Country flag of the DX operator
     dx_flag: str | None = None
     # Continent of the DX operator
-    dx_continent: str | None = None
+    dx_continent: Continent | None = None
     # DXCC ID of the DX operator
     dx_dxcc_id: int | None = None
     # CQ zone of the DX operator
@@ -68,9 +70,8 @@ class Spot:
     # lookup
     dx_latitude: float | None = None
     dx_longitude: float | None = None
-    # DX Location source. Indicates how accurate the location might be. Values: "SPOT", "GRID", "SIG REF LOOKUP",
+    # DX Location source. Indicates how accurate the location might be.
-    # "HOME QTH", "DXCC", "NONE"
+    dx_location_source: LocationSourceForSpot | None = None
-    dx_location_source: str = "NONE"
     # DX Location good. Indicates that the software thinks the location data is good enough to plot on a map. This is
     # true if the location source is "SPOT", "SIG REF LOOKUP" or "GRID", or if the location source is "HOME QTH" and the
     # DX callsign doesn't have a suffix like /P.
@@ -85,7 +86,7 @@ class Spot:
     # Country flag of the spotter
     de_flag: str | None = None
     # Continent of the spotter
-    de_continent: str | None = None
+    de_continent: Continent | None = None
     # DXCC ID of the spotter
     de_dxcc_id: int | None = None
     # If this is an APRS/Packet/etc spot, what SSID was the spotter/receiver using?
@@ -103,11 +104,11 @@ class Spot:
     # General QSO info
 
     # Reported mode, such as SSB, PHONE, CW, FT8...
-    mode: str | None = None
+    mode: Mode | None = None
-    # Inferred mode "family". One of "CW", "PHONE" or "DIGI".
+    # Inferred mode "family".
-    mode_type: str | None = None
+    mode_type: ModeType | None = None
-    # Source of the mode information. "SPOT", "COMMENT", "BANDPLAN" or "NONE"
+    # Source of the mode information.
-    mode_source: str = "NONE"
+    mode_source: ModeSource | None = None
     # Frequency, in Hz
     freq: float | None = None
     # Band, defined by the frequency, e.g. "40m" or "70cm"
@@ -124,7 +125,7 @@ class Spot:
     # Special Interest Group (SIG), e.g. outdoor activity programme such as POTA
     sig: str | None = None
     # SIG references. We allow multiple here for e.g. n-fer activations, unlike ADIF SIG_INFO
-    sig_refs: list | None = None
+    sig_refs: list = field(default_factory=list)
 
     # Timing info
 
@@ -146,6 +147,11 @@ class Spot:
     # The ID the source gave it, if any.
     source_id: str | None = None
 
+    # Display info
+
+    # Icon to use when displaying this spot in the web UI. Chosen from the Font Awesome set.
+    icon: str | None = None
+
     def __post_init__(self):
         """Normalise fields that don't survive a plain dict to Spot conversion. This is used in the "add spot" API
         endpoint where the client is submitting JSON, and we want to recreate a full Spot object, including nested
@@ -184,8 +190,8 @@ class Spot:
             dx_call_info = get_call_info(self.dx_call, credentials)
             if self.dx_call and not self.dx_country:
                 self.dx_country = dx_call_info.country
-            if self.dx_call and not self.dx_continent:
+            if self.dx_call and dx_call_info.continent and not self.dx_continent:
-                self.dx_continent = dx_call_info.continent
+                self.dx_continent = Continent(dx_call_info.continent)
             if self.dx_call and not self.dx_dxcc_id:
                 self.dx_dxcc_id = dx_call_info.dxcc_id
             if self.dx_dxcc_id and not self.dx_flag:
@@ -222,8 +228,8 @@ class Spot:
             ):
                 if not self.de_country:
                     self.de_country = de_call_info.country
-                if not self.de_continent:
+                if de_call_info.continent and not self.de_continent:
-                    self.de_continent = de_call_info.continent
+                    self.de_continent = Continent(de_call_info.continent)
                 if not self.de_dxcc_id:
                     self.de_dxcc_id = de_call_info.dxcc_id
                 if self.de_dxcc_id and not self.de_flag:
@@ -240,17 +246,13 @@ class Spot:
 
             # Mode from comments or bandplan
             if self.mode:
-                self.mode_source = "SPOT"
+                self.mode_source = ModeSource.SPOT
             if self.comment and not self.mode:
                 self.mode = infer_mode_from_comment(self.comment)
-                self.mode_source = "COMMENT"
+                self.mode_source = ModeSource.COMMENT
             if self.freq and not self.mode:
                 self.mode = infer_mode_from_frequency(self.freq)
-                self.mode_source = "BANDPLAN"
+                self.mode_source = ModeSource.BANDPLAN
-
-            # Normalise mode if necessary.
-            if self.mode in MODE_ALIASES:
-                self.mode = MODE_ALIASES[self.mode]
 
             # Mode type from mode
             if self.mode and not self.mode_type:
@@ -258,19 +260,19 @@ class Spot:
 
             # If we have a latitude or grid at this point, it can only have been provided by the spot itself
             if self.dx_latitude or self.dx_grid:
-                self.dx_location_source = "SPOT"
+                self.dx_location_source = LocationSourceForSpot.SPOT
 
             # Set the top-level "SIG" if it is missing but we have at least one SIG ref.
-            if not self.sig and self.sig_refs and len(self.sig_refs) > 0:
+            if not self.sig and self.sig_refs:
                 self.sig = self.sig_refs[0].sig.upper()
 
             # See if we already have a SIG reference, but the comment looks like it contains more for the same SIG. This
             # should catch e.g. POTA comments like "2-fer: GB-0001 GB-0002".
-            if self.comment and self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0].sig:
+            if self.comment and self.sig_refs and self.sig_refs[0].sig:
                 sig = self.sig_refs[0].sig.upper()
                 regex = get_ref_regex_for_sig(sig)
                 if regex:
-                    all_comment_ref_matches = re.finditer(r"(^|\W)(" + regex + r")(^|\W)", self.comment, re.IGNORECASE)
+                    all_comment_ref_matches = re.finditer(r"(^|\W)(" + regex + r")($|\W)", self.comment, re.IGNORECASE)
                     for ref_match in all_comment_ref_matches:
                         self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(2).upper(), sig=sig))
 
@@ -298,10 +300,23 @@ class Spot:
                         for ref_match in ref_matches:
                             self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig))
 
+            # See if the comment looks like it contains any SIG references *without* the corresponding SIG name, but
+            # where the SIG reference is unique-looking enough that we can't confuse it with any other SIG.
+            if self.comment:
+                for sig in SIGS:
+                    if sig.refs_globally_unique and sig.ref_regex:
+                        ref_matches = re.finditer(r"(^|\W)(" + sig.ref_regex + r")($|\W)", self.comment, re.IGNORECASE)
+                        for ref_match in ref_matches:
+                            # First of all, if we haven't got a SIG for this spot set yet, now we have. This covers things
+                            # like cluster spots where the comment is just "OHFF-1234", now we know it's WWFF.
+                            if not self.sig:
+                                self.sig = sig.name
+                            self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(2).upper(), sig=sig.name))
+
             # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference
             # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs
             # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths
-            if self.sig_refs and len(self.sig_refs) > 0:
+            if self.sig_refs:
                 for sig_ref in self.sig_refs:
                     sig_ref = populate_missing_sig_ref_info(sig_ref)
                     # If the spot itself doesn't have location yet, but the SIG ref does, extract it
@@ -311,13 +326,13 @@ class Spot:
                         self.dx_latitude = sig_ref.latitude
                         self.dx_longitude = sig_ref.longitude
                         if self.sig == "WAB" or self.sig == "WAI" or self.sig == "Tiles":
-                            self.dx_location_source = "GRID"
+                            self.dx_location_source = LocationSourceForSpot.GRID
                         else:
-                            self.dx_location_source = "SIG REF LOOKUP"
+                            self.dx_location_source = LocationSourceForSpot.SIG_REF_LOOKUP
 
             # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG
             # and apply it to the whole spot.
-            if self.sig_refs and len(self.sig_refs) > 0 and not self.sig:
+            if self.sig_refs and not self.sig:
                 self.sig = self.sig_refs[0].sig
 
             # Parse "de_grid<prop_mode>dx_grid" structures from the comment, e.g. "JN61ES(ES)JM56XT" or "JO02GQ<>KN17LG".
@@ -334,7 +349,7 @@ class Spot:
                         self.de_grid = grid_mode_grid_match.group(1).upper()
                     if not self.dx_grid:
                         self.dx_grid = grid_mode_grid_match.group(4).upper()
-                        self.dx_location_source = "GRID"
+                        self.dx_location_source = LocationSourceForSpot.GRID
 
                     # And extract propagation mode (group 2 for <...>, group 3 for (...)):
                     mode_tag = (grid_mode_grid_match.group(2) or grid_mode_grid_match.group(3) or "").upper()
@@ -345,6 +360,18 @@ class Spot:
                             self.propagation_mode = mode_tag
                             logger.info(f"Seen a new propagation mode tag not yet in the system: {mode_tag}")
 
+            # Set SIGs based on propagation mode
+            if self.propagation_mode == "Satellite":
+                if not self.sig:
+                    self.sig = "AMSAT"
+                if not any(sig_ref.sig == "AMSAT" for sig_ref in self.sig_refs):
+                    self.sig_refs.append(SIGRef(sig="AMSAT"))
+            if self.propagation_mode == "Earth-Moon-Earth":
+                if not self.sig:
+                    self.sig = "EME"
+                if not any(sig_ref.sig == "EME" for sig_ref in self.sig_refs):
+                    self.sig_refs.append(SIGRef(sig="EME"))
+
             # Parse "de_grid -> dx_grid" structures from the comment
             if self.comment:
                 grid_mode_grid_match = re.search(
@@ -355,7 +382,7 @@ class Spot:
                     # regex matches, so extract grids:
                     if not self.dx_grid:
                         self.dx_grid = grid_mode_grid_match.group(1).upper()
-                        self.dx_location_source = "GRID"
+                        self.dx_location_source = LocationSourceForSpot.GRID
                     if not self.de_grid:
                         self.de_grid = grid_mode_grid_match.group(2).upper()
 
@@ -401,7 +428,7 @@ class Spot:
 
             # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable string,
             # otherwise see what they have set on an online lookup service.
-            if self.sig_refs and len(self.sig_refs) > 0:
+            if self.sig_refs:
                 qth = self.sig_refs[0].id
                 if self.sig_refs[0].name:
                     qth += f" {self.sig_refs[0].name}"
@@ -421,16 +448,25 @@ class Spot:
                 elif self.dx_call:
                     self.dx_itu_zone = dx_call_info.itu_zone
 
+            # DXCC lookup from callsign if nothing else has provided it
+            if self.dx_call and not self.dx_dxcc_id:
+                for regex, entity_code in DATA_STORE.dxcc_lookup_by_call_regex:
+                    if regex.pattern and regex.match(self.dx_call):
+                        self.dx_dxcc_id = entity_code
+                        break
+            if self.dx_dxcc_id and not self.dx_flag:
+                self.dx_flag = get_flag_for_dxcc(self.dx_dxcc_id)
+
             # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
             # is likely at home.
             self.dx_location_good = bool(
                 self.dx_latitude
                 and self.dx_longitude
                 and (
-                    self.dx_location_source == "SPOT"
+                    self.dx_location_source == LocationSourceForSpot.SPOT
-                    or self.dx_location_source == "SIG REF LOOKUP"
+                    or self.dx_location_source == LocationSourceForSpot.SIG_REF_LOOKUP
-                    or self.dx_location_source == "GRID"
+                    or self.dx_location_source == LocationSourceForSpot.GRID
-                    or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or ""))
+                    or (self.dx_location_source == LocationSourceForSpot.HOME_QTH and "/" not in (self.dx_call or ""))
                 )
             )
 
@@ -446,6 +482,11 @@ class Spot:
                 self.de_longitude = de_call_info.longitude
                 self.de_grid = de_call_info.grid
 
+            # Icon for the spot should be the icon of the first SIG ref if present, otherwise a radio tower
+            self.icon = "fa-tower-cell"
+            if self.sig_refs and self.sig_refs[0].icon:
+                self.icon = self.sig_refs[0].icon
+
         except Exception:
             logger.exception("Exception while inferring missing data from spot")
 
@@ -457,16 +498,14 @@ class Spot:
     def _append_sig_ref_if_missing(self, new_sig_ref):
         """Append a sig_ref to the list, so long as it's not already there."""
 
-        sig_refs = self.sig_refs or []
-        self.sig_refs = sig_refs
         new_sig_ref.id = new_sig_ref.id.strip().upper()
         new_sig_ref.sig = new_sig_ref.sig.strip().upper()
         if new_sig_ref.id == "":
             return
-        for sig_ref in sig_refs:
+        for sig_ref in self.sig_refs:
             if sig_ref.id == new_sig_ref.id and sig_ref.sig == new_sig_ref.sig:
                 return
-        sig_refs.append(new_sig_ref)
+        self.sig_refs.append(new_sig_ref)
 
     def expired(self):
         """Decide if this spot has expired (in which case it should not be added to the system in the first place, and not

datafiles/didbase-stations.csv

@@ -1,12 +1,9 @@
 AA343,"Almaty, Kazakhstan"
 AL945,"Alpena, United States"
 AT138,"Athens, Greece"
-AU930,"Austin, United States"
-BR52P,"Brisbane, Australia"
 BVJ03,"Boa Vista, Brazil"
 CAJ2M,"Cachoeira Paulista, Brazil"
 CB53N,"Canberra, Australia"
-CGK21,"Campo Grande, Brazil"
 DB049,"Dourbes, Belgium"
 DW41K,"Darwin, Australia"
 EA036,"El Arenosillo, Spain"
@@ -18,24 +15,18 @@ GM037,"Gibilmanna, Italy"
 GR13L,"Grahamstown, South Africa"
 HE13N,"Hermanus, South Africa"
 HO54K,"Hobart, Australia"
-IC437,"I-Cheon, South Korea"
 IF843,"Idaho National Laboratory, United States"
 JI91J,"Jicamarca, Peru"
 JR055,"Juliusruh, Germany"
 LAA38,"Lajes Terceira Island, Portugal"
 LL721,"Lualualei, United States"
-LM42B,"Learmonth, Australia"
-LV12P,"Louisvale, South Africa"
 MHJ45,"Millstone Hill, United States"
-ML10L,"Malindi, Kenya"
 NI135,"Nicosia, Cyprus"
 NI63_,"Norfolk, Australia"
-PA836,"Portt Arguello, United States"
+PA836,"Port Arguello, United States"
 PE43K,"Perth, Australia"
-PF765,"Poker Flat, United States"
 PQ052,"Pruhonice, Czechia"
 RO041,"Rome, Italy"
-SAA0K,"Saoluis, Brazil"
 SO148,"Sopron, Hungary"
 TR169,"Tromso, Norway"
 TV51R,"Townsville, Australia"

docs/docker.md

@@ -13,6 +13,7 @@ services:
     restart: unless-stopped
     ports:
       - "8080:8080"
+      - "7373:7373" # For telnet if required
     volumes:
       - ./config.yml:/app/config.yml
       - ./cache:/app/cache
@@ -23,15 +24,16 @@ You can replace `#main` with any other branch or tag reference, for example `#1.
 
 Save the file. You will still need to create a copy of `config-example.yml` and name it `config.yml`, though with the
 Docker setup nothing has actually been downloaded yet, so you will have to copy the example from the repository some
-other way, e.g. [from the repo in a web browser](https://git.ianrenton.com/ian/spothole/src/branch/main/config-example.yml).
+other way,
+e.g. [from the repo in a web browser](https://git.ianrenton.com/ian/spothole/src/branch/main/config-example.yml).
 
 With that in place, run `docker compose up` and you should be good to go. To detach, press `d` or run the command with
 the `-d` flag.
 
 ### nginx Reverse Proxy with Docker
 
-In a containerised setup, it's typical to run an nginx reverse proxy in one container, alongside certbot for renewal
+In a containerised setup, it's typical to run an nginx reverse proxy in one container, alongside certbot for renewal of
-of HTTPS certificates, and then applications like Spothole in a separate container. In this case, there are a couple of
+HTTPS certificates, and then applications like Spothole in a separate container. In this case, there are a couple of
 variations of the docker compose file above, and the nginx reverse proxy configuration covered [here](./nginx.md), that
 you will want to make.
 
@@ -60,8 +62,8 @@ networks:
 ```
 
 In your nginx site configuration, you'll want to refer to the Spothole container directly, and drop the block that
-allows nginx to access static files directly, as these will be inaccessible in another container. So you may end up
+allows nginx to access static files directly, as these will be inaccessible in another container. So you may end up with
-with something like:
+something like:
 
 ```nginx
 server {
@@ -146,13 +148,13 @@ server {
 ```
 
 If desired, you could even change the port on which Spothole runs from 8080 to a plain 80, in which case your
-`proxy_pass` statements could drop the `:8080` suffix. Since Spothole is in a container, it can serve HTTP on port 80
+`proxy_pass` statements could drop the `:8080` suffix. Since Spothole is in a container, it can serve HTTP on port 80 if
-if desired, because it doesn't conflict with the host system.
+desired, because it doesn't conflict with the host system.
 
 ### Restoring the static files bypass
 
-If you would still like to bypass Spothole's web server for the static files, and serve them with nginx, you can
+If you would still like to bypass Spothole's web server for the static files, and serve them with nginx, you can do. The
-do. The easiest way is to run another nginx container to serve the files, so your Spothole `compose.yaml` becomes:
+easiest way is to run another nginx container to serve the files, so your Spothole `compose.yaml` becomes:
 
 ```yaml
 services:
@@ -183,8 +185,8 @@ networks:
     external: true
 ```
 
-Then you can re-add the block that handles the `/static` path in your nginx reverse proxy config, but this time
+Then you can re-add the block that handles the `/static` path in your nginx reverse proxy config, but this time point it
-point it at the new container rather than at a filesystem path:
+at the new container rather than at a filesystem path:
 
 ```nginx
     # Load static assets from the spothole-static-nginx container

docs/modifying.md

@@ -16,9 +16,12 @@ To navigate your way around the source code, this list may help.
 * `/providers/solarconditions` - Classes providing solar and propagation by accessing the APIs of other services
 * `/providers/staticdata` - Classes providing static lookup data by accessing bundled data files or the APIs of other
   services
+* `/providers/callsign` - Classes providing callsign lookup data by accessing bundled data files or the APIs of other
+  services
 * `/providers/sigrefdata` - Classes providing SIG reference lookup data by accessing bundled data files or the APIs of
   other services
-* `/server` - Classes for running Spothole's own web server
+* `/webserver` - Classes for running Spothole's own web server
+* `/telnetserver` - Classes for running Spothole's telnet server
 * `spothole.py` - Main application script
 
 *Templates*
@@ -48,8 +51,8 @@ To navigate your way around the source code, this list may help.
 ### Extending the server
 
 Spothole is designed to be easily extensible. If you want to write your own spot provider, for example, simply add a
-module to the `providers.spot` package containing your class. (Currently, in order to be loaded correctly, the module (
+module to the `providers.spot` package containing your class. (Currently, in order to be loaded correctly, the module
-file) name should be the same as the class name, but lower case.)
+(file) name should be the same as the class name, but lower case.)
 
 Your class should extend "SpotProvider"; if it operates by polling an HTTP Server on a timer, it can instead extend "
 HTTPSpotProvider" where some of the work is done for you.

docs/nginx.md

@@ -2,8 +2,8 @@
 
 Web servers generally serve their pages from port 80. However, it's best not to serve Spothole's web interface directly
 on port 80, as that requires root privileges on a Linux system. It also and prevents us using HTTPS to serve a secure
-site, since Spothole itself doesn't directly support acting as an HTTPS server. The normal solution to this is to use
+site, since Spothole itself doesn't directly support acting as an HTTPS server. The normal solution to this is to use a
-a "reverse proxy" setup, where a general web server handles HTTP and HTTP requests (to port 80 & 443 respectively), then
+"reverse proxy" setup, where a general web server handles HTTP and HTTP requests (to port 80 & 443 respectively), then
 passes on the request to the back-end application (in this case Spothole). nginx is a common choice for this general web
 server.
 
@@ -89,19 +89,14 @@ server {
 ```
 
 One further change you might want to make to the file above is the `add_header Access-Control-Allow-Origin` statements.
-These are what's used on
+These are what's used on my own Spothole server to make sure that other third-party web-based software can get the data
-my own Spothole server to make sure that other third-party web-based software can get the data from my instance, and
+from my instance, and applies to any endpoint underneath `/api`. If you want *your* Spothole instance to be set up the
-applies to any endpoint underneath `/api`. If you want
+same way, so that others can write software in JavaScript that can access it, leave this intact. But if you want your
-*your* Spothole instance to be set up the same way, so that others can write software in JavaScript that can access it,
+Spothole instance to only be usable by scripts running on the web server you write, you can remove these lines. (Note
-leave this intact. But if you want your Spothole instance to only be usable by scripts running on the web server you
+that this doesn't stop other people writing *non-web-based* software that accesses your Spothole API—the
-write,
+enforcement of cross-origin headers only happens within the user's browser. If you need to lock your instance down so
-you can remove these lines. (Note that this doesn't stop other people writing *non-web-based* software that accesses
+that no-one else can access it with *any* software, that's an aspect of nginx or firewall config that you will need to
-your
+find help with elsewhere.)
-Spothole API—the enforcement of cross-origin headers only happens within the user's browser. If you need to lock
-your
-instance down so that no-one else can access it with *any* software, that's an aspect of nginx or firewall config that
-you will need
-to find help with elsewhere.)
 
 Now, make a symbolic link to enable the site:
 

providers/alert/bota.py

@@ -3,6 +3,7 @@ from datetime import datetime, timedelta
 import pytz
 from bs4 import BeautifulSoup
 
+from core.enums import AlertType
 from data.alert import Alert
 from data.sig_ref import SIGRef
 from providers.alert.http_alert_provider import HTTPAlertProvider
@@ -57,7 +58,7 @@ class BOTA(HTTPAlertProvider):
                         dx_calls=[dx_call],
                         sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
                         start_time=date_time.timestamp(),
-                        is_dxpedition=False,
+                        alert_type=AlertType.XOTA,
                     )
 
                     new_alerts.append(alert)

providers/alert/hamsat.py

@@ -0,0 +1,56 @@
+from datetime import datetime
+
+import pytz
+
+from core.enums import AlertType
+from data.alert import Alert
+from data.sig_ref import SIGRef
+from providers.alert.http_alert_provider import HTTPAlertProvider
+
+
+class Hamsat(HTTPAlertProvider):
+    """Alert provider for Hamsat (hams.at)"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://hams.at/api/alerts"
+
+    def __init__(self, provider_config):
+        super().__init__("Hamsat", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json()["data"]:
+            # Convert to our alert format
+            freqs_modes = source_alert.get("mode", "")
+            if "mhz" in source_alert:
+                if "mhz_direction" in source_alert:
+                    freqs_modes = f"{source_alert['mhz']!s} {source_alert['mhz_direction']}, {freqs_modes}"
+                else:
+                    freqs_modes = f"{source_alert['mhz']!s}, {freqs_modes}"
+
+            alert = Alert(
+                source=self.name,
+                source_id=source_alert["id"],
+                dx_calls=[source_alert["callsign"].upper()],
+                freqs_modes=freqs_modes,
+                comment=source_alert["comment"],
+                # Fudge a SIG ref to provide the remaining bits of data we need: the satellite and the operator's grid
+                sig_refs=[
+                    SIGRef(
+                        sig="AMSAT",
+                        id=f"{source_alert['satellite']['name']} from {source_alert['grids'][0]}",
+                    )
+                ],
+                start_time=datetime.strptime(source_alert["aos_at"], "%Y-%m-%dT%H:%M:%SZ")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp(),
+                end_time=datetime.strptime(source_alert["los_at"], "%Y-%m-%dT%H:%M:%SZ")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp(),
+                alert_type=AlertType.SATELLITE,
+            )
+
+            # Add to our list
+            new_alerts.append(alert)
+        return new_alerts

providers/alert/http_alert_provider.py

@@ -27,11 +27,15 @@ class HTTPAlertProvider(AlertProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} alert API every {self._poll_interval!s} seconds.")
-        self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} alert worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/alert/ical_alert_provider.py

@@ -0,0 +1,47 @@
+from datetime import datetime, time
+from typing import cast
+
+import pytz
+from icalendar import Calendar, Event
+
+from data.alert import Alert
+from providers.alert.http_alert_provider import HTTPAlertProvider
+
+
+class ICALAlertProvider(HTTPAlertProvider):
+    """Generic alert provider for iCal calendars. Defines an abstract method event_to_alert(event) that subclasses must
+    implement, and use it to convert an iCal event to an Alert object based on whatever format their iCal events use."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        super().__init__(name, provider_config, url, poll_interval)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        cal = Calendar.from_ical(http_response.content)
+
+        # Iterate through events, passing each one in turn to the subclass' event_to_alert method to turn it into
+        # a Spothole alert object
+        for component in cal.walk():
+            if component.name != "VEVENT":
+                continue
+
+            event = cast(Event, component)
+            alert = self.event_to_alert(event)
+            new_alerts.append(alert)
+        return new_alerts
+
+    def event_to_alert(self, event: Event) -> Alert:
+        """Convert an ICal event to an Alert object. Subclasses must implement this method."""
+
+    @staticmethod
+    def _to_utc_timestamp(value):
+        """Convert a date or datetime value from an iCal field into a UTC UNIX timestamp."""
+
+        # Datetime object so we can treat it as-is, check if it has a non-UTC tz and convert it if necessary
+        if isinstance(value, datetime):
+            if value.tzinfo is None:
+                value = pytz.UTC.localize(value)
+            return value.astimezone(pytz.UTC).timestamp()
+
+        # Date object so this is an all day event
+        return pytz.UTC.localize(datetime.combine(value, time.min)).timestamp()

providers/alert/ng3k.py

@@ -6,19 +6,20 @@ import pytz
 from rss_parser import Parser
 from rss_parser.models.rss import RSS
 
+from core.enums import AlertType
 from data.alert import Alert
 from providers.alert.http_alert_provider import HTTPAlertProvider
 
 
 class NG3K(HTTPAlertProvider):
-    """Alert provider NG3K DXpedition list"""
+    """Alert provider for NG3K DXpedition list"""
 
-    POLL_INTERVAL_SEC = 1800
+    POLL_INTERVAL_DAYS = 1
     ALERTS_URL = "https://www.ng3k.com/adxo.xml"
     AS_CALL_PATTERN = re.compile("as ([a-z0-9/]+)", re.IGNORECASE)
 
     def __init__(self, provider_config):
-        super().__init__("NG3K", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("NG3K", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
@@ -88,7 +89,7 @@ class NG3K(HTTPAlertProvider):
                 comment=f"{by}; {comment}; {qsl_info}",
                 start_time=start_timestamp,
                 end_time=end_timestamp,
-                is_dxpedition=True,
+                alert_type=AlertType.DXPEDITION,
             )
 
             # Add to our list.

providers/alert/parksnpeaks.py

@@ -3,6 +3,7 @@ from datetime import datetime
 
 import pytz
 
+from core.enums import AlertType
 from data.alert import Alert
 from data.sig_ref import SIGRef
 from providers.alert.http_alert_provider import HTTPAlertProvider
@@ -51,7 +52,7 @@ class ParksNPeaks(HTTPAlertProvider):
                 comment=source_alert["Comments"],
                 sig_refs=sigrefs,
                 start_time=start_time,
-                is_dxpedition=False,
+                alert_type=AlertType.XOTA,
             )
 
             # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
@@ -59,6 +60,7 @@ class ParksNPeaks(HTTPAlertProvider):
                 "POTA",
                 "SOTA",
                 "WWFF",
+                "HEMA",
                 "SIOTA",
                 "ZLOTA",
                 "KRMNPA",

providers/alert/pota.py

@@ -2,6 +2,7 @@ from datetime import datetime
 
 import pytz
 
+from core.enums import AlertType
 from data.alert import Alert
 from data.sig_ref import SIGRef
 from providers.alert.http_alert_provider import HTTPAlertProvider
@@ -44,7 +45,7 @@ class POTA(HTTPAlertProvider):
                 end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"], "%Y-%m-%d%H:%M")
                 .replace(tzinfo=pytz.UTC)
                 .timestamp(),
-                is_dxpedition=False,
+                alert_type=AlertType.XOTA,
             )
 
             # Add to our list, but exclude any old spots that POTA can sometimes give us where even the end time is

providers/alert/rsgb_ical_alert_provider.py

@@ -0,0 +1,75 @@
+import re
+
+from icalendar import Event
+
+from core.enums import AlertType, Continent
+from data.alert import Alert
+from providers.alert.ical_alert_provider import ICALAlertProvider
+
+
+class RSGBICALAlertProvider(ICALAlertProvider):
+    """Generic alert provider for RSGB contest iCal calendars. Builds on the generic iCal alert provider by adding
+    handling specific to how RSGB's iCal events are formatted. This is still effectively an abstract class itself;
+    RSGB has two contest calendars (HF & VHF) that each subclass this."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        super().__init__(name, provider_config, url, poll_interval)
+
+    FREQ_PATTERN = re.compile(r"([\d.]+(?:MHz|GHz))|SHF")
+
+    def event_to_alert(self, event: Event) -> Alert:
+        """Convert an iCal event in RSGB's format to an Alert object."""
+
+        summary = str(event.get("summary", "")).strip()
+
+        # Ensure summaries start with "RSGB" to avoid any confusion
+        if not summary.startswith("RSGB "):
+            summary = "RSGB " + summary
+
+        # Extract freq from summary. HF contests don't give frequencies, all VHF ones do
+        match = self.FREQ_PATTERN.search(summary)
+        if match:
+            freqs_modes = match.group()
+        else:
+            freqs_modes = "HF bands"
+        freqs_modes = freqs_modes + ", "
+
+        # Extract mode from summary
+        if "FMAC" in summary:
+            freqs_modes = freqs_modes + "FM"
+        elif "CW" in summary:
+            freqs_modes = freqs_modes + "CW"
+        elif "SSB" in summary:
+            freqs_modes = freqs_modes + "SSB"
+        elif "FT8" in summary:
+            freqs_modes = freqs_modes + "FT8"
+        elif "FT4" in summary:
+            freqs_modes = freqs_modes + "FT4"
+        elif "DATA" in summary:
+            freqs_modes = freqs_modes + "Data modes"
+        else:
+            freqs_modes = freqs_modes + "All modes"
+
+        dtstart = event.get("dtstart")
+        dtend = event.get("dtend")
+        start_timestamp = self._to_utc_timestamp(dtstart.dt)
+        end_timestamp = self._to_utc_timestamp(dtend.dt) - 1 if dtend is not None else start_timestamp
+
+        # Convert to our alert format
+        alert = Alert(
+            source=self.name,
+            dx_calls=[],
+            dx_country="United Kingdom",
+            dx_dxcc_id=235,
+            dx_continent=Continent.EU,
+            dx_cq_zone=14,
+            dx_itu_zone=27,
+            dx_flag="🇬🇧",
+            freqs_modes=freqs_modes,
+            comment=summary,
+            start_time=start_timestamp,
+            end_time=end_timestamp,
+            alert_type=AlertType.CONTEST,
+        )
+
+        return alert

providers/alert/rsgbhfcontests.py

@@ -0,0 +1,11 @@
+from providers.alert.rsgb_ical_alert_provider import RSGBICALAlertProvider
+
+
+class RSGBHFContests(RSGBICALAlertProvider):
+    """Alert provider for RSGB HF Contest calendar"""
+
+    POLL_INTERVAL_DAYS = 30
+    ALERTS_URL = "https://calendar.google.com/calendar/ical/a5ff31ebb1b4834dc7fff4c5415ae8251c6a9aa11f98c6af6e472b6c552b1915%40group.calendar.google.com/public/basic.ics"
+
+    def __init__(self, provider_config):
+        super().__init__("RSGB HF Contests", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60)

providers/alert/rsgbvhfcontests.py

@@ -0,0 +1,11 @@
+from providers.alert.rsgb_ical_alert_provider import RSGBICALAlertProvider
+
+
+class RSGBVHFContests(RSGBICALAlertProvider):
+    """Alert provider for RSGB VHF Contest calendar"""
+
+    POLL_INTERVAL_DAYS = 30
+    ALERTS_URL = "https://calendar.google.com/calendar/ical/40f3552bff39a016f1cdca205864177070dcad68d55be17eb061cb021f39f96c%40group.calendar.google.com/public/basic.ics"
+
+    def __init__(self, provider_config):
+        super().__init__("RSGB VHF Contests", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60)

providers/alert/sota.py

@@ -2,6 +2,7 @@ from datetime import datetime
 
 import pytz
 
+from core.enums import AlertType
 from data.alert import Alert
 from data.sig_ref import SIGRef
 from providers.alert.http_alert_provider import HTTPAlertProvider
@@ -44,7 +45,7 @@ class SOTA(HTTPAlertProvider):
                 start_time=datetime.strptime(source_alert["dateActivated"], "%Y-%m-%dT%H:%M:%SZ")
                 .replace(tzinfo=pytz.UTC)
                 .timestamp(),
-                is_dxpedition=False,
+                alert_type=AlertType.XOTA,
             )
 
             # Add to our list

providers/alert/wa7bnm.py

@@ -0,0 +1,41 @@
+from icalendar import Event
+
+from core.enums import AlertType
+from data.alert import Alert
+from providers.alert.ical_alert_provider import ICALAlertProvider
+
+
+class WA7BNM(ICALAlertProvider):
+    """Alert provider for the WA7BNM contest calendar."""
+
+    POLL_INTERVAL_DAYS = 1
+    ALERTS_URL = "https://contestcalendar.com/weeklycontcustom.php"
+
+    def __init__(self, provider_config):
+        super().__init__(
+            "WA7BNM Contest Calendar", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60
+        )
+
+    def event_to_alert(self, event: Event) -> Alert:
+        """Convert an iCal event in WA7BNM's format to an Alert object."""
+
+        summary = str(event.get("summary", ""))
+        url = str(event.get("url", ""))
+
+        dtstart = event.get("dtstart")
+        dtend = event.get("dtend")
+        start_timestamp = self._to_utc_timestamp(dtstart.dt)
+        end_timestamp = self._to_utc_timestamp(dtend.dt) - 1 if dtend is not None else start_timestamp
+
+        # Convert to our alert format
+        alert = Alert(
+            source=self.name,
+            dx_calls=[],
+            comment=summary,
+            url=url,
+            start_time=start_timestamp,
+            end_time=end_timestamp,
+            alert_type=AlertType.CONTEST,
+        )
+
+        return alert

providers/alert/wwff.py

@@ -2,6 +2,7 @@ from datetime import datetime
 
 import pytz
 
+from core.enums import AlertType
 from data.alert import Alert
 from data.sig_ref import SIGRef
 from providers.alert.http_alert_provider import HTTPAlertProvider
@@ -34,7 +35,7 @@ class WWFF(HTTPAlertProvider):
                 end_time=datetime.strptime(source_alert["utc_end"], "%Y-%m-%d %H:%M:%S")
                 .replace(tzinfo=pytz.UTC)
                 .timestamp(),
-                is_dxpedition=False,
+                alert_type=AlertType.XOTA,
             )
 
             # Add to our list

providers/callsigndata/callsign_data_provider.py

@@ -42,7 +42,13 @@ class CallsignDataProvider:
 
         if self.enabled:
             if callsign in self._storage:
+                try:
                     return self._storage[callsign]
+                except (ValueError, KeyError):
+                    # Catch exceptions due to e.g. old versions of objects in the cache that are no longer compatible,
+                    # and remove them from the cache.
+                    del self._storage[callsign]
+                    return None
             else:
                 c = self._perform_new_lookup(callsign, lookup_credentials)
                 if c:

providers/callsigndata/file_download_callsign_data_provider.py

@@ -32,11 +32,15 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
-        self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} callsign data worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/callsigndata/hamqth.py

@@ -11,8 +11,9 @@ from requests_cache import CachedSession
 from core.config import SERVER_OWNER_CALLSIGN
 from core.constants import HTTP_HEADERS, SOFTWARE_VERSION
 from core.data_store import CACHE_DIR, DATA_STORE
+from core.enums import Continent
 from core.url_data_cache import URLDataCache
-from data.callsign import Callsign
+from data.callsign import Callsign, LocationSourceForCallsign
 from providers.callsigndata.api_query_callsign_data_provider import (
     APIQueryCallsignDataProvider,
 )
@@ -142,12 +143,12 @@ class HamQTH(APIQueryCallsignDataProvider):
             name=data.get("nick", None),
             qth=data.get("qth", None),
             country=data.get("country", None),
-            continent=data.get("continent", None),
+            continent=Continent(data["continent"]) if "continent" in data else None,
             latitude=lat,
             longitude=lon,
             grid=grid,
             dxcc_id=int(data["adif"]) if "adif" in data else None,
             cq_zone=int(data["cq"]) if "cq" in data else None,
             itu_zone=int(data["itu"]) if "itu" in data else None,
-            location_source="HOME QTH",
+            location_source=LocationSourceForCallsign.HOME_QTH,
         )

providers/callsigndata/qrz.py

@@ -10,11 +10,10 @@ from requests_cache import CachedSession
 
 from core.constants import HTTP_HEADERS
 from core.data_store import CACHE_DIR, DATA_STORE
+from core.enums import Continent, LocationSourceForCallsign
 from core.url_data_cache import URLDataCache
 from data.callsign import Callsign
-from providers.callsigndata.api_query_callsign_data_provider import (
+from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
-    APIQueryCallsignDataProvider,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -170,12 +169,12 @@ class QRZ(APIQueryCallsignDataProvider):
             name=name,
             qth=data.get("addr2", None),
             country=data.get("country", None),
-            continent=data.get("continent", None),
+            continent=Continent(data["continent"]) if "continent" in data else None,
             latitude=lat,
             longitude=lon,
             grid=grid,
             dxcc_id=int(data["adif"]) if "adif" in data else None,
             cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
             itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
-            location_source="HOME QTH",
+            location_source=LocationSourceForCallsign.HOME_QTH,
         )

providers/sigrefdata/arlhs.py

@@ -1,6 +1,7 @@
 import csv
 from time import sleep
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -27,7 +28,7 @@ class ARLHS(FileDownloadSIGRefDataProvider):
                         sig=self.SIG,
                         id=ref_id,
                         name=row.get("Name", None),
-                        ref_type="Lighthouse",
+                        ref_type=SIGRefType.LIGHTHOUSE,
                         url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
                         latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
                         longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,

providers/sigrefdata/cota.py

@@ -0,0 +1,52 @@
+from time import sleep
+
+from pyhamtools.locator import latlong_to_locator
+
+from core.enums import SIGRefType
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class COTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Castles on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "COTA"
+    DATA_URL = "https://www.cotagroup.org/cotagroup/map/data/castles-all-7d90ee2a5e1175e5dece1bbf9dc87504.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data[2]:
+                ref_id = ref[3]
+                name = ref[4]
+                lat = float(ref[0])
+                lon = float(ref[1])
+                grid = latlong_to_locator(lat, lon)
+
+                new_data.append(
+                    SIGRef(
+                        sig=self.SIG,
+                        id=ref_id,
+                        name=name,
+                        ref_type=SIGRefType.CASTLE,
+                        grid=grid,
+                        latitude=lat,
+                        longitude=lon,
+                    )
+                )
+
+                # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
+                # of the data in this case
+                if self._stop_event.is_set():
+                    break
+
+                # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+                # is available for other threads e.g. the web server.
+                sleep(0.001)
+
+        return new_data

providers/sigrefdata/dce.py

@@ -0,0 +1,42 @@
+import io
+from time import sleep
+
+import pandas as pd
+
+from core.enums import SIGRefType
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class DCE(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Diploma Castillos de España"""
+
+    POLL_INTERVAL_DAYS = 365
+    SIG = "DCE"
+    DATA_URL = "https://www.acracb.org/dce/descargas/General/directorio_referencias_dce.xls"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+
+        file_stream = io.BytesIO(http_response.content)
+        df = pd.read_excel(file_stream, engine="xlrd", header=None)
+
+        for index, row in df.iterrows():
+            if row.iloc[0] and row.iloc[2]:
+                new_data.append(
+                    SIGRef(sig=self.SIG, id=row.iloc[0].strip(), name=row.iloc[2].strip(), ref_type=SIGRefType.CASTLE)
+                )
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
+        return new_data

providers/sigrefdata/defe.py

@@ -0,0 +1,46 @@
+import io
+from time import sleep
+
+import pandas as pd
+
+from core.enums import SIGRefType
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class DEFE(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Diploma Estationes de Ferrocarril de España"""
+
+    POLL_INTERVAL_DAYS = 365
+    SIG = "DEFE"
+    DATA_URL = "https://www.acracb.org/defe/descargas/General/directorio_referencias_defe.xls"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+
+        file_stream = io.BytesIO(http_response.content)
+        df = pd.read_excel(file_stream, engine="xlrd", header=None)
+
+        for index, row in df.iterrows():
+            # Skip the header row
+            if str(row.iloc[0]) == "NºDEFE":
+                continue
+
+            if row.iloc[0] and row.iloc[1]:
+                new_data.append(
+                    SIGRef(sig=self.SIG, id=row.iloc[0].strip(), name=row.iloc[1].strip(), ref_type=SIGRefType.BUILDING)
+                )
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
+        return new_data

providers/sigrefdata/dme.py

@@ -3,6 +3,7 @@ from time import sleep
 
 from pyhamtools.locator import latlong_to_locator
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.local_file_sig_ref_data_provider import (
     LocalFileSIGRefDataProvider,
@@ -22,7 +23,11 @@ class DME(LocalFileSIGRefDataProvider):
         new_data = []
         with open(path, encoding="latin-1") as _f:
             for row in csv.DictReader(_f, delimiter=";"):
-                ref_id = row["COD_INE"][:5]
+                # Store reference IDs with the "DME-" prefix rather than just the number. This will prevent Spothole
+                # from agressively thinking every number in a spot comment is DME after it's seen "DME" once. The only
+                # numbers that count are straight after "DME " or "DME-". The dash versus space is normalised in
+                # sig_lookup_helper.py.
+                ref_id = "DME-" + row["COD_INE"][:5]
                 latitude = (
                     float(row["LATITUD_ETRS89_REGCAN95"].replace(",", "."))
                     if row.get("LATITUD_ETRS89_REGCAN95")
@@ -37,7 +42,7 @@ class DME(LocalFileSIGRefDataProvider):
                 ref = SIGRef(
                     sig=self.SIG,
                     id=ref_id,
-                    ref_type="Town",
+                    ref_type=SIGRefType.TOWN,
                     name=f"{row['NOMBRE_ACTUAL']}, {row['PROVINCIA']}",
                     latitude=latitude,
                     longitude=longitude,
@@ -48,7 +53,7 @@ class DME(LocalFileSIGRefDataProvider):
 
                 # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
                 # of the data in this case
-                if self._stop:
+                if self._stop_event.is_set():
                     break
 
                 # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time

providers/sigrefdata/dmue.py

@@ -0,0 +1,37 @@
+import csv
+from time import sleep
+
+from core.enums import SIGRefType
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class DMUE(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Diploma Museos de España"""
+
+    POLL_INTERVAL_DAYS = 365
+    SIG = "DMUE"
+    DATA_URL = "https://dmue.radiogalena.es/nom_dmue.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+
+        for row in csv.reader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
+            if len(row) > 1 and row[0] and row[1]:
+                new_data.append(
+                    SIGRef(sig=self.SIG, id=row[0].strip(), name=row[1].strip(), ref_type=SIGRefType.BUILDING)
+                )
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
+        return new_data

providers/sigrefdata/dmve.py

@@ -0,0 +1,50 @@
+import io
+from time import sleep
+
+import pandas as pd
+
+from core.enums import SIGRefType
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class DMVE(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Diploma Monumentos y Vestigios de España"""
+
+    POLL_INTERVAL_DAYS = 365
+    SIG = "DMVE"
+    DATA_URL = "https://www.acracb.org/dmve/descargas/General/directorio_referencias_dmve.xls"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+
+        file_stream = io.BytesIO(http_response.content)
+        # Despide the .xls extension this is actually an xlsx file, so we need openpyxl not xlrd
+        df = pd.read_excel(file_stream, engine="openpyxl", header=None)
+
+        for index, row in df.iterrows():
+            ref = row.iloc[0]
+            name = row.iloc[1]
+
+            # Skip the header row and blank rows
+            if str(ref) == "REF.":
+                continue
+            if pd.isna(ref) or pd.isna(name):
+                continue
+
+            if ref and name:
+                new_data.append(SIGRef(sig=self.SIG, id=ref.strip(), name=name.strip(), ref_type=SIGRefType.BUILDING))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
+        return new_data

providers/sigrefdata/dtmba.py

@@ -0,0 +1,35 @@
+from time import sleep
+
+from core.enums import SIGRefType
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class DTMBA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Diploma Teatri Musei Belle Arti"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "DTMBA"
+    DATA_URL = "https://www.iu1fig.com/share/iz0eik/dtmba/export.php"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in http_response.content.decode("utf-8-sig").splitlines():
+            split = row.split(";")
+            ref_id = split[0]
+            ref_name = split[1]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref_name, ref_type=SIGRefType.BUILDING))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
+        return new_data

providers/sigrefdata/fea.py

@@ -0,0 +1,56 @@
+from io import BytesIO
+from time import sleep
+
+import pdfplumber
+
+from core.enums import SIGRefType
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class FEA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Diploma Faros de España"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "FEA"
+    DATA_URL = "http://ea5ol.net/Lista%20Faros.pdf"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+
+        # Use PDFPlumber to extract the tables in the PDF
+        with pdfplumber.open(BytesIO(http_response.content)) as pdf:
+            all_rows = []
+
+            for page_number, page in enumerate(pdf.pages, start=1):
+                tables = page.extract_tables()
+
+                for table_number, table in enumerate(tables, start=1):
+                    if not table:
+                        continue
+
+                    rows = [row for row in table if any(cell and cell.strip() for cell in row)]
+                    all_rows.extend(rows)
+
+        for row in all_rows:
+            if not "REF" in row[0] and not "\n" in row[0]:
+                # FEA references are technically [DE]\-\d{4}(\.\d)? but spotters always seem to miss out the D- or E-
+                # prefix and just use FEA-1234 or FEA 1234, so we add both copies to the database.
+                ref_id_1 = row[0].strip()
+                ref_id_2 = ref_id_1.replace("D-", "FEA-").replace("E-", "FEA-")
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id_1, name=row[1].strip(), ref_type=SIGRefType.LIGHTHOUSE))
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id_2, name=row[1].strip(), ref_type=SIGRefType.LIGHTHOUSE))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
+        return new_data

providers/sigrefdata/file_download_sig_ref_data_provider.py

@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Event, Thread
+from threading import Thread
 
 import pytz
 from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
@@ -21,19 +21,21 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         self._url = url
         self._poll_interval = poll_interval
         self._thread = None
-        self._stop_event = Event()
         self._url_data_cache = URLDataCache(f"sigrefdata_{sig_name}")
 
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
-        self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}")
+        self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         super().stop()
-        self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.sig_name} SIG ref data worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/sigrefdata/gma.py

@@ -1,6 +1,7 @@
 import csv
 from time import sleep
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -26,7 +27,7 @@ class GMA(FileDownloadSIGRefDataProvider):
                     sig=self.SIG,
                     id=ref_id,
                     name=row.get("Name", None),
-                    ref_type="Summit",
+                    ref_type=SIGRefType.SUMMIT,
                     url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
                     latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
                     longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,

providers/sigrefdata/illw.py

@@ -1,6 +1,7 @@
 import csv
 from time import sleep
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -27,7 +28,7 @@ class ILLW(FileDownloadSIGRefDataProvider):
                         sig=self.SIG,
                         id=ref_id,
                         name=row.get("Name", None),
-                        ref_type="Lighthouse",
+                        ref_type=SIGRefType.LIGHTHOUSE,
                         url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
                         latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
                         longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,

providers/sigrefdata/iota.py

@@ -3,6 +3,7 @@ from time import sleep
 
 from pyhamtools.locator import latlong_to_locator
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -44,7 +45,7 @@ class IOTA(FileDownloadSIGRefDataProvider):
                         sig=self.SIG,
                         id=ref_id,
                         name=ref["name"],
-                        ref_type="Island",
+                        ref_type=SIGRefType.ISLAND,
                         grid=grid,
                         latitude=latitude,
                         longitude=longitude,

providers/sigrefdata/llota.py

@@ -2,6 +2,7 @@ from time import sleep
 
 from pyhamtools.locator import locator_to_latlong
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -32,7 +33,7 @@ class LLOTA(FileDownloadSIGRefDataProvider):
                         sig=self.SIG,
                         id=ref_id,
                         name=str(ref["name"]),
-                        ref_type="Lake",
+                        ref_type=SIGRefType.LAKE,
                         url=f"https://llota.app/list/ref/{ref_id}",
                         grid=grid,
                         latitude=ll[0],

providers/sigrefdata/mota.py

@@ -1,6 +1,7 @@
 import csv
 from time import sleep
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -26,7 +27,7 @@ class MOTA(FileDownloadSIGRefDataProvider):
                     sig=self.SIG,
                     id=ref_id,
                     name=row.get("Name", None),
-                    ref_type="Mill",
+                    ref_type=SIGRefType.MILL,
                     url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
                     latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
                     longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,

providers/sigrefdata/pga.py

@@ -0,0 +1,58 @@
+from time import sleep
+
+from bs4 import BeautifulSoup
+
+from core.enums import SIGRefType
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class PGA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Polish Gmina Award"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "PGA"
+    DATA_URL = "http://www.spga.pl/lista_pga2.php"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        soup = BeautifulSoup(http_response.text, "html.parser")
+
+        # Iterate through tables in the page
+        for table in soup.find_all("table"):
+            header_cells = table.find_all(["th", "td"], limit=10)
+            header_texts = [c.get_text(strip=True) for c in header_cells]
+
+            # If it has "PGA" and "Nazwa" in the header, it's the main data table
+            if any("PGA" in t for t in header_texts) and any("Nazwa" in t for t in header_texts):
+                # Iterate through all rows except the first
+                rows = table.find_all("tr")
+                for row in rows[1:]:
+                    cells = row.find_all("td")
+                    ref_id = cells[0].get_text(strip=True)
+                    name = cells[1].get_text(strip=True)
+                    if not ref_id:
+                        continue
+
+                    new_data.append(
+                        SIGRef(
+                            sig=self.SIG,
+                            id=ref_id,
+                            name=name,
+                            ref_type=SIGRefType.REGION,
+                        )
+                    )
+
+                    # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
+                    # of the data in this case
+                    if self._stop_event.is_set():
+                        break
+
+                    # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+                    # is available for other threads e.g. the web server.
+                    sleep(0.001)
+
+        return new_data

providers/sigrefdata/pnp_kml_sig_ref_data_provider.py

@@ -4,6 +4,7 @@ from time import sleep
 from fastkml import kml
 from pyhamtools.locator import latlong_to_locator
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -43,7 +44,7 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
                         sig=self.sig_name,
                         id=ref_id,
                         name=placemark.name,
-                        ref_type="Park",
+                        ref_type=SIGRefType.PARK,
                         url=f"https://parksnpeaks.org/getPark.php?actPark={ref_id}",
                         latitude=latitude,
                         longitude=longitude,

providers/sigrefdata/pota.py

@@ -1,6 +1,7 @@
 import csv
 from time import sleep
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -26,7 +27,7 @@ class POTA(FileDownloadSIGRefDataProvider):
                     sig=self.SIG,
                     id=ref_id,
                     name=row.get("name", None),
-                    ref_type="Park",
+                    ref_type=SIGRefType.PARK,
                     url=f"https://pota.app/#/park/{ref_id}",
                     grid=row.get("grid", None),
                     latitude=float(row["latitude"]) if "latitude" in row and row["latitude"] != "" else None,

providers/sigrefdata/sig_ref_data_provider.py

@@ -1,5 +1,6 @@
 import logging
 from datetime import datetime
+from threading import Event
 
 import pytz
 
@@ -19,7 +20,7 @@ class SIGRefDataProvider:
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
         self.reference_count = 0
-        self._stop = False
+        self._stop_event = Event()
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -30,7 +31,7 @@ class SIGRefDataProvider:
         """Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
         super()."""
 
-        self._stop = True
+        self._stop_event.set()
 
     def _add_data(self, new_data):
         """Add all the provided reference data objects to the data store."""
@@ -43,7 +44,7 @@ class SIGRefDataProvider:
                 # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
                 # within the first few minutes of startup, we need a way to abort this expensive process of filling up the
                 # disk cache.
-                if self._stop:
+                if self._stop_event.is_set():
                     break
 
         self.reference_count = len(new_data)

providers/sigrefdata/siota.py

@@ -1,6 +1,7 @@
 import csv
 from time import sleep
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -26,7 +27,7 @@ class SIOTA(FileDownloadSIGRefDataProvider):
                     sig=self.SIG,
                     id=ref_id,
                     name=row.get("NAME", None),
-                    ref_type="Silo",
+                    ref_type=SIGRefType.SILO,
                     grid=row.get("LOCATOR", None),
                     latitude=float(row["LAT"]) if "LAT" in row else None,
                     longitude=float(row["LNG"]) if "LNG" in row else None,

providers/sigrefdata/sota.py

@@ -3,6 +3,7 @@ from time import sleep
 
 from pyhamtools.locator import latlong_to_locator
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -30,7 +31,7 @@ class SOTA(FileDownloadSIGRefDataProvider):
                 sig=self.SIG,
                 id=ref_id,
                 name=row.get("SummitName", None),
-                ref_type="Summit",
+                ref_type=SIGRefType.SUMMIT,
                 url=f"https://www.sotadata.org.uk/en/summit/{ref_id}",
                 latitude=latitude,
                 longitude=longitude,

providers/sigrefdata/toilets.py

@@ -1,5 +1,6 @@
 import csv
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.local_file_sig_ref_data_provider import (
     LocalFileSIGRefDataProvider,
@@ -26,7 +27,7 @@ class Toilets(LocalFileSIGRefDataProvider):
                         sig=self.SIG,
                         id=row["ref"],
                         name=row["ref"],
-                        ref_type="Toilet",
+                        ref_type=SIGRefType.TOILET,
                         latitude=float(row["lat"]),
                         longitude=float(row["lon"]),
                     )
@@ -34,7 +35,7 @@ class Toilets(LocalFileSIGRefDataProvider):
 
                 # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
                 # of the data in this case
-                if self._stop:
+                if self._stop_event.is_set():
                     break
 
         return new_data

providers/sigrefdata/towers.py

@@ -1,6 +1,7 @@
 import csv
 from time import sleep
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -26,7 +27,7 @@ class Towers(FileDownloadSIGRefDataProvider):
                     sig=self.SIG,
                     id=ref_id,
                     name=row.get("Nazev", None),
-                    ref_type="Tower",
+                    ref_type=SIGRefType.TOWER,
                     url=f"https://wwtota.com/seznam/karta_rozhledny.php?ref={ref_id}",
                     grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
                     latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,

providers/sigrefdata/wca.py

@@ -4,6 +4,7 @@ from time import sleep
 
 from pyhamtools.locator import latlong_to_locator
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -45,7 +46,7 @@ class WCA(FileDownloadSIGRefDataProvider):
                     sig=self.SIG,
                     id=ref_id,
                     name=row.get("CLEAN NAME", None),
-                    ref_type="Castle",
+                    ref_type=SIGRefType.CASTLE,
                     url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
                     latitude=latitude,
                     longitude=longitude,

providers/sigrefdata/wota.py

@@ -1,5 +1,6 @@
 from time import sleep
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -33,7 +34,7 @@ class WOTA(FileDownloadSIGRefDataProvider):
                     id=ref_id,
                     name=feature["properties"]["title"],
                     url=url,
-                    ref_type="Summit",
+                    ref_type=SIGRefType.SUMMIT,
                     grid=feature["properties"]["qthLocator"],
                     latitude=feature["geometry"]["coordinates"][1],
                     longitude=feature["geometry"]["coordinates"][0],

providers/sigrefdata/wwbota.py

@@ -1,6 +1,7 @@
 import csv
 from time import sleep
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -26,7 +27,7 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
                     sig=self.SIG,
                     id=ref_id,
                     name=row.get("Name", None),
-                    ref_type="Bunker",
+                    ref_type=SIGRefType.BUNKER,
                     url=f"https://bunkerwiki.org/?s={ref_id}" if ref_id.startswith("B/G") else None,
                     grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
                     latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,

providers/sigrefdata/wwff.py

@@ -1,6 +1,7 @@
 import csv
 from time import sleep
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -26,7 +27,7 @@ class WWFF(FileDownloadSIGRefDataProvider):
                     sig=self.SIG,
                     id=ref_id,
                     name=row.get("name", None),
-                    ref_type="Park",
+                    ref_type=SIGRefType.PARK,
                     url=f"https://wwff.co/directory/?showRef={ref_id}",
                     grid=row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None,
                     latitude=float(row["latitude"])

providers/sigrefdata/zlota.py

@@ -2,6 +2,7 @@ from time import sleep
 
 from pyhamtools.locator import latlong_to_locator
 
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import (
     FileDownloadSIGRefDataProvider,
@@ -26,12 +27,16 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
                 ref_id = ref["code"]
                 latitude = ref["latitude"]
                 longitude = ref["longitude"]
+                try:
+                    ref_type = SIGRefType(ref["asset_type"].title().upper())
+                except ValueError:
+                    ref_type = None
 
                 new_ref = SIGRef(
                     sig=self.SIG,
                     id=ref_id,
                     name=ref["name"],
-                    ref_type=ref["asset_type"].title(),
+                    ref_type=ref_type,
                     url=f"https://ontheair.nz/assets/{ref_id.replace('/', '_')}",
                     latitude=latitude,
                     longitude=longitude,

providers/solarconditions/giroionosonde.py

@@ -67,11 +67,15 @@ class GIROIonosonde(SolarConditionsProvider):
 
     def start(self):
         logger.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
-        self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider")
+        self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning("GIRO ionosonde worker thread did not exit on time and will be killed.")
 
     def _run(self):
         # Real interval at which we poll is the "once per hour" divided by the number of stations, so each one gets

providers/solarconditions/http_solar_conditions_provider.py

@@ -25,11 +25,15 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
 
     def start(self):
         logger.info(f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
-        self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} solar conditions worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/solarconditions/kc2gprop.py

@@ -32,11 +32,15 @@ class KC2GProp(SolarConditionsProvider):
 
     def start(self):
         logger.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
-        self._thread = Thread(target=self._run, name="KC2GPropProvider")
+        self._thread = Thread(target=self._run, name="KC2GPropProvider", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning("KC2G ionosonde worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

providers/spot/aprsis.py

@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Thread
+from threading import Event, Thread
 
 import aprslib
 import pytz
@@ -17,27 +17,44 @@ class APRSIS(SpotProvider):
 
     def __init__(self, provider_config):
         super().__init__("APRS-IS", provider_config)
-        self._thread = Thread(target=self._connect, name="APRSISSpotProvider")
+        self._thread = None
-        self._thread.daemon = True
         self._aprsis = None
+        self._stop_event = Event()
 
     def start(self):
+        self._thread = Thread(target=self._run, name="APRSISSpotProvider", daemon=True)
         self._thread.start()
 
-    def _connect(self):
+    def _run(self):
+        while not self._stop_event.is_set():
+            try:
                 self._aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN)
                 self.status = "Connecting"
                 logger.info("APRS-IS connecting...")
                 self._aprsis.connect()
-        self._aprsis.consumer(self._handle)
                 logger.info("APRS-IS connected.")
+                self._aprsis.consumer(self._handle, immortal=True)
+
+            except Exception:
+                if not self._stop_event.is_set():
+                    self.status = "Error"
+                    logger.exception("Exception in APRS-IS provider")
+
+            if not self._stop_event.is_set():
+                self._stop_event.wait(timeout=5)
 
     def stop(self):
         self.status = "Shutting down"
+        self._stop_event.set()
+        if self._aprsis:
             self._aprsis.close()
-        self._thread.join()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("APRS-IS worker thread did not exit on time and will be killed.")
 
     def _handle(self, data):
+        try:
             # Split SSID in "from" call and store separately
             from_parts = str(data["from"]).split("-")
             dx_call = from_parts[0].upper()
@@ -63,3 +80,6 @@ class APRSIS(SpotProvider):
             self.status = "OK"
             self.last_update_time = datetime.now(pytz.UTC)
             logger.debug("Data received from APRS-IS.")
+
+        except Exception:
+            logger.exception("Exception handling APRS-IS packet")

providers/spot/dxcluster.py

@@ -1,8 +1,7 @@
 import logging
 import re
 from datetime import datetime
-from threading import Thread
+from threading import Event, Thread
-from time import sleep
 
 import pytz
 import telnetlib3
@@ -41,23 +40,26 @@ class DXCluster(SpotProvider):
             self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
         )
         self._telnet = None
-        self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}")
+        self._thread = None
-        self._thread.daemon = True
+        self._stop_event = Event()
-        self._running = True
 
     def start(self):
+        self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
-        self._running = False
+        self._stop_event.set()
         if self._telnet:
             self._telnet.close()
-        self._thread.join()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning(f"DX Cluster {self._hostname} worker thread did not exit on time and will be killed.")
 
     def _handle(self):
-        while self._running:
+        while not self._stop_event.is_set():
             connected = False
-            while not connected and self._running:
+            while not connected and not self._stop_event.is_set():
                 try:
                     self.status = "Connecting"
                     logger.info(f"DX Cluster {self._hostname} connecting...")
@@ -69,14 +71,14 @@ class DXCluster(SpotProvider):
                 except ConnectionRefusedError:
                     self.status = "Error"
                     logger.warning(f"Connection refused to DX cluster {self._hostname}")
-                    sleep(300)
+                    self._stop_event.wait(timeout=300)
                 except Exception:
                     self.status = "Error"
                     logger.exception(f"Exception while connecting to DX Cluster Provider ({self._hostname}).")
-                    sleep(5)
+                    self._stop_event.wait(timeout=5)
 
             self.status = "Waiting for Data"
-            while connected and self._running:
+            while connected and not self._stop_event.is_set():
                 try:
                     # Check new telnet info against regular expression
                     telnet_output = self._telnet.read_until("\n".encode("latin-1"))
@@ -106,19 +108,19 @@ class DXCluster(SpotProvider):
 
                 except EOFError:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Restarting"
                         logger.warning(f"Disconnected from DX Cluster {self._hostname}. Reconnecting...")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"DX Cluster {self._hostname} shutting down...")
                         self.status = "Shutting down"
                 except Exception:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Error"
                         logger.exception(f"Exception in DX Cluster Provider ({self._hostname})")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"DX Cluster {self._hostname} shutting down...")
                         self.status = "Shutting down"

providers/spot/gma.py

@@ -4,6 +4,7 @@ from datetime import datetime
 import pytz
 
 from core.constants import HTTP_HEADERS
+from core.enums import Mode, SIGRefType
 from core.url_data_cache import URLDataCache
 from data.sig_ref import SIGRef
 from data.spot import Spot
@@ -65,7 +66,7 @@ class GMA(HTTPSpotProvider):
                     if (source_spot["QRG"] != "" and source_spot["QRG"] != "QRT")
                     else None,
                     # Filter out some weird mode strings
-                    mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
+                    mode=Mode.from_name(source_spot["MODE"].upper()) if "<>" not in source_spot["MODE"] else None,
                     comment=source_spot["TEXT"],
                     sig_refs=[
                         SIGRef(
@@ -113,24 +114,31 @@ class GMA(HTTPSpotProvider):
                                 match ref_info["reftype"]:
                                     case "Summit":
                                         spot.sig_refs[0].sig = "GMA"
+                                        spot.sig_refs[0].ref_type = SIGRefType.SUMMIT
                                         spot.sig = "GMA"
                                     case "IOTA Island":
                                         spot.sig_refs[0].sig = "IOTA"
+                                        spot.sig_refs[0].ref_type = SIGRefType.ISLAND
                                         spot.sig = "IOTA"
                                     case "GMA Island":
                                         spot.sig_refs[0].sig = "GMA Islands"
+                                        spot.sig_refs[0].ref_type = SIGRefType.ISLAND
                                         spot.sig = "GMA Islands"
                                     case "Lighthouse (ILLW)":
                                         spot.sig_refs[0].sig = "ILLW"
+                                        spot.sig_refs[0].ref_type = SIGRefType.LIGHTHOUSE
                                         spot.sig = "ILLW"
                                     case "Lighthouse (ARLHS)":
                                         spot.sig_refs[0].sig = "ARLHS"
+                                        spot.sig_refs[0].ref_type = SIGRefType.LIGHTHOUSE
                                         spot.sig = "ARLHS"
                                     case "Castle":
                                         spot.sig_refs[0].sig = "WCA"
+                                        spot.sig_refs[0].ref_type = SIGRefType.CASTLE
                                         spot.sig = "WCA"
                                     case "Mill":
                                         spot.sig_refs[0].sig = "MOTA"
+                                        spot.sig_refs[0].ref_type = SIGRefType.MILL
                                         spot.sig = "MOTA"
                                     case _:
                                         logger.warning(

providers/spot/hema.py

@@ -7,6 +7,7 @@ import requests
 from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
 from core.constants import HTTP_HEADERS
+from core.enums import Mode, SIGRefType
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -59,7 +60,7 @@ class HEMA(HTTPSpotProvider):
                             dx_call=spot_items[2].upper(),
                             de_call=spotter_comment_match.group(1).upper(),
                             freq=float(freq_mode_match.group(1)) * 1000000,
-                            mode=freq_mode_match.group(2).upper(),
+                            mode=Mode.from_name(freq_mode_match.group(2).upper()),
                             comment=spotter_comment_match.group(2),
                             sig="HEMA",
                             sig_refs=[
@@ -69,6 +70,7 @@ class HEMA(HTTPSpotProvider):
                                     name=spot_items[4],
                                     latitude=float(spot_items[7]),
                                     longitude=float(spot_items[8]),
+                                    ref_type=SIGRefType.SUMMIT,
                                 )
                             ],
                             time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M")

providers/spot/http_spot_provider.py

@@ -28,12 +28,16 @@ class HTTPSpotProvider(SpotProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} spot API every {self._poll_interval!s} seconds.")
-        self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
         self._wakeup_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} spot worker thread did not exit on time and will be killed.")
 
     def force_poll(self):
         """Trigger an immediate poll without waiting for the normal interval."""

providers/spot/llota.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+from core.enums import Mode, SIGRefType
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -31,7 +32,7 @@ class LLOTA(HTTPSpotProvider):
                 dx_call=source_spot["callsign"].upper(),
                 de_call=spotter.upper() if spotter else None,
                 freq=float(source_spot["frequency"]) * 1000000,
-                mode=source_spot["mode"].upper(),
+                mode=Mode.from_name(source_spot["mode"].upper()),
                 comment=comment,
                 sig="LLOTA",
                 sig_refs=[
@@ -39,6 +40,7 @@ class LLOTA(HTTPSpotProvider):
                         id=source_spot["reference"],
                         sig="LLOTA",
                         name=source_spot["reference_name"],
+                        ref_type=SIGRefType.LAKE,
                     )
                 ],
                 time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp(),

providers/spot/parksnpeaks.py

@@ -6,6 +6,7 @@ import pytz
 import requests
 
 from core.constants import HTTP_HEADERS
+from core.enums import Mode
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -19,7 +20,6 @@ class ParksNPeaks(HTTPSpotProvider):
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
     SUBMIT_URL = "https://www.parksnpeaks.org/api/SPOT/"
-    SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
     SUBMITTABLE_SIGS = [
         "POTA",
         "SOTA",
@@ -52,7 +52,7 @@ class ParksNPeaks(HTTPSpotProvider):
                     if (source_spot["actFreq"] != "")
                     else None,
                     # Seen PNP spots with empty frequency, and with comma-separated thousands digits
-                    mode=source_spot["actMode"].upper(),
+                    mode=Mode.from_name(source_spot["actMode"].upper()),
                     comment=source_spot["actComments"],
                     time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S")
                     .replace(tzinfo=pytz.UTC)
@@ -87,6 +87,7 @@ class ParksNPeaks(HTTPSpotProvider):
                         "POTA",
                         "SOTA",
                         "WWFF",
+                        "HEMA",
                         "SIOTA",
                         "ZLOTA",
                         "KRMNPA",

providers/spot/pota.py

@@ -4,6 +4,7 @@ import pytz
 import requests
 
 from core.constants import HTTP_HEADERS
+from core.enums import Mode, SIGRefType
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -29,8 +30,8 @@ class POTA(HTTPSpotProvider):
                 source_id=source_spot["spotId"],
                 dx_call=source_spot["activator"].upper(),
                 de_call=source_spot["spotter"].upper(),
-                freq=float(source_spot["frequency"]) * 1000,
+                freq=float(source_spot["frequency"]) * 1000 if source_spot["frequency"] != "INVALID" else None,
-                mode=source_spot["mode"].upper(),
+                mode=Mode.from_name(source_spot["mode"].upper()),
                 comment=source_spot["comments"],
                 sig="POTA",
                 sig_refs=[
@@ -40,6 +41,7 @@ class POTA(HTTPSpotProvider):
                         name=source_spot["name"],
                         latitude=source_spot["latitude"],
                         longitude=source_spot["longitude"],
+                        ref_type=SIGRefType.PARK
                     )
                 ],
                 time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S")

providers/spot/rbn.py

@@ -1,8 +1,7 @@
 import logging
 import re
 from datetime import datetime
-from threading import Thread
+from threading import Event, Thread
-from time import sleep
 
 import pytz
 import telnetlib3
@@ -30,23 +29,26 @@ class RBN(SpotProvider):
         super().__init__(name, provider_config)
         self._port = provider_config["port"]
         self._telnet = None
-        self._thread = Thread(target=self._handle, name=f"RBNSpotProvider-{self.name}")
+        self._thread = None
-        self._thread.daemon = True
+        self._stop_event = Event()
-        self._running = True
 
     def start(self):
+        self._thread = Thread(target=self._handle, name=f"RBNSpotProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
-        self._running = False
+        self._stop_event.set()
         if self._telnet:
             self._telnet.close()
-        self._thread.join()
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning(f"RBN (port {self._port!s}) worker thread did not exit on time and will be killed.")
 
     def _handle(self):
-        while self._running:
+        while not self._stop_event.is_set():
             connected = False
-            while not connected and self._running:
+            while not connected and not self._stop_event.is_set():
                 try:
                     self.status = "Connecting"
                     logger.info(f"RBN port {self._port!s} connecting...")
@@ -58,10 +60,10 @@ class RBN(SpotProvider):
                 except Exception:
                     self.status = "Error"
                     logger.exception(f"Exception while connecting to RBN (port {self._port!s}).")
-                    sleep(5)
+                    self._stop_event.wait(timeout=5)
 
             self.status = "Waiting for Data"
-            while connected and self._running:
+            while connected and not self._stop_event.is_set():
                 try:
                     # Check new telnet info against regular expression
                     telnet_output = self._telnet.read_until("\n".encode("latin-1"))
@@ -91,19 +93,19 @@ class RBN(SpotProvider):
 
                 except EOFError:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Restarting"
                         logger.warning(f"Disconnected from RBN provider (port {self._port!s}). Reconnecting...")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"RBN provider (port {self._port!s}) shutting down...")
                         self.status = "Shutting down"
                 except Exception:
                     connected = False
-                    if self._running:
+                    if not self._stop_event.is_set():
                         self.status = "Error"
                         logger.exception(f"Exception in RBN provider (port {self._port!s})")
-                        sleep(5)
+                        self._stop_event.wait(timeout=5)
                     else:
                         logger.info(f"RBN provider (port {self._port!s}) shutting down...")
                         self.status = "Shutting down"

providers/spot/sota.py

@@ -4,7 +4,8 @@ from datetime import datetime
 import requests
 from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
-from core.constants import DV_SUB_MODES, HTTP_HEADERS, SSB_SUB_MODES
+from core.constants import HTTP_HEADERS
+from core.enums import Mode, SIGRefType
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -53,7 +54,7 @@ class SOTA(HTTPSpotProvider):
                         if (source_spot["frequency"] is not None)
                         else None,
                         # Seen SOTA spots with no frequency!
-                        mode=source_spot["mode"].upper(),
+                        mode=Mode.from_name(source_spot["mode"].upper()),
                         comment=source_spot["comments"],
                         sig="SOTA",
                         sig_refs=[
@@ -64,6 +65,7 @@ class SOTA(HTTPSpotProvider):
                                 latitude=source_spot["latitude"],
                                 longitude=source_spot["longitude"],
                                 activation_score=source_spot["points"],
+                                ref_type=SIGRefType.SUMMIT,
                             )
                         ],
                         dx_latitude=source_spot["latitude"],
@@ -98,11 +100,6 @@ class SOTA(HTTPSpotProvider):
             # https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js
             mode = spot.mode
             if mode and mode not in self.VALID_MODES:
-                if mode in SSB_SUB_MODES:
-                    mode = "SSB"
-                elif mode in DV_SUB_MODES:
-                    mode = "DV"
-                else:
                 mode = "Data"
 
             body = {

providers/spot/sse_spot_provider.py

@@ -65,7 +65,7 @@ class SSESpotProvider(SpotProvider):
                     self._url,
                     headers=HTTP_HEADERS,
                     latest_event_id=self._last_event_id,
-                    timeout=10,
+                    timeout=30,
                     on_open=self._on_open,
                     on_error=self._on_error,
                 ) as event_source:

providers/spot/tiles.py

@@ -3,7 +3,8 @@ from datetime import datetime
 
 import requests
 
-from core.constants import HTTP_HEADERS, SSB_SUB_MODES
+from core.constants import HTTP_HEADERS
+from core.enums import LocationSourceForSpot, Mode, SIGRefType
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -55,7 +56,7 @@ class Tiles(HTTPSpotProvider):
                 # No separate spotter callsign, assume all spots are self-spots
                 de_call=source_spot["call_sign"].upper(),
                 freq=freq,
-                mode=source_spot["mode"].upper(),
+                mode=Mode.from_name(source_spot["mode"].upper()),
                 comment=source_spot["notes"],
                 sig="Tiles",
                 # Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
@@ -67,13 +68,14 @@ class Tiles(HTTPSpotProvider):
                         name=source_spot["maidenhead_grid"],
                         latitude=source_spot["latitude"],
                         longitude=source_spot["longitude"],
+                        ref_type=SIGRefType.GRID
                     )
                 ],
                 time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
                 dx_grid=source_spot["maidenhead_grid"],
                 dx_latitude=source_spot["latitude"],
                 dx_longitude=source_spot["longitude"],
-                dx_location_source="GRID",
+                dx_location_source=LocationSourceForSpot.GRID,
             )
 
             # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
@@ -92,9 +94,7 @@ class Tiles(HTTPSpotProvider):
             if spot.mode:
                 mode = spot.mode
                 if mode not in self.VALID_MODES:
-                    if mode in SSB_SUB_MODES:
+                    if mode == "OLIVIA":
-                        mode = "SSB"
-                    elif mode == "OLIVIA":
                         mode = "Olivia"
                     elif mode == "JS8":
                         mode = "JS8Call"

providers/spot/towers.py

@@ -1,6 +1,9 @@
 import json
 from datetime import datetime
 
+import pytz
+
+from core.enums import SIGRefType
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -32,10 +35,10 @@ class Towers(HTTPSpotProvider):
                 freq=likely_freq,
                 comment=source_spot["comment"],
                 sig="Towers",
-                sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
+                sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers", ref_type=SIGRefType.TOWER)],
-                time=datetime.strptime(
+                time=datetime.strptime(response_json["updated"][:10] + source_spot["time"], "%Y-%m-%d%H:%M")
-                    response_json["updated"][:10] + source_spot["time"], "%Y-%m-%d%H:%M"
+                .replace(tzinfo=pytz.utc)
-                ).timestamp(),
+                .timestamp(),
             )
 
             # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do

providers/spot/ukpacketnet.py

@@ -3,6 +3,7 @@ from datetime import datetime
 
 import pytz
 
+from core.enums import Mode
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
 
@@ -77,7 +78,7 @@ class UKPacketNet(HTTPSpotProvider):
                                     dx_call=heard["callsign"].upper(),
                                     de_call=node["callsign"].upper(),
                                     freq=freq,
-                                    mode="PKT",
+                                    mode=Mode.PKT,
                                     comment=comment,
                                     time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S")
                                     .replace(tzinfo=pytz.UTC)

providers/spot/websocket_spot_provider.py

@@ -1,7 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Thread
+from threading import Event, Thread
-from time import sleep
 
 import pytz
 from websocket import create_connection
@@ -20,22 +19,24 @@ class WebsocketSpotProvider(SpotProvider):
         self._url = url
         self._ws = None
         self._thread = None
-        self._stopped = False
+        self._stop_event = Event()
         self._last_event_id = None
 
     def start(self):
         logger.info(f"Set up websocket connection to {self.name} spot API.")
-        self._stopped = False
+        self._stop_event.clear()
         self._thread = Thread(target=self._run, name=f"WebsocketSpotProvider-{self.name}")
         self._thread.daemon = True
         self._thread.start()
 
     def stop(self):
-        self._stopped = True
+        self._stop_event.set()
         if self._ws:
             self._ws.close()
         if self._thread:
-            self._thread.join()
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} websocket worker thread did not exit on time and will be killed.")
 
     def _on_open(self):
         self.status = "Waiting for Data"
@@ -44,14 +45,19 @@ class WebsocketSpotProvider(SpotProvider):
         self.status = "Connecting"
 
     def _run(self):
-        while not self._stopped:
+        while not self._stop_event.is_set():
             try:
                 logger.debug(f"Connecting to {self.name} spot API...")
                 self.status = "Connecting"
                 self._ws = create_connection(self._url, header=HTTP_HEADERS)
                 self.status = "Connected"
+
+                # Keep reading from this same connection until it drops or we're asked to stop, rather than
+                # reconnecting for every message.
+                while not self._stop_event.is_set():
                     data = self._ws.recv()
-                if data:
+                    if not data:
+                        break
                     try:
                         new_spot = self._ws_message_to_spot(data)
                         if new_spot:
@@ -69,7 +75,16 @@ class WebsocketSpotProvider(SpotProvider):
                 logger.exception(f"Exception in Websocket Spot Provider ({self.name})")
             else:
                 self.status = "Disconnected"
-            sleep(5)  # Wait before trying to reconnect
+            finally:
+                if self._ws:
+                    try:
+                        self._ws.close()
+                    except Exception:
+                        # No problem, we were getting rid of this object anyway.
+                        pass
+                    self._ws = None
+            if not self._stop_event.is_set():
+                self._stop_event.wait(timeout=5)  # Wait before trying to reconnect
 
     def _ws_message_to_spot(self, b):
         """Convert a WS message received from the API into a spot. The exact message data (in bytes) is provided here so the

providers/spot/wota.py

@@ -8,6 +8,7 @@ import pytz
 from rss_parser import Parser
 from rss_parser.models.rss import RSS
 
+from core.enums import Mode, SIGRefType
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -76,7 +77,9 @@ class WOTA(HTTPSpotProvider):
                     except Exception:
                         logger.warning(f"Could not parse WOTA spot description: {source_spot.description}")
 
-                    time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+                    time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(
+                        pytz.UTC
+                    )
 
                     # Convert to our spot format
                     spot = Spot(
@@ -85,10 +88,10 @@ class WOTA(HTTPSpotProvider):
                         dx_call=dx_call,
                         de_call=spotter,
                         freq=freq_hz,
-                        mode=mode,
+                        mode=Mode.from_name(mode),
                         comment=comment,
                         sig="WOTA",
-                        sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
+                        sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name, ref_type=SIGRefType.SUMMIT)] if ref else [],
                         time=time.timestamp(),
                     )
 

providers/spot/wwbota.py

@@ -1,6 +1,7 @@
 import json
 from datetime import datetime
 
+from core.enums import Mode, SIGRefType
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.sse_spot_provider import SSESpotProvider
@@ -26,6 +27,7 @@ class WWBOTA(SSESpotProvider):
                 name=ref["name"],
                 latitude=ref["lat"],
                 longitude=ref["long"],
+                ref_type=SIGRefType.BUNKER,
             )
             refs.append(sigref)
 
@@ -34,7 +36,7 @@ class WWBOTA(SSESpotProvider):
             dx_call=source_spot["call"].upper(),
             de_call=source_spot["spotter"].upper(),
             freq=float(source_spot["freq"]) * 1000000,
-            mode=source_spot["mode"].upper() if "mode" in source_spot else None,
+            mode=Mode.from_name(source_spot["mode"].upper()) if "mode" in source_spot else None,
             comment=source_spot["comment"],
             sig="WWBOTA",
             sig_refs=refs,

providers/spot/wwff.py

@@ -2,6 +2,7 @@ from datetime import datetime
 
 import pytz
 
+from core.enums import Mode, SIGRefType
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -27,7 +28,7 @@ class WWFF(HTTPSpotProvider):
                 dx_call=source_spot["activator"].upper(),
                 de_call=source_spot["spotter"].upper(),
                 freq=float(source_spot["frequency_khz"]) * 1000,
-                mode=source_spot["mode"].upper(),
+                mode=Mode.from_name(source_spot["mode"].upper()),
                 comment=source_spot["remarks"],
                 sig="WWFF",
                 sig_refs=[
@@ -37,6 +38,7 @@ class WWFF(HTTPSpotProvider):
                         name=source_spot["reference_name"],
                         latitude=source_spot["latitude"],
                         longitude=source_spot["longitude"],
+                        ref_type=SIGRefType.PARK
                     )
                 ],
                 time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),

providers/spot/xota.py

@@ -3,6 +3,7 @@ from datetime import datetime
 
 import pytz
 
+from core.enums import Mode
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.websocket_spot_provider import WebsocketSpotProvider
@@ -34,7 +35,7 @@ class XOTA(WebsocketSpotProvider):
             source_id=source_spot["id"],
             dx_call=source_spot["stationCallSign"].upper(),
             freq=float(source_spot["freq"]) * 1000,
-            mode=source_spot["mode"].upper(),
+            mode=Mode.from_name(source_spot["mode"].upper()),
             sig=self.SIG,
             sig_refs=[
                 SIGRef(

providers/spot/zlota.py

@@ -2,6 +2,7 @@ from datetime import datetime
 
 import pytz
 
+from core.enums import Mode
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -32,7 +33,7 @@ class ZLOTA(HTTPSpotProvider):
                 dx_call=source_spot["activator"].upper(),
                 de_call=source_spot["spotter"].upper(),
                 freq=freq_hz,
-                mode=source_spot["mode"].upper().strip(),
+                mode=Mode.from_name(source_spot["mode"].upper().strip()),
                 comment=source_spot["comments"],
                 sig="ZLOTA",
                 sig_refs=[

providers/staticdata/file_download_static_data_provider.py

@@ -29,11 +29,15 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logger.info(f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.")
-        self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}")
+        self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}", daemon=True)
         self._thread.start()
 
     def stop(self):
         self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=35)
+            if self._thread.is_alive():
+                logger.warning(f"{self.name} static data worker thread did not exit on time and will be killed.")
 
     def _run(self):
         while True:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "spothole"
-version = "2.0.5"
+version = "2.1"
 authors = [
     { name = "Ian Renton", email = "ian@ianrenton.com" },
 ]

requirements.txt

@@ -1,6 +1,6 @@
 pyyaml~=6.0.3
 requests-cache~=1.2.1
-pyhamtools~=0.12.0
+pyhamtools~=0.13.2
 telnetlib3~=2.0.8
 pytz~=2025.2
 requests~=2.32.4
@@ -16,8 +16,13 @@ beautifulsoup4~=4.14.2
 websocket-client~=1.8.0
 tornado~=6.4.2
 tornado_eventsource~=3.0.0
+pandas~=3.0.0
 geopandas~=0.13.2
 simplejson~=4.1.1
 cachetools~=7.1.6
 fastkml~=1.4.0
 ruff~=0.16.3
+pdfplumber~=0.11.10
+xlrd~=2.0.2
+openpyxl~=3.1.5
+icalendar~=7.3.0

spothole.py

@@ -5,25 +5,35 @@ import signal
 import sys
 
 from core.cleanup import CLEANUP_TIMER
-from core.config import LOG_LEVEL, SERVER_OWNER_CALLSIGN
+from core.config import LOG_LEVEL, SERVER_OWNER_CALLSIGN, TELNET_SERVER_ENABLED, TELNET_SERVER_PORT
 from core.constants import SOFTWARE_VERSION
 from core.data_providers import DATA_PROVIDERS
 from core.data_store import DATA_STORE
 from core.status_reporter import StatusReporter
-from server.webserver import WEB_SERVER
+from telnetserver.telnetserver import TELNET_SERVER
+from webserver.webserver import WEB_SERVER
 
 logger = logging.getLogger(__name__)
 
+_shutdown_in_progress = False
+
 
 def shutdown(_signum=None, _frame=None):
     """Shutdown function"""
 
+    # Check if this is the second time a shutdown was asked for, if so immediately kill the program.
+    global _shutdown_in_progress
+    if _shutdown_in_progress:
+        os._exit(1)
+    _shutdown_in_progress = True
+
     logger.info("Stopping program...")
     WEB_SERVER.stop()
+    TELNET_SERVER.stop()
     DATA_PROVIDERS.stop()
     CLEANUP_TIMER.stop()
     DATA_STORE.close()
-    os._exit(0)
+    logger.info("Stopped.")
 
 
 # Main function
@@ -41,8 +51,9 @@ if __name__ == "__main__":
     logger.info("Starting...")
     logger.info(f"This is Spothole version {SOFTWARE_VERSION}. This instance is run by {SERVER_OWNER_CALLSIGN}.")
 
-    # Shut down gracefully on SIGINT
+    # Shut down gracefully on SIGINT or SIGTERM
     signal.signal(signal.SIGINT, shutdown)
+    signal.signal(signal.SIGTERM, shutdown)
 
     # Set up data store
     DATA_STORE.setup()
@@ -57,10 +68,16 @@ if __name__ == "__main__":
     status_reporter = StatusReporter(run_interval=5)
     status_reporter.start()
 
+    # Run the telnet server
+    if TELNET_SERVER_ENABLED:
+        TELNET_SERVER.start(port=TELNET_SERVER_PORT)
+
     # Set up the web server
     WEB_SERVER.setup()
 
-    # Run the web server. This is the blocking call that keeps the application running in the main thread, so this must
+    # Run the web server
-    # be the last thing we do. web_server.stop() triggers an await condition in the web server which finishes the main
-    # thread.
     WEB_SERVER.start()
+
+    # Block the main thread until a termination signal arrives and shutdown() is running.
+    while not _shutdown_in_progress:
+        signal.pause()

static/apidocs/openapi.yml

@@ -15,6 +15,19 @@ info:
 
     ## Changelog
     
+    ### 2.1
+    
+    * Added AMSAT, EME, DTMBA, FEA, BIWOTA, COTA & PGA SIGs
+    * Removed the distinction between LSB & USB (both will now show as SSB) and between the various digital voice modes, which will now show as DV.
+    * Added `sig_type`, `icon`, `region_flag` and `refs_globally_unique` to SIG data
+    * Added `icon` to spot and alert data
+    * Added `alert_type` and `url` to alert data
+    * Added `contests_skip_max_duration_check` to alert query parameters
+    * SIG reference types (e.g. "Park") are now capitalised to match other enums
+    * Added the ability to get only certain fields of spots and alerts from the API by using the `fields` query parameter.
+    * Replace `last_page_access` with `page_requests_per_hour` and `last_api_access` with `api_requests_per_hour` in the web server stats.
+    * Added `sse_client_count` to the `webserver` stats, and a new `telnet` object with `client_count`.
+
     ### 2.0
 
     * **Breaking change:** The "add spot" API has changed to enable future support for upstream submission to the spotting services associated with various SIGs. Instead of just posting the spot object itself as the JSON content of the POST, this has moved into a `spot` object within the structure. A new `handling` object alongside it contains the `submit_upstream`, `upstream_provider`, `upstream_credentials`, and `captcha_token` fields which control the server handling of the spot.
@@ -130,6 +143,7 @@ paths:
         - $ref: '#/components/parameters/SpotTextIncludes'
         - $ref: '#/components/parameters/SpotNeedsGoodLocation'
         - $ref: '#/components/parameters/SpotAllowQrt'
+        - $ref: '#/components/parameters/SpotFields'
         - $ref: '#/components/parameters/QrzUsername'
         - $ref: '#/components/parameters/QrzPassword'
         - $ref: '#/components/parameters/QrzSessionKey'
@@ -170,6 +184,7 @@ paths:
         - $ref: '#/components/parameters/SpotTextIncludes'
         - $ref: '#/components/parameters/SpotNeedsGoodLocation'
         - $ref: '#/components/parameters/SpotAllowQrt'
+        - $ref: '#/components/parameters/SpotFields'
         - $ref: '#/components/parameters/QrzUsername'
         - $ref: '#/components/parameters/QrzPassword'
         - $ref: '#/components/parameters/QrzSessionKey'
@@ -201,11 +216,13 @@ paths:
         - $ref: '#/components/parameters/AlertReceivedSince'
         - $ref: '#/components/parameters/AlertMaxDuration'
         - $ref: '#/components/parameters/AlertDxpeditionsSkipMaxDurationCheck'
+        - $ref: '#/components/parameters/AlertContestsSkipMaxDurationCheck'
         - $ref: '#/components/parameters/AlertSource'
         - $ref: '#/components/parameters/AlertSig'
         - $ref: '#/components/parameters/AlertDxContinent'
         - $ref: '#/components/parameters/AlertDxCallIncludes'
         - $ref: '#/components/parameters/AlertTextIncludes'
+        - $ref: '#/components/parameters/AlertFields'
         - $ref: '#/components/parameters/QrzUsername'
         - $ref: '#/components/parameters/QrzPassword'
         - $ref: '#/components/parameters/QrzSessionKey'
@@ -235,11 +252,13 @@ paths:
       parameters:
         - $ref: '#/components/parameters/AlertMaxDuration'
         - $ref: '#/components/parameters/AlertDxpeditionsSkipMaxDurationCheck'
+        - $ref: '#/components/parameters/AlertContestsSkipMaxDurationCheck'
         - $ref: '#/components/parameters/AlertSource'
         - $ref: '#/components/parameters/AlertSig'
         - $ref: '#/components/parameters/AlertDxContinent'
         - $ref: '#/components/parameters/AlertDxCallIncludes'
         - $ref: '#/components/parameters/AlertTextIncludes'
+        - $ref: '#/components/parameters/AlertFields'
         - $ref: '#/components/parameters/QrzUsername'
         - $ref: '#/components/parameters/QrzPassword'
         - $ref: '#/components/parameters/QrzSessionKey'
@@ -645,6 +664,16 @@ components:
       schema:
         type: boolean
         default: true
+    SpotFields:
+      name: fields
+      in: query
+      description: >
+        Filter the fields you receive in each spot, to conserve data bandwidth for constrained applications.
+        Supply a comma-separated list of the fields you want to receive, using the names of the fields returned by the
+        `/spots` call, e.g. `id,dx_call,freq,mode,time`. If the "fields" parameter is not supplied, all fields will be
+        included in the spot data.
+      schema:
+        type: string
     AlertMaxDuration:
       name: max_duration
       in: query
@@ -653,8 +682,8 @@ components:
         time minus start time, if end time is set, otherwise the activation is assumed to be short and
         therefore to always pass this check. This is useful to filter out people who alert POTA
         activations lasting months or even years, but note it will also include multi-day or multi-week
-        DXpeditions that you might otherwise be interested in. See the
+        DXpeditions or contests that you might otherwise be interested in. See the
-        dxpeditions_skip_max_duration_check parameter for the workaround.
+        dxpeditions_skip_max_duration_check and contests_skip_max_duration_check parameters for the workaround.
       schema:
         type: integer
     AlertDxpeditionsSkipMaxDurationCheck:
@@ -667,6 +696,16 @@ components:
         on the air most of the time.
       schema:
         type: boolean
+    AlertContestsSkipMaxDurationCheck:
+      name: contests_skip_max_duration_check
+      in: query
+      description: >
+        Return contest alerts even if they last longer than max_duration. This allows the user to
+        filter out multi-day/multi-week POTA alerts where the operator likely won't be on the air most
+        of the time, but keep multi-day/multi-week contests where contesters likely *will* be
+        on the air most of the time.
+      schema:
+        type: boolean
     AlertSource:
       name: source
       in: query
@@ -773,6 +812,16 @@ components:
         you will get all the more recent alerts back, without duplicating the previous latest spot.
       schema:
         type: number
+    AlertFields:
+      name: fields
+      in: query
+      description: >
+        Filter the fields you receive in each alert, to conserve data bandwidth for constrained applications.
+        Supply a comma-separated list of the fields you want to receive, using the names of the fields returned by the
+        `/alerts` call, e.g. `id,dx_calls,freqs_modes,start_time`. If the "fields" parameter is not supplied, all fields
+        will be included in the alert data.
+      schema:
+        type: string
     CallParam:
       name: call
       in: query
@@ -852,9 +901,27 @@ components:
         - WAB
         - WAI
         - DME
+        - DMF
+        - FEA
+        - DMUE
+        - DMVE
+        - DCE
+        - DEFE
+        - DTMBA
+        - BIWOTA
+        - COTA
+        - PGA
         - Toilets
       example: POTA
 
+    SIGType:
+      type: string
+      enum:
+        - WORLDWIDE
+        - REGIONAL
+        - EVENT
+      example: WORLDWIDE
+
     SIGNameIncludingNoSIG:
       oneOf:
         - $ref: "#/components/schemas/SIGName"
@@ -862,6 +929,36 @@ components:
           enum: [ NO_SIG ]
       example: POTA
 
+    SIGRefType:
+      type: string
+      enum:
+        - PARK
+        - SUMMIT
+        - CASTLE
+        - LIGHTHOUSE
+        - ISLAND
+        - BUNKER
+        - MILL
+        - SILO
+        - BEACH
+        - LAKE
+        - TOWER
+        - WATERWAY
+        - TOWN
+        - BUILDING
+        - REGION
+        - GRID
+        - TOILET
+      example: PARK
+
+    AlertType:
+      type: string
+      enum:
+        - XOTA
+        - DXPEDITION
+        - CONTEST
+      example: XOTA
+
     Continent:
       type: string
       enum:
@@ -911,15 +1008,9 @@ components:
         - CW
         - PHONE
         - SSB
-        - USB
-        - LSB
         - AM
         - FM
         - DV
-        - DMR
-        - DSTAR
-        - C4FM
-        - M17
         - DATA
         - FT8
         - FT4
@@ -976,7 +1067,7 @@ components:
         - NONE
       example: SPOT
 
-    LocationSourceForAlert:
+    LocationSourceForCallsign:
       type: string
       enum:
         - "HOME QTH"
@@ -999,9 +1090,8 @@ components:
           description: SIG reference name
           example: Null Country Park
         ref_type:
-          type: string
           description: SIG reference type
-          example: Park
+          $ref: "#/components/schemas/SIGRefType"
         url:
           type: string
           description: SIG reference URL, which the user can look up for more information
@@ -1241,6 +1331,10 @@ components:
             Propagation mode, if known. This is only populated when the upstream spot specifically states it; Spothole
             does not try to determine it using its own algorithm.
           $ref: "#/components/schemas/PropagationMode"
+        icon:
+          type: string
+          description: Icon to use when displaying this spot in the web UI. Chosen from the Font Awesome set.
+          example: "fa-tower-cell"
 
 
     SpotSubmission:
@@ -1394,6 +1488,13 @@ components:
           items:
             $ref: '#/components/schemas/SIGRef'
           description: SIG references. We allow multiple here for e.g. n-fer activations, unlike ADIF SIG_INFO
+        alert_type:
+          description: "The type of alert this is: xOTA, DXpedition, or Contest."
+          $ref: "#/components/schemas/AlertType"
+        url:
+          type: string
+          description: A URL linking to more information about the alert, e.g. DXpedition or contest info.
+          example: "https://www.rsgbcc.org/cgi-bin/contest_rules.pl?year=2026&contest=144backpack1"
         source:
           type: string
           description: Where we got the alert from.
@@ -1402,6 +1503,10 @@ components:
           type: string
           description: The ID the source gave it, if any.
           example: "GUID-123456"
+        icon:
+          type: string
+          description: Icon to use when displaying this spot in the web UI. Chosen from the Font Awesome set.
+          example: "fa-tower-cell"
 
 
     AlertStream:
@@ -1486,6 +1591,12 @@ components:
           type: string
           description: The full name of the SIG
           example: Parks on the Air
+        sig_type:
+          type: boolean
+          description: >
+            Whether the SIG is worldwide, regional, or for a specific event. Generally for Spothole's own internal use,
+            clients probably won't need this. Used to group them in the web UI.
+          $ref: "#/components/schemas/SIGType"
         comment_names:
           type: array
           description: >
@@ -1502,6 +1613,24 @@ components:
             Regex that matches this SIG's reference IDs. Generally for Spothole's own internal use,
             clients probably won't need this.
           example: "[A-Z]{2}\\-\\d+"
+        refs_globally_unique:
+          type: boolean
+          description: >
+            Identifies that the SIG's reference ID structure defined by its regex is unique across all programmes and
+            anything else we expect a user to put in a spot comment, and therefore we can pull references out of
+            spot comments without also needing to see the SIG name first. For example, "OHFF-1234" or "B/G-1234" are
+            obviously WWFF and WWBOTA, nothing else looks like those. But "SZ09" could be WAB or Tiles, "GB1234" could
+            conceivably be POTA or ILLW, etc. Generally for Spothole's own internal use, clients probably won't need
+            this.
+        icon:
+          type: string
+          description: Icon from the Font Awesome set that represents this SIG, for use in the front end.
+          example: "fa-tree"
+        region_flag:
+          type: string
+          description: >
+            Flag emoji, if this SIG is specific to a country or region. If null, this SIG is treated as worldwide.
+          example: "🇺🇳"
 
     SolarConditions:
       type: object
@@ -1978,14 +2107,25 @@ components:
               type: string
               description: The status of the web server
               example: OK
-            last_page_access:
+            page_requests_per_hour:
-              type: number
+              type: integer
-              description: The last time a page was accessed on the web server, UTC seconds since UNIX epoch.
+              description: The number of page requests handled by the web server in the last hour
-              example: 1759579508
+              example: 123
-            last_api_access:
+            api_requests_per_hour:
-              type: number
+              type: integer
-              description: The last time an API endpoint was accessed on the web server, UTC seconds since UNIX epoch.
+              description: The number of API requests handled by the web server in the last hour
-              example: 1759579508
+              example: 123
+            sse_client_count:
+              type: integer
+              description: The number of clients currently connected to SSE streams
+              example: 5
+        "telnet":
+          type: object
+          properties:
+            client_count:
+              type: integer
+              description: The number of clients currently connected to the telnet server
+              example: 2
         spot_providers:
           type: array
           description: An array of all the spot providers.
@@ -2030,7 +2170,7 @@ components:
           description: An array of all the supported modes.
           items:
             type: string
-            example: "LSB"
+            example: "SSB"
         mode_types:
           type: array
           description: An array of all the supported mode types.
@@ -2174,7 +2314,7 @@ components:
             Where we got the location (grid/latitude/longitude) from. Unlike a spot where we might
             have a summit position or WAB square, here the only options are an online QTH lookup, or
             a location based purely on DXCC, or nothing.
-          $ref: "#/components/schemas/LocationSourceForAlert"
+          $ref: "#/components/schemas/LocationSourceForCallsign"
 
     GridLookup:
       type: object

static/js/alerts.js

@@ -13,7 +13,6 @@ function loadAlerts() {
         url: '/api/v2/alerts' + buildQueryString(), dataType: 'json', success: function (jsonData) {
             // Store last updated time
             lastUpdateTime = moment.utc();
-            updateRefreshDisplay();
             // Store data
             alerts = jsonData;
             // Update table
@@ -38,6 +37,9 @@ function buildQueryString() {
     if ($("#dxpeditions_skip_max_duration_check")[0].checked) {
         str = str + "&dxpeditions_skip_max_duration_check=true";
     }
+    if ($("#contests_skip_max_duration_check")[0].checked) {
+        str = str + "&contests_skip_max_duration_check=true";
+    }
     return str;
 }
 
@@ -52,7 +54,7 @@ function updateTable() {
     const showDX = $("#tableShowDX")[0].checked;
     const showFreqsModes = $("#tableShowFreqsModes")[0].checked;
     const showComment = $("#tableShowComment")[0].checked;
-    const showSource = $("#tableShowSource")[0].checked;
+    const showType = $("#tableShowType")[0].checked;
     const showRef = $("#tableShowRef")[0].checked;
 
     // Populate table with headers
@@ -73,8 +75,8 @@ function updateTable() {
     if (showComment) {
         table.find('thead tr').append(`<th class='bg-primary-subtle hideonmobile'>Comment</th>`);
     }
-    if (showSource) {
+    if (showType) {
-        table.find('thead tr').append(`<th class='bg-primary-subtle hideonmobile'>Source</th>`);
+        table.find('thead tr').append(`<th class='bg-primary-subtle hideonmobile'>Type</th>`);
     }
     if (showRef) {
         table.find('thead tr').append(`<th class='bg-primary-subtle hideonmobile'>Ref.</th>`);
@@ -149,7 +151,7 @@ function addAlertRowsToTable(tbody, alerts) {
         const showDX = $("#tableShowDX")[0].checked;
         const showFreqsModes = $("#tableShowFreqsModes")[0].checked;
         const showComment = $("#tableShowComment")[0].checked;
-        const showSource = $("#tableShowSource")[0].checked;
+        const showType = $("#tableShowType")[0].checked;
         const showRef = $("#tableShowRef")[0].checked;
 
         // Get times for the alert, and convert to local time if necessary.
@@ -208,10 +210,14 @@ function addAlertRowsToTable(tbody, alerts) {
         if (a["dx_calls"] != null) {
             dx_calls_html = a["dx_calls"].map(call => `<a class='dx-link' href='https://qrz.com/db/${call}' target='_new'>${call}</a>`).join(", ");
         }
+        if (dx_calls_html === "" && a["alert_type"] === "CONTEST") {
+            // Contest = true and no DX callsigns, so display "Contest"
+            dx_calls_html = "Contest"
+        }
 
         // Format DXpedition country
         let dx_country_html = "";
-        if (a["is_dxpedition"] === true && a["dx_country"] != null && a["dx_country"] !== "") {
+        if (a["alert_type"] === "DXPEDITION" && a["dx_country"] != null && a["dx_country"] !== "") {
             dx_country_html = `<br/>${a["dx_country"]}`;
         }
 
@@ -227,10 +233,37 @@ function addAlertRowsToTable(tbody, alerts) {
             commentText = escapeHtml(a["comment"]);
         }
 
-        // Sig or fallback to source
+        // Format extra text, like URL and attribution
-        let sigSourceText = a["source"];
+        let subComment = a["url"] != null || a["source"] === "NG3K" || a["source"] === "WA7BNM Contest Calendar";
+        if (subComment) {
+            let subCommentText = ""
+            if (a["url"] != null) {
+                subCommentText += `<a href="${escapeHtml(a['url'])}" target="_new" style="text-decoration: none">More info</a>`;
+            }
+            if ((a["source"] === "NG3K" || a["source"] === "WA7BNM Contest Calendar")) {
+                if (subCommentText !== "") {
+                    subCommentText += " | ";
+                }
+                subCommentText += `From ${a["source"]}`;
+            }
+            commentText += `<div class="mt-2 small text-secondary">${subCommentText}</div>`;
+        }
+
+
+        // Type, SIG or fallback to source
+        let sigTypeText = a["source"];
+        if (a["alert_type"] === "CONTEST") {
+            sigTypeText = "Contest";
+        } else if (a["alert_type"] === "DXPEDITION") {
+            sigTypeText = "DXpedition";
+        } else if (a["alert_type"] === "SATELLITE") {
+            sigTypeText = "Satellite";
+        } else if (a["alert_type"] === "XOTA") {
             if (a["sig"]) {
-            sigSourceText = a["sig"];
+                sigTypeText = a["sig"];
+            } else {
+                sigTypeText = "xOTA";
+            }
         }
 
         // Format sig_refs
@@ -263,8 +296,8 @@ function addAlertRowsToTable(tbody, alerts) {
         if (showComment) {
             $tr.append(`<td class='hideonmobile'>${commentText}</td>`);
         }
-        if (showSource) {
+        if (showType) {
-            $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid ${sigToIcon(a["sig"], "fa-globe-africa")}'></i></span> ${sigSourceText}</td>`);
+            $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid ${a["icon"]}'></i></span> ${sigTypeText}</td>`);
         }
         if (showRef) {
             $tr.append(`<td class='hideonmobile'>${sig_refs}</td>`);
@@ -281,8 +314,8 @@ function addAlertRowsToTable(tbody, alerts) {
         }
 
         const $td2 = $("<td colspan='100'>");
-        if (showSource) {
+        if (showType) {
-            $td2.append(`<span class='icon-wrapper'><i class='fa-solid ${sigToIcon(a["sig"], "fa-globe-africa")}'></i></span> `);
+            $td2.append(`<span class='icon-wrapper'><i class='fa-solid ${a["icon"]}'></i></span> `);
         }
         if (showRef) {
             $td2.append(`${sig_refs} `);
@@ -331,33 +364,10 @@ function filtersUpdated() {
     saveSettings();
 }
 
-// Update the refresh timing display
-function updateRefreshDisplay() {
-    if (lastUpdateTime != null) {
-        let secSinceUpdate = moment.duration(moment().diff(lastUpdateTime)).asSeconds();
-        let count = REFRESH_INTERVAL_SEC;
-        let updatingString = "Updating..."
-        if (secSinceUpdate < REFRESH_INTERVAL_SEC) {
-            count = REFRESH_INTERVAL_SEC - secSinceUpdate;
-            let number;
-            if (count <= 60) {
-                number = count.toFixed(0);
-                updatingString = "<span class='nowrap'>Updating in " + number + " second" + (number !== "1" ? "s" : "") + ".</span>";
-            } else {
-                number = Math.round(count / 60.0).toFixed(0);
-                updatingString = "<span class='nowrap'>Updating in " + number + " minute" + (number !== "1" ? "s" : "") + ".</span>";
-            }
-        }
-        $("#timing-container").html("Last updated at " + lastUpdateTime.format('HH:mm') + " UTC. " + updatingString);
-    }
-}
-
 // Startup
 $(document).ready(function () {
     // Call loadOptions(), this will then trigger loading alerts and setting up timers.
     loadOptions();
-    // Update the refresh timing display every second
-    setInterval(updateRefreshDisplay, 1000);
 });
 
 // Reload alerts on becoming visible. This forces a refresh when used as a PWA and the user switches back to the PWA

static/js/common.js

@@ -285,7 +285,6 @@ function callWanted(call) {
     }
 }
 
-
 // Startup
 $(document).ready(function () {
     usePreferredTheme();

static/js/map.js

@@ -223,7 +223,7 @@ function updateMap() {
 // Get an icon for a spot, based on its band, using PSK Reporter colours, its program etc.
 function getIcon(s) {
     return L.ExtraMarkers.icon({
-        icon: sigToIcon(s["sig"], "fa-tower-cell"),
+        icon: s["icon"],
         iconColor: bandToContrastColor(s["band"]),
         markerColor: bandToColor(s["band"]),
         shape: 'circle',
@@ -243,13 +243,19 @@ function getTooltipText(s) {
         dx_call = dx_call + "-" + s["dx_ssid"];
     }
 
+    // Format dx country
+    let dx_country = s["dx_country"];
+    if (dx_country == null) {
+        dx_country = "Unknown or not a country";
+    }
+
     // Format DX flag
     let dx_flag = "<i class='fa-solid fa-globe-africa'></i>";
     if (dx_call == null) {
         dx_flag = "";
     }
-    if (s["dx_flag"] && s["dx_flag"] != null && s["dx_flag"] !== "") {
+    if (s["dx_dxcc_id"] && s["dx_dxcc_id"] != null && s["dx_dxcc_id"] !== 0) {
-        dx_flag = s["dx_flag"];
+        dx_flag = `<img src="static/img/flags/${s['dx_dxcc_id']}.png" class="flag" width="24" alt="${dx_country}" title="${dx_country}"/>`;
     }
 
     // Format the frequency
@@ -303,7 +309,7 @@ function getTooltipText(s) {
     ttt += "<br/>";
 
     // Source / SIG / Ref
-    ttt += `<span class='nowrap'><span class='icon-wrapper'><i class='fa-solid ${sigToIcon(s["sig"], "fa-tower-cell")}'></i></span>&nbsp;${sigSourceText} ${sig_refs}</span><br/>`;
+    ttt += `<span class='nowrap'><span class='icon-wrapper'><i class='fa-solid ${s["icon"]}'></i></span>&nbsp;${sigSourceText} ${sig_refs}</span><br/>`;
 
     // Time
     ttt += `<span class='icon-wrapper'><i class='fa-solid fa-clock markerPopupIcon'></i></span>&nbsp;${moment.unix(s["time"]).fromNow()}`;