Remove accidentally committed debug code

# Conflicts:
#	core/constants.py
#	pyproject.toml
#	templates/add_spot.html
#	templates/alerts.html
#	templates/bands.html
#	templates/base.html
#	templates/conditions.html
#	templates/map.html
#	templates/spots.html
#	templates/status.html

.idea/modules.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ProjectModuleManager">
     <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/Spothole.iml" filepath="$PROJECT_DIR$/.idea/Spothole.iml" />
+      <module fileurl="file://$PROJECT_DIR$/.idea/spothole.iml" filepath="$PROJECT_DIR$/.idea/spothole.iml" />
     </modules>
   </component>
 </project>

.idea/runConfigurations/Run.xml

@@ -1,6 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="Run" type="PythonConfigurationType" factoryName="Python">
-    <module name="Spothole" />
+    <module name="spothole" />
     <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />

config-example.yml

@@ -68,7 +68,7 @@ spot_providers:
   - class: "DXCluster"
     # Clusters have a "name" property in case you want to say which cluster node the data is from, or connect to
     # several clusters and name them separately. The name will be used as the "source" field for the spots received.
-    name: "HRD Cluster"
+    name: "Cluster"
     enabled: true
     host: "hrd.wa9pie.net"
     port: 8000
@@ -84,7 +84,7 @@ spot_providers:
     allow_rbn_spots: false
 
   - class: "DXCluster"
-    name: "W3LPL Cluster"
+    name: "Cluster"
     enabled: false
     host: "w3lpl.net"
     port: 7373
@@ -246,12 +246,24 @@ sig_ref_data_providers:
   - class: "DME"
     enabled: true
 
+  - class: "FEA"
+    enabled: true
+
   - class: "KRMNPA"
     enabled: true
 
   - class: "SANPCPA"
     enabled: true
 
+  - class: "DTMBA"
+    enabled: true
+
+  - class: "COTA"
+    enabled: true
+
+  - class: "PGA"
+    enabled: true
+
   - class: "Toilets"
     enabled: true
 
@@ -345,4 +357,6 @@ web_ui_options:
   # (home) page, although being arbitrary HTML you can also use a div with absolute, relative, float placement etc. This
   # is designed for a "donate/support the server" type button, though you are free to do whatever you want with it.
   # As the server owner you are responsible for the safe usage of this option!
-  support_button_html: ""
+  support_button_html: ""
+  # CartoDB API key. Required to use the CartoDB layers without watermark. Get one from https://carto.com/basemaps/apikey/.
+  cartodb_api_key: ""

core/constants.py

@@ -3,7 +3,7 @@ from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "2.0.1"
+SOFTWARE_VERSION = "2.1-pre"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"}
@@ -16,145 +16,249 @@ SIGS = [
         comment_names=["POTA"],
         description="Parks on the Air",
         ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST",
+        icon="fa-tree",
+        refs_globally_unique=False
     ),
     SIG(
         name="SOTA",
         comment_names=["SOTA"],
         description="Summits on the Air",
         ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
+        icon="fa-mountain-sun",
+        refs_globally_unique=False
     ),
     SIG(
         name="WWFF",
         comment_names=["WWFF"],
         description="World Wide Flora & Fauna",
         ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}",
+        icon="fa-seedling",
+        refs_globally_unique=True
     ),
     SIG(
         name="GMA",
         comment_names=["GMA"],
         description="Global Mountain Activity",
         ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
+        icon="fa-person-hiking",
+        refs_globally_unique=False
     ),
     SIG(
         name="WWBOTA",
         comment_names=["WWBOTA", "BOTA"],
         description="Worldwide Bunkers on the Air",
         ref_regex=r"B\/[A-Z0-9]{1,3}\-\d{3,4}",
+        icon="fa-radiation",
+        refs_globally_unique=True
     ),
     SIG(
         name="HEMA",
         comment_names=["HEMA"],
         description="HuMPs Excluding Marilyns Award",
         ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}",
+        icon="fa-mound",
+        refs_globally_unique=False
     ),
     SIG(
         name="IOTA",
         comment_names=["IOTA"],
         description="Islands on the Air",
         ref_regex=r"[A-Z]{2}\-\d{3}",
+        icon="fa-book-atlas",
+        refs_globally_unique=False
     ),
     SIG(
-        name="GMA Island",
+        name="GMA Islands",
         comment_names=[],
         description="Global Mountain Activity - Islands",
         ref_regex=r"(([A-Z]{2}\-\d{3})|([A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}))",
-    ),
+        icon="fa-person-hiking",
-    SIG(
+        refs_globally_unique=False
-        name="MOTA",
-        comment_names=["MOTA"],
-        description="Mills on the Air",
-        ref_regex=r"X\d{4,6}",
     ),
     SIG(
         name="ARLHS",
         comment_names=["ARLHS"],
         description="Amateur Radio Lighthouse Society",
         ref_regex=r"[A-Z]{3}[\- ]\d{3,4}",
+        icon="fa-house-flood-water",
+        refs_globally_unique=False
     ),
     SIG(
         name="ILLW",
         comment_names=["ILLW"],
         description="International Lighthouse & Lightship Weekend",
         ref_regex=r"[A-Z]{2}\d{4}",
+        icon="fa-house-flood-water",
+        refs_globally_unique=False
+    ),
+    SIG(
+        name="MOTA",
+        comment_names=["MOTA"],
+        description="Mills on the Air",
+        ref_regex=r"X\d{4,6}",
+        icon="fa-fan",
+        refs_globally_unique=True
     ),
     SIG(
         name="SIOTA",
         comment_names=["SIOTA"],
         description="Silos on the Air",
         ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d",
+        icon="fa-wheat-awn",
+        refs_globally_unique=False
     ),
     SIG(
         name="WCA",
         comment_names=["WCA"],
         description="World Castles Award",
         ref_regex=r"[A-Z0-9]{1,3}\-\d{5}",
+        icon="fa-chess-rook",
+        refs_globally_unique=False
     ),
     SIG(
         name="ZLOTA",
         comment_names=["ZLOTA"],
         description="New Zealand on the Air",
         ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}",
+        icon="fa-kiwi-bird",
+        region_flag="🇳🇿",
+        refs_globally_unique=True
     ),
     SIG(
         name="WOTA",
         comment_names=["WOTA"],
         description="Wainwrights on the Air",
         ref_regex=r"[A-Z]{3}-[0-9]{2}",
+        icon="fa-w",
+        region_flag="🇬🇧",
+        refs_globally_unique=False
+    ),
+    SIG(name="BOTA",
+        comment_names=[],
+        description="Beaches on the Air",
+        icon="fa-umbrella-beach"
     ),
-    SIG(name="BOTA", comment_names=[], description="Beaches on the Air"),
     SIG(
         name="KRMNPA",
         comment_names=["KRMNPA"],
         description="Keith Roget Memorial National Parks Award",
         ref_regex=r"VKFF\-\d{4}",
+        icon="fa-earth-oceania",
+        region_flag="🇦🇺"
     ),
     SIG(
         name="SANPCPA",
         comment_names=["SANPCPA"],
         description="South Australian National Parks and Conservation Parks Award",
         ref_regex=r"VKFF\-\d{4}",
+        icon="fa-earth-oceania",
+        region_flag="🇦🇺"
     ),
     SIG(
         name="LLOTA",
         comment_names=["LLOTA"],
         description="Lagos y Lagunas on the Air",
         ref_regex=r"LL[A-Z]{2}\-\d{4}",
+        icon="fa-water",
+        refs_globally_unique=True
     ),
     SIG(
         name="Towers",
         comment_names=["TOTA"],
         description="Towers on the Air",
         ref_regex=r"[A-Z]{2,3}R\-\d{4}",
+        icon="fa-tower-observation",
+        refs_globally_unique=False
     ),
     SIG(
         name="Tiles",
         comment_names=[],
         description="Tiles on the Air",
         ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}",
+        icon="fa-square",
+        refs_globally_unique=False
     ),
     SIG(
         name="WAB",
         comment_names=["WAB"],
         description="Worked All Britain",
         ref_regex=r"[A-Z]{1,2}[0-9]{2}",
+        icon="fa-table-cells-large",
+        region_flag="🇬🇧",
+        refs_globally_unique=False
     ),
     SIG(
         name="WAI",
         comment_names=["WAI"],
         description="Worked All Ireland",
         ref_regex=r"[A-Z][0-9]{2}",
+        icon="fa-table-cells-large",
+        region_flag="🇮🇪",
+        refs_globally_unique=False
     ),
     SIG(
         name="DME",
         comment_names=["DME"],
-        description="Diplomas de Municipios Españoles",
+        description="Diploma Municipios de España",
         ref_regex=r"\d{4,5}",
+        icon="fa-building",
+        region_flag="🇪🇸",
+        refs_globally_unique=False
+    ),
+    SIG(
+        name="FEA",
+        comment_names=["FEA"],
+        description="Diploma Faros de España",
+        # FEA references are technically [DE]\-\d{4}(\.\d)? but spotters always seem to miss out the D- or E-
+        # prefix and just use FEA-1234 or FEA 1234, so allow for that. The FEA sigref data provider adds both
+        # forms to the database.
+        ref_regex=r"([DE]|FEA)[\- ]\d{4}(\.\d)?",
+        icon="fa-house-flood-water",
+        region_flag="🇪🇸",
+        refs_globally_unique=True
+    ),
+    SIG(
+        name="DTMBA",
+        comment_names=["DTMBA"],
+        description="Diploma Teatri Musei e Belle Arti",
+        ref_regex=r"I-?[0-9]{3,4}\s?[A-Z]{2}",
+        icon="fa-masks-theater",
+        region_flag="🇮🇹",
+        refs_globally_unique=True
+    ),
+    SIG(
+        name="BIWOTA",
+        comment_names=["BIWOTA"],
+        description="British Inland Waterways on the Air",
+        icon="fa-ship",
+        region_flag="🇬🇧"
+    ),
+    SIG(
+        name="COTA",
+        comment_names=["COTA"],
+        description="Castles on the Air",
+        ref_regex=r"[A-Z]{3}\-[0-9]{3,5}",
+        icon="fa-chess-rook",
+        region_flag="🇩🇪",
+        refs_globally_unique=False
+    ),
+    SIG(
+        name="PGA",
+        comment_names=["PGA"],
+        description="Polish Gmina Award",
+        ref_regex=r"[A-Z]{2}[0-9]{2}",
+        icon="fa-g",
+        region_flag="🇵🇱",
+        refs_globally_unique=False
     ),
     SIG(
         name="Toilets",
         comment_names=[],
         description="Toilets on the Air",
         ref_regex=r"T\-[0-9]{2}",
+        icon="fa-toilet",
+        region_flag="🏴‍☠️",
+        refs_globally_unique=True
     ),
 ]
 
@@ -163,16 +267,9 @@ CW_MODES = ["CW"]
 PHONE_MODES = [
     "PHONE",
     "SSB",
-    "USB",
-    "LSB",
     "AM",
     "FM",
-    "DV",
+    "DV"
-    "DMR",
-    "DSTAR",
-    "C4FM",
-    "FUSION",
-    "M17",
 ]
 DATA_MODES = [
     "DATA",
@@ -190,14 +287,19 @@ DATA_MODES = [
 ]
 ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
 MODE_TYPES = ["CW", "PHONE", "DATA"]
-SSB_SUB_MODES = ["USB", "LSB"]
-DV_SUB_MODES = ["DMR", "DSTAR", "C4FM", "FUSION", "M17"]
 
 # Mode aliases. Sometimes we get spots with a mode described in a different way that is effectively the same as a mode
 # we already know, or we want to normalise things for consistency. The lookup table for this is here. Incoming spots
 # that match a key in this table will be converted to the corresponding value, so only the modes above will actually be
 # present in the spots.
 MODE_ALIASES = {
+    "USB": "SSB",
+    "LSB": "SSB",
+    "DMR": "DV",
+    "DSTAR": "DV",
+    "C4FM": "DV",
+    "FUSION": "DV",
+    "M17": "DV",
     "RTT": "RTTY",
     "BPSK": "PSK",
     "PSK31": "PSK",

core/data_store.py

@@ -6,6 +6,7 @@ import diskcache
 
 from core.config import MAX_ALERT_AGE, MAX_SPOT_AGE
 from core.live_data_cache import LiveDataCache
+from core.single_object_data_cache import SingleObjectDataCache
 from data.solar_conditions import SolarConditions
 
 logger = logging.getLogger(__name__)
@@ -34,10 +35,8 @@ class DataStore:
         self.dxcc_data = None
         self.dxcc_lookup_by_call_regex = []
         self.sigrefs = None
-        self.status_data = {}
+        self.status = None
-        self._status = None
+        self.solar_conditions = None
-        self.solar_conditions = {}
-        self._solar = None
         # ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
         # caches, they can just be straight objects
         self.cq_zone_data = None
@@ -46,16 +45,11 @@ class DataStore:
     def setup(self):
         Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
 
-        # Standard disk cache for solar data and status data, but each cache contains only a single object which we
+        # For solar data and status data, we use a wrapper around disk cache where each cache contains only a single
-        # expose to the wider application
+        # object exposed to the wider application, and provides a store() method for callers to notify diskcache that
-        self._solar = diskcache.Cache(f"{CACHE_DIR}solar")
+        # the object has changed and needs to be re-cached.
-        if "solar_conditions" not in self._solar:
+        self.solar_conditions = SingleObjectDataCache(f"{CACHE_DIR}solar", SolarConditions())
-            self._solar.add("solar_conditions", SolarConditions())
+        self.status = SingleObjectDataCache(f"{CACHE_DIR}status", {})
-        self.solar_conditions = self._solar.get("solar_conditions")
-        self._status = diskcache.Cache(f"{CACHE_DIR}status")
-        if "status_data" not in self._status:
-            self._status.add("status_data", {})
-        self.status_data = self._status.get("status_data")
 
         # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
         # a regular basis but there's no need for a TTL since old data is better than no data.
@@ -116,13 +110,10 @@ class DataStore:
             self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
 
     def close(self):
-        self.spots.save_snapshot()
-        self.alerts.save_snapshot()
-
         self.spots.close()
         self.alerts.close()
-        self._solar.close()
+        self.solar_conditions.close()
-        self._status.close()
+        self.status.close()
         self.dxcc_data.close()
         self.sigrefs.close()
         self.callsign_data_countryfiles.close()

core/sig_lookup_helper.py

@@ -19,6 +19,11 @@ def get_sig_ref_info(sig, ref_id):
         logger.debug("Failed to look up sig_ref info, sig or ref were not set.")
         return None
 
+    # Sometimes we allow spaces instead of dashes in references due to common usage that way, but official reference
+    # lists never do, so convert them here.
+    ref_id.replace(" ", "-")
+
+    # Prepare the object to be returned
     sig_ref = SIGRef(sig=sig, id=ref_id)
 
     try:
@@ -29,6 +34,11 @@ def get_sig_ref_info(sig, ref_id):
         if sig.upper() == "DME":
             ref_id = ref_id.zfill(5)
 
+        # DTMBA spotters sometimes include spaces and dashes, our regex allows them but they must be removed here so we
+        # can look up against the official list which doesn't have them
+        if sig.upper() == "DTMBA":
+            ref_id = ref_id.replace("-", "").replace(" ", "")
+
         # If the SIG is HEMA, we have no current lookup for this so just skip the lookup here.
         if sig.upper() == "HEMA":
             sig_ref.ref_type = "Summit"
@@ -73,6 +83,10 @@ def get_sig_ref_info(sig, ref_id):
                 sig_ref.url = f"https://www.beachesontheair.com/beaches/{sig_ref.name.lower().replace(' ', '-')}"
             return sig_ref
 
+        elif sig.upper() == "BIWOTA":
+            # For BIWOTA there are no references, all we can set is a type
+            sig_ref.ref_type = "Waterway"
+
         elif sig.upper() == "GMA Islands":
             # GMA Islands is a bit of a mess of GMA and IOTA references. Try looking them both up and see what returns
             # the best result.

core/single_object_data_cache.py

@@ -0,0 +1,39 @@
+import logging
+import threading
+
+import diskcache
+
+logger = logging.getLogger(__name__)
+
+
+class SingleObjectDataCache:
+    """Cache for status and solar conditions. This uses DiskCache, but unlike the standard DiskCache users like SIG and
+    callsign lookup handlers, status and solar conditions are persisted as a single object. If we just load the object
+    from DiskCache and modify it, DiskCache doesn't know that it's been updated and needs re-caching, so we provide a
+    store() method that any functions updating the object can call afterwards."""
+
+    def __init__(self, cache_dir, object_if_empty):
+        """Initialize a SingleObjectDataCache. Provide the directory to load the cache from and save it to. If the cache
+        is empty, the provided object_if_empty parameter will be used to initialise it."""
+
+        self._lock = threading.Lock()
+        self._cache = diskcache.Cache(cache_dir)
+        # This cache stores a single object, doesn't matter what it's called so "object" will do
+        if "object" not in self._cache:
+            self._cache.add("object", object_if_empty)
+        self._obj = self._cache.get("object")
+
+    def get(self):
+        """Get the data object. This can then be manipulated as necessary across multiple threads. Any function
+        modifying the object must remember to call store() afterwards."""
+        return self._obj
+
+    def store(self):
+        """Store the updated object in the cache. Any function modifying the object must remember to call this
+        afterwards."""
+        with self._lock:
+            self._cache.set("object", self._obj)
+
+    def close(self):
+        self.store()
+        self._cache.close()

core/status_reporter.py

@@ -25,8 +25,9 @@ class StatusReporter:
         self._stop_event = Event()
         self._startup_time = datetime.now(pytz.UTC)
 
-        DATA_STORE.status_data["software_version"] = SOFTWARE_VERSION
+        DATA_STORE.status.get()["software_version"] = SOFTWARE_VERSION
-        DATA_STORE.status_data["server_owner_callsign"] = SERVER_OWNER_CALLSIGN
+        DATA_STORE.status.get()["server_owner_callsign"] = SERVER_OWNER_CALLSIGN
+        DATA_STORE.status.store()
 
     def start(self):
         """Start the reporter thread"""
@@ -50,11 +51,11 @@ class StatusReporter:
     def _report(self):
         """Write status information"""
 
-        DATA_STORE.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
+        DATA_STORE.status.get()["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
-        DATA_STORE.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
+        DATA_STORE.status.get()["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
-        DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
+        DATA_STORE.status.get()["num_spots"] = len(DATA_STORE.spots.values())
-        DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
+        DATA_STORE.status.get()["num_alerts"] = len(DATA_STORE.alerts.values())
-        DATA_STORE.status_data["spot_providers"] = [
+        DATA_STORE.status.get()["spot_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -69,7 +70,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.spot_providers
         ]
-        DATA_STORE.status_data["alert_providers"] = [
+        DATA_STORE.status.get()["alert_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -80,7 +81,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.alert_providers
         ]
-        DATA_STORE.status_data["solar_condition_providers"] = [
+        DATA_STORE.status.get()["solar_condition_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -91,7 +92,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.solar_condition_providers
         ]
-        DATA_STORE.status_data["static_data_providers"] = [
+        DATA_STORE.status.get()["static_data_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -102,7 +103,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.static_data_providers
         ]
-        DATA_STORE.status_data["sig_ref_data_providers"] = [
+        DATA_STORE.status.get()["sig_ref_data_providers"] = [
             {
                 "sig_name": p.sig_name,
                 "enabled": p.enabled,
@@ -114,7 +115,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.sig_ref_data_providers
         ]
-        DATA_STORE.status_data["callsign_data_providers"] = [
+        DATA_STORE.status.get()["callsign_data_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -126,13 +127,13 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.callsign_data_providers
         ]
-        DATA_STORE.status_data["cleanup"] = {
+        DATA_STORE.status.get()["cleanup"] = {
             "status": CLEANUP_TIMER.status,
             "last_ran": CLEANUP_TIMER.last_cleanup_time.replace(tzinfo=pytz.UTC).timestamp()
             if CLEANUP_TIMER.last_cleanup_time
             else 0,
         }
-        DATA_STORE.status_data["webserver"] = {
+        DATA_STORE.status.get()["webserver"] = {
             "status": WEB_SERVER.web_server_metrics["status"],
             "last_api_access": WEB_SERVER.web_server_metrics["last_api_access_time"]
             .replace(tzinfo=pytz.UTC)
@@ -147,6 +148,7 @@ class StatusReporter:
             else 0,
             "page_access_count": WEB_SERVER.web_server_metrics["page_access_counter"],
         }
+        DATA_STORE.status.store()
 
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)

core/utils.py

@@ -1,4 +1,5 @@
 import logging
+import re
 
 import simplejson
 from pyhamtools.frequency import freq_to_band
@@ -30,10 +31,10 @@ def infer_mode_from_comment(comment):
     """Infer a mode from the comment"""
 
     for mode in ALL_MODES:
-        if mode in comment.upper():
+        if re.search(r"(^|\W)" + mode + r"($|\W)", comment, re.IGNORECASE):
             return mode
     for mode in MODE_ALIASES:
-        if mode in comment.upper():
+        if re.search(r"(^|\W)" + mode + r"($|\W)", comment, re.IGNORECASE):
             return MODE_ALIASES[mode]
     return None
 
@@ -70,7 +71,7 @@ def infer_mode_from_frequency(freq):
         mode = freq_to_band(khz)["mode"]
         # Some additional common digimode ranges in addition to what the 3rd-party freq_to_band function returns.
         # This is mostly here just because freq_to_band is very specific about things like FT8 frequencies, and e.g.
-        # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
+        # a spot at 7074.5 kHz will be indicated as SSB, even though it's clearly in the FT8 range. Future updates
         # might include other common digimode centres of activity here, but this achieves the main goal of keeping
         # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
         if (

data/sig.py

@@ -19,3 +19,14 @@ class SIG:
     comment_names: list[str] = field(default_factory=list)
     # Regex matcher for references, e.g. for POTA r"[A-Z]{2}\-\d+".
     ref_regex: str | None = None
+    # Identifies that the SIG's reference ID structure defined by its regex is unique across all programmes and
+    # anything else we expect a user to put in a spot comment, and therefore we can pull references out of
+    # spot comments without also needing to see the SIG name first. For example, "OHFF-1234" or "B/G-1234" are
+    # obviously WWFF and WWBOTA, nothing else looks like those. But "SZ09" could be WAB or Tiles, "GB1234" could
+    # conceivably be POTA or ILLW, etc.
+    refs_globally_unique: bool = False
+    # Icon to use in the UI when referencing this SIG. Chosen from the Font Awesome set.
+    icon: str | None = None
+    # Emoji flag for the country or region where this SIG is relevant, if any. If None, this implies the SIG is in
+    # worldwide usage.
+    region_flag: str | None = None

data/spot.py

@@ -11,7 +11,7 @@ from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 
 from core.call_lookup_helper import get_call_info
 from core.config import MAX_SPOT_AGE
-from core.constants import MODE_ALIASES, PROPAGATION_MODES
+from core.constants import MODE_ALIASES, PROPAGATION_MODES, SIGS
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.sig_utils import (
@@ -270,7 +270,7 @@ class Spot:
                 sig = self.sig_refs[0].sig.upper()
                 regex = get_ref_regex_for_sig(sig)
                 if regex:
-                    all_comment_ref_matches = re.finditer(r"(^|\W)(" + regex + r")(^|\W)", self.comment, re.IGNORECASE)
+                    all_comment_ref_matches = re.finditer(r"(^|\W)(" + regex + r")($|\W)", self.comment, re.IGNORECASE)
                     for ref_match in all_comment_ref_matches:
                         self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(2).upper(), sig=sig))
 
@@ -298,6 +298,19 @@ class Spot:
                         for ref_match in ref_matches:
                             self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig))
 
+            # See if the comment looks like it contains any SIG references *without* the corresponding SIG name, but
+            # where the SIG reference is unique-looking enough that we can't confuse it with any other SIG.
+            if self.comment:
+                for sig in SIGS:
+                    if sig.refs_globally_unique and sig.ref_regex:
+                        ref_matches = re.finditer(r"(^|\W)(" + sig.ref_regex + r")($|\W)", self.comment, re.IGNORECASE)
+                        for ref_match in ref_matches:
+                            # First of all, if we haven't got a SIG for this spot set yet, now we have. This covers things
+                            # like cluster spots where the comment is just "OHFF-1234", now we know it's WWFF.
+                            if not self.sig:
+                                self.sig = sig.name
+                            self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(2).upper(), sig=sig.name))
+
             # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference
             # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs
             # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths

datafiles/didbase-stations.csv

@@ -1,12 +1,9 @@
 AA343,"Almaty, Kazakhstan"
 AL945,"Alpena, United States"
 AT138,"Athens, Greece"
-AU930,"Austin, United States"
-BR52P,"Brisbane, Australia"
 BVJ03,"Boa Vista, Brazil"
 CAJ2M,"Cachoeira Paulista, Brazil"
 CB53N,"Canberra, Australia"
-CGK21,"Campo Grande, Brazil"
 DB049,"Dourbes, Belgium"
 DW41K,"Darwin, Australia"
 EA036,"El Arenosillo, Spain"
@@ -18,24 +15,18 @@ GM037,"Gibilmanna, Italy"
 GR13L,"Grahamstown, South Africa"
 HE13N,"Hermanus, South Africa"
 HO54K,"Hobart, Australia"
-IC437,"I-Cheon, South Korea"
 IF843,"Idaho National Laboratory, United States"
 JI91J,"Jicamarca, Peru"
 JR055,"Juliusruh, Germany"
 LAA38,"Lajes Terceira Island, Portugal"
 LL721,"Lualualei, United States"
-LM42B,"Learmonth, Australia"
-LV12P,"Louisvale, South Africa"
 MHJ45,"Millstone Hill, United States"
-ML10L,"Malindi, Kenya"
 NI135,"Nicosia, Cyprus"
 NI63_,"Norfolk, Australia"
-PA836,"Portt Arguello, United States"
+PA836,"Port Arguello, United States"
 PE43K,"Perth, Australia"
-PF765,"Poker Flat, United States"
 PQ052,"Pruhonice, Czechia"
 RO041,"Rome, Italy"
-SAA0K,"Saoluis, Brazil"
 SO148,"Sopron, Hungary"
 TR169,"Tromso, Norway"
 TV51R,"Townsville, Australia"

providers/sigrefdata/cota.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class COTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Castles on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "COTA"
+    DATA_URL = "https://www.cotagroup.org/cotagroup/map/data/castles-all-7d90ee2a5e1175e5dece1bbf9dc87504.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data[2]:
+                ref_id = ref[3]
+                name = ref[4]
+                lat = float(ref[0])
+                lon = float(ref[1])
+                grid = latlong_to_locator(lat, lon)
+
+                new_data.append(
+                    SIGRef(
+                        sig=self.SIG,
+                        id=ref_id,
+                        name=name,
+                        ref_type="Castle",
+                        grid=grid,
+                        latitude=lat,
+                        longitude=lon,
+                    )
+                )
+
+                # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
+                # of the data in this case
+                if self._stop_event.is_set():
+                    break
+
+                # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+                # is available for other threads e.g. the web server.
+                sleep(0.001)
+
+        return new_data

providers/sigrefdata/dtmba.py

@@ -0,0 +1,41 @@
+from time import sleep
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class DTMBA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Diploma Teatri Musei Belle Arti"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "DTMBA"
+    DATA_URL = "https://www.iu1fig.com/share/iz0eik/dtmba/export.php"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in http_response.content.decode("utf-8-sig").splitlines():
+            split = row.split(";")
+            ref_id = split[0]
+            ref_name = split[1]
+            new_data.append(
+                SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=ref_name,
+                    ref_type="Building"
+                )
+            )
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
+        return new_data

providers/sigrefdata/fea.py

@@ -0,0 +1,55 @@
+from io import BytesIO
+from time import sleep
+
+import pdfplumber
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class FEA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Diploma Faros de España"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "FEA"
+    DATA_URL = "http://ea5ol.net/Lista%20Faros.pdf"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+
+        # Use PDFPlumber to extract the tables in the PDF
+        with pdfplumber.open(BytesIO(http_response.content)) as pdf:
+            all_rows = []
+
+            for page_number, page in enumerate(pdf.pages, start=1):
+                tables = page.extract_tables()
+
+                for table_number, table in enumerate(tables, start=1):
+                    if not table:
+                        continue
+
+                    rows = [row for row in table if any(cell and cell.strip() for cell in row)]
+                    all_rows.extend(rows)
+
+        for row in all_rows:
+            if not "REF" in row[0] and not "\n" in row[0]:
+                # FEA references are technically [DE]\-\d{4}(\.\d)? but spotters always seem to miss out the D- or E-
+                # prefix and just use FEA-1234 or FEA 1234, so we add both copies to the database.
+                ref_id_1 = row[0].strip()
+                ref_id_2 = ref_id_1.replace("D-", "FEA-").replace("E-", "FEA-")
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id_1, name=row[1].strip(), ref_type="Lighthouse"))
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id_2, name=row[1].strip(), ref_type="Lighthouse"))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
+        return new_data

providers/sigrefdata/pga.py

@@ -0,0 +1,57 @@
+from time import sleep
+
+from bs4 import BeautifulSoup
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class PGA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Polish Gmina Award"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "PGA"
+    DATA_URL = "http://www.spga.pl/lista_pga2.php"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        soup = BeautifulSoup(http_response.text, "html.parser")
+
+        # Iterate through tables in the page
+        for table in soup.find_all("table"):
+            header_cells = table.find_all(["th", "td"], limit=10)
+            header_texts = [c.get_text(strip=True) for c in header_cells]
+
+            # If it has "PGA" and "Nazwa" in the header, it's the main data table
+            if any("PGA" in t for t in header_texts) and any("Nazwa" in t for t in header_texts):
+                # Iterate through all rows except the first
+                rows = table.find_all("tr")
+                for row in rows[1:]:
+                    cells = row.find_all("td")
+                    ref_id = cells[0].get_text(strip=True)
+                    name = cells[1].get_text(strip=True)
+                    if not ref_id:
+                        continue
+
+                    new_data.append(
+                        SIGRef(
+                            sig=self.SIG,
+                            id=ref_id,
+                            name=name,
+                            ref_type="Region",
+                        )
+                    )
+
+                    # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
+                    # of the data in this case
+                    if self._stop_event.is_set():
+                        break
+
+                    # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+                    # is available for other threads e.g. the web server.
+                    sleep(0.001)
+
+        return new_data

providers/solarconditions/giroionosonde.py

@@ -20,7 +20,7 @@ POLL_INTERVAL = 7200
 # To avoid looking up all stations in the GIRO system and working out which ones are providing live data, this has been
 # manually determined and a CSV provided of all the stations that we can query for live data.
 STATIONS_INDEX = "datafiles/didbase-stations.csv"
-LGDC_URL = "https://lgdc.uml.edu/common/DIDBGetValues"
+LGDC_URL = "https://lgdc.uml.edu/fastchar/getbest"
 HISTORY_HOURS = 24
 
 
@@ -140,7 +140,7 @@ class GIROIonosonde(SolarConditionsProvider):
 
         from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S")
         to_str = to_time.strftime("%Y.%m.%d+%H:%M:%S")
-        url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUFD,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}"
+        url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUF%28D%29,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}"
         try:
             http_response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
             if not http_response.ok:

providers/solarconditions/solar_conditions_provider.py

@@ -16,7 +16,7 @@ class SolarConditionsProvider:
         self.enabled = provider_config.get("enabled", True)
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._solar_conditions = DATA_STORE.solar_conditions
+        self._solar_conditions = DATA_STORE.solar_conditions.get()
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -36,3 +36,4 @@ class SolarConditionsProvider:
                 if hasattr(self._solar_conditions, key):
                     setattr(self._solar_conditions, key, value)
             self._solar_conditions.infer_descriptions()
+            DATA_STORE.solar_conditions.store()

providers/spot/gma.py

@@ -118,8 +118,8 @@ class GMA(HTTPSpotProvider):
                                         spot.sig_refs[0].sig = "IOTA"
                                         spot.sig = "IOTA"
                                     case "GMA Island":
-                                        spot.sig_refs[0].sig = "GMA Island"
+                                        spot.sig_refs[0].sig = "GMA Islands"
-                                        spot.sig = "GMA Island"
+                                        spot.sig = "GMA Islands"
                                     case "Lighthouse (ILLW)":
                                         spot.sig_refs[0].sig = "ILLW"
                                         spot.sig = "ILLW"

providers/spot/parksnpeaks.py

@@ -19,7 +19,6 @@ class ParksNPeaks(HTTPSpotProvider):
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
     SUBMIT_URL = "https://www.parksnpeaks.org/api/SPOT/"
-    SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
     SUBMITTABLE_SIGS = [
         "POTA",
         "SOTA",

providers/spot/sota.py

@@ -4,7 +4,7 @@ from datetime import datetime
 import requests
 from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
-from core.constants import DV_SUB_MODES, HTTP_HEADERS, SSB_SUB_MODES
+from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -98,12 +98,7 @@ class SOTA(HTTPSpotProvider):
             # https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js
             mode = spot.mode
             if mode and mode not in self.VALID_MODES:
-                if mode in SSB_SUB_MODES:
+                mode = "Data"
-                    mode = "SSB"
-                elif mode in DV_SUB_MODES:
-                    mode = "DV"
-                else:
-                    mode = "Data"
 
             body = {
                 "activatorCallsign": spot.dx_call,

providers/spot/tiles.py

@@ -3,7 +3,7 @@ from datetime import datetime
 
 import requests
 
-from core.constants import HTTP_HEADERS, SSB_SUB_MODES
+from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -92,9 +92,7 @@ class Tiles(HTTPSpotProvider):
             if spot.mode:
                 mode = spot.mode
                 if mode not in self.VALID_MODES:
-                    if mode in SSB_SUB_MODES:
+                    if mode == "OLIVIA":
-                        mode = "SSB"
-                    elif mode == "OLIVIA":
                         mode = "Olivia"
                     elif mode == "JS8":
                         mode = "JS8Call"

pyproject.toml

@@ -1,8 +1,8 @@
 [project]
-name = "Spothole"
+name = "spothole"
-version = "2.0.1"
+version = "2.1-pre"
 authors = [
-  { name="Ian Renton", email="ian@ianrenton.com" },
+    { name = "Ian Renton", email = "ian@ianrenton.com" },
 ]
 description = "Spothole is a utility to aggregate \"spots\" from amateur radio DX clusters and xOTA spotting sites, and provide an open JSON API as well as a website to browse the data."
 readme = "README.md"

requirements.txt

@@ -20,4 +20,5 @@ geopandas~=0.13.2
 simplejson~=4.1.1
 cachetools~=7.1.6
 fastkml~=1.4.0
-ruff~=0.16.3
+ruff~=0.16.3
+pdfplumber~=0.11.10

server/webserver.py

@@ -111,7 +111,7 @@ class WebServer:
             (
                 r"/api/v2/solar",
                 APISolarConditionsHandler,
-                {"solar_conditions": self._data_store.solar_conditions, **handler_opts},
+                {"solar_conditions": self._data_store.solar_conditions.get(), **handler_opts},
             ),
             (
                 r"/api/v2/dxstats",
@@ -121,12 +121,12 @@ class WebServer:
             (
                 r"/api/v2/options",
                 APIOptionsHandler,
-                {"status_data": self._data_store.status_data, **handler_opts},
+                {"status_data": self._data_store.status.get(), **handler_opts},
             ),
             (
                 r"/api/v2/status",
                 APIStatusHandler,
-                {"status_data": self._data_store.status_data, **handler_opts},
+                {"status_data": self._data_store.status.get(), **handler_opts},
             ),
             (r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}),
             (r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),

spothole.py

@@ -10,6 +10,7 @@ from core.constants import SOFTWARE_VERSION
 from core.data_providers import DATA_PROVIDERS
 from core.data_store import DATA_STORE
 from core.status_reporter import StatusReporter
+from core.utils import infer_mode_from_comment
 from server.webserver import WEB_SERVER
 
 logger = logging.getLogger(__name__)

static/apidocs/openapi.yml

@@ -15,6 +15,12 @@ info:
 
     ## Changelog
     
+    ### 2.1
+    
+    * Added DTMBA, FEA, BIWOTA, COTA & PGA SIGs
+    * Removed the distinction between LSB & USB (both will now show as SSB) and between the various digital voice modes, which will now show as DV.
+    * Added `icon`, `region_flag` and `refs_globally_unique` to SIG information
+    
     ### 2.0
 
     * **Breaking change:** The "add spot" API has changed to enable future support for upstream submission to the spotting services associated with various SIGs. Instead of just posting the spot object itself as the JSON content of the POST, this has moved into a `spot` object within the structure. A new `handling` object alongside it contains the `submit_upstream`, `upstream_provider`, `upstream_credentials`, and `captcha_token` fields which control the server handling of the spot.
@@ -852,6 +858,11 @@ components:
         - WAB
         - WAI
         - DME
+        - FEA
+        - DTMBA
+        - BIWOTA
+        - COTA
+        - PGA
         - Toilets
       example: POTA
 
@@ -911,15 +922,9 @@ components:
         - CW
         - PHONE
         - SSB
-        - USB
-        - LSB
         - AM
         - FM
         - DV
-        - DMR
-        - DSTAR
-        - C4FM
-        - M17
         - DATA
         - FT8
         - FT4
@@ -1502,6 +1507,24 @@ components:
             Regex that matches this SIG's reference IDs. Generally for Spothole's own internal use,
             clients probably won't need this.
           example: "[A-Z]{2}\\-\\d+"
+        refs_globally_unique:
+          type: boolean
+          description: >
+            Identifies that the SIG's reference ID structure defined by its regex is unique across all programmes and
+            anything else we expect a user to put in a spot comment, and therefore we can pull references out of
+            spot comments without also needing to see the SIG name first. For example, "OHFF-1234" or "B/G-1234" are
+            obviously WWFF and WWBOTA, nothing else looks like those. But "SZ09" could be WAB or Tiles, "GB1234" could
+            conceivably be POTA or ILLW, etc. Generally for Spothole's own internal use, clients probably won't need
+            this.
+        icon:
+          type: string
+          description: Icon from the Font Awesome set that represents this SIG, for use in the front end.
+          example: "fa-tree"
+        region_flag:
+          type: string
+          description: >
+            Flag emoji, if this SIG is specific to a country or region. If null, this SIG is treated as worldwide.
+          example: "🇺🇳"
 
     SolarConditions:
       type: object
@@ -2030,7 +2053,7 @@ components:
           description: An array of all the supported modes.
           items:
             type: string
-            example: "LSB"
+            example: "SSB"
         mode_types:
           type: array
           description: An array of all the supported mode types.

static/js/alerts.js

@@ -264,7 +264,7 @@ function addAlertRowsToTable(tbody, alerts) {
             $tr.append(`<td class='hideonmobile'>${commentText}</td>`);
         }
         if (showSource) {
-            $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid ${sigToIcon(a["sig"], "fa-globe-africa")}'></i></span> ${sigSourceText}</td>`);
+            $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid ${sigToIcon(a["sig"], options["sigs"], "fa-globe-africa")}'></i></span> ${sigSourceText}</td>`);
         }
         if (showRef) {
             $tr.append(`<td class='hideonmobile'>${sig_refs}</td>`);
@@ -282,7 +282,7 @@ function addAlertRowsToTable(tbody, alerts) {
 
         const $td2 = $("<td colspan='100'>");
         if (showSource) {
-            $td2.append(`<span class='icon-wrapper'><i class='fa-solid ${sigToIcon(a["sig"], "fa-globe-africa")}'></i></span> `);
+            $td2.append(`<span class='icon-wrapper'><i class='fa-solid ${sigToIcon(a["sig"], options["sigs"], "fa-globe-africa")}'></i></span> `);
         }
         if (showRef) {
             $td2.append(`${sig_refs} `);

static/js/common.js

@@ -285,6 +285,19 @@ function callWanted(call) {
     }
 }
 
+// Get the Font Awesome icon for a given SIG. If the SIG is unknown, the provided default symbol will be returned
+// The sig is provided by name as a string, while all_sigs contains all the SIG info from the options call. This
+// is because sometimes the sig name arrives via a spot and therefore we need to look up the rest of the SIG data
+// separately.
+function sigToIcon(sig, all_sigs, defaultIcon) {
+    if (sig) {
+        const match = all_sigs.find((check_sig) => check_sig["name"].toUpperCase() === sig.toUpperCase());
+        if (match) {
+            return match["icon"];
+        }
+    }
+    return defaultIcon;
+}
 
 // Startup
 $(document).ready(function () {

static/js/map.js

@@ -223,7 +223,7 @@ function updateMap() {
 // Get an icon for a spot, based on its band, using PSK Reporter colours, its program etc.
 function getIcon(s) {
     return L.ExtraMarkers.icon({
-        icon: sigToIcon(s["sig"], "fa-tower-cell"),
+        icon: sigToIcon(s["sig"], options["sigs"], "fa-tower-cell"),
         iconColor: bandToContrastColor(s["band"]),
         markerColor: bandToColor(s["band"]),
         shape: 'circle',
@@ -303,7 +303,7 @@ function getTooltipText(s) {
     ttt += "<br/>";
 
     // Source / SIG / Ref
-    ttt += `<span class='nowrap'><span class='icon-wrapper'><i class='fa-solid ${sigToIcon(s["sig"], "fa-tower-cell")}'></i></span>&nbsp;${sigSourceText} ${sig_refs}</span><br/>`;
+    ttt += `<span class='nowrap'><span class='icon-wrapper'><i class='fa-solid ${sigToIcon(s["sig"], options["sigs"], "fa-tower-cell")}'></i></span>&nbsp;${sigSourceText} ${sig_refs}</span><br/>`;
 
     // Time
     ttt += `<span class='icon-wrapper'><i class='fa-solid fa-clock markerPopupIcon'></i></span>&nbsp;${moment.unix(s["time"]).fromNow()}`;
@@ -394,9 +394,11 @@ function setBasemap(basemapname) {
         }
         // OpenStreetMap.Mapnik.Dark is a synthetic variant that uses Mapnik tiles with a CSS filter applied
         const providerName = basemapname === "OpenStreetMap.Mapnik.Dark" ? "OpenStreetMap.Mapnik" : basemapname;
+        const isCartoDB = basemapname === "CartoDB.Voyager" || basemapname === "CartoDB.DarkMatter";
         backgroundTileLayer = L.tileLayer.provider(providerName, {
             opacity: parseFloat($("#basemapOpacity").val()),
-            edgeBufferTiles: 1
+            edgeBufferTiles: 1,
+            key: isCartoDB ? CARTODB_API_KEY : null
         });
         backgroundTileLayer.addTo(map);
         backgroundTileLayer.bringToBack();
@@ -529,9 +531,11 @@ function setUpMap() {
     // Add basemap
     loadedBasemap = $("#basemap").val();
     const initialProviderName = loadedBasemap === "OpenStreetMap.Mapnik.Dark" ? "OpenStreetMap.Mapnik" : loadedBasemap;
+    const isCartoDB = loadedBasemap === "CartoDB.Voyager" || loadedBasemap === "CartoDB.DarkMatter";
     backgroundTileLayer = L.tileLayer.provider(initialProviderName, {
         opacity: parseFloat($("#basemapOpacity").val()),
-        edgeBufferTiles: 1
+        edgeBufferTiles: 1,
+        key: isCartoDB ? CARTODB_API_KEY : null
     });
     backgroundTileLayer.addTo(map);
     backgroundTileLayer.bringToBack();

static/js/spots.js

@@ -398,7 +398,7 @@ function createNewTableRowsForSpot(s, highlightNew) {
         $tr.append(`<td class='nowrap hideonmobile'>${distanceText}</td>`);
     }
     if (showType) {
-        $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid ${sigToIcon(s["sig"], "fa-tower-cell")}'></i></span> ${typeText}</td>`);
+        $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid ${sigToIcon(s["sig"], options["sigs"], "fa-tower-cell")}'></i></span> ${typeText}</td>`);
     }
     if (showRef) {
         $tr.append(`<td class='hideonmobile' style='max-width: 11em;'>${sig_refs}</td>`);
@@ -430,7 +430,7 @@ function createNewTableRowsForSpot(s, highlightNew) {
     const $td2 = $("<td colspan='100'>");
     const $td2floatleft = $(`<div style="float: left;">`);
     if (showType) {
-        $td2floatleft.append(`<span class='icon-wrapper'><i class='fa-solid ${sigToIcon(s["sig"], "fa-tower-cell")}'></i></span> ${typeText} `);
+        $td2floatleft.append(`<span class='icon-wrapper'><i class='fa-solid ${sigToIcon(s["sig"], options["sigs"], "fa-tower-cell")}'></i></span> ${typeText} `);
     }
     if (showRef) {
         $td2floatleft.append(`${sig_refs} `);

static/js/spotsbandsandmap.js

@@ -40,14 +40,23 @@ function setHamHFBandToggles() {
 
 // Generate SIGs filter card. This one is also a special case.
 function generateSIGsMultiToggleFilterCard(sig_options) {
-    const $grid = $('<div class="row row-cols-2 row-cols-xxl-3 g-1 mb-1">');
+    const $grid1 = $('<div class="row row-cols-2 g-1 mb-1">');
-    sig_options.forEach(o => {
+    $("#sig-options").append('<strong>Worldwide</strong>');
+    sig_options.filter(o => o["region_flag"] == null).forEach(o => {
         const domSafeName = o["name"].replace(/^[^A-Za-z0-9]+|[^\w]+/gi, "");
-        $grid.append(`<div class="col"><div class="form-check"><input type="checkbox" class="form-check-input filter-button-sig storeable-checkbox" id="filter-button-sig-${domSafeName}" value="${o['name']}" autocomplete="off" onClick="filtersUpdated()" checked><label class="form-check-label" id="filter-button-label-sig-${domSafeName}" for="filter-button-sig-${domSafeName}" title="${o['description']}"><i class="fa-solid ${sigToIcon(o['name'], 'fa-tower-cell')}"></i> ${o['name']}</label></div></div>`);
+        $grid1.append(`<div class="col"><div class="form-check"><input type="checkbox" class="form-check-input filter-button-sig storeable-checkbox" id="filter-button-sig-${domSafeName}" value="${o['name']}" autocomplete="off" onClick="filtersUpdated()" checked><label class="form-check-label" id="filter-button-label-sig-${domSafeName}" for="filter-button-sig-${domSafeName}" title="${o['description']}"><i class="fa-solid ${o['icon']}"></i> ${o['name']}</label></div></div>`);
     });
+    $("#sig-options").append($grid1);
+    const $grid2 = $('<div class="row row-cols-2 g-1 mb-1">');
+    $("#sig-options").append('<strong>Regional</strong>');
+    sig_options.filter(o => o["region_flag"] != null).forEach(o => {
+        const domSafeName = o["name"].replace(/^[^A-Za-z0-9]+|[^\w]+/gi, "");
+        $grid2.append(`<div class="col"><div class="form-check"><input type="checkbox" class="form-check-input filter-button-sig storeable-checkbox" id="filter-button-sig-${domSafeName}" value="${o['name']}" autocomplete="off" onClick="filtersUpdated()" checked><label class="form-check-label" id="filter-button-label-sig-${domSafeName}" for="filter-button-sig-${domSafeName}" title="${o['description']}"><i class="fa-solid ${o['icon']}"></i> ${o['name']} ${o['region_flag']}</label></div></div>`);
+    });
+    $("#sig-options").append($grid2);
+    $("#sig-options").append('<strong>Other</strong>');
     // Bonus "NO_SIG" / "General DX" option
-    $grid.append(`<div class="w-100"><div class="form-check"><input type="checkbox" class="form-check-input filter-button-sig storeable-checkbox" id="filter-button-sig-NO_SIG" value="NO_SIG" autocomplete="off" onClick="filtersUpdated()" checked><label class="form-check-label" id="filter-button-label-sig-NO_SIG" for="filter-button-sig-NO_SIG"><i class="fa-solid fa-tower-cell"></i> General DX</label></div></div>`);
+    $("#sig-options").append(`<div class="w-100"><div class="form-check"><input type="checkbox" class="form-check-input filter-button-sig storeable-checkbox" id="filter-button-sig-NO_SIG" value="NO_SIG" autocomplete="off" onClick="filtersUpdated()" checked><label class="form-check-label" id="filter-button-label-sig-NO_SIG" for="filter-button-sig-NO_SIG"><i class="fa-solid fa-tower-cell"></i> General DX</label></div></div>`);
-    $("#sig-options").append($grid);
     $("#sig-options").append(`<div class="mt-1"><a href="#" onclick="toggleFilterButtons('sig', true); return false;">All</a> &nbsp; <a href="#" onclick="toggleFilterButtons('sig', false); return false;">None</a></div>`);
 }
 
@@ -64,7 +73,7 @@ function generateModesMultiToggleFilterCard(mode_options) {
 
 // Set the mode toggles that relate to Analog Voice.
 function setVoiceModeToggles() {
-    const modes = ["PHONE", "SSB", "LSB", "USB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "M17"];
+    const modes = ["PHONE", "SSB", "AM", "FM", "DV"];
     $(".filter-button-mode").each(function () {
         $(this).prop('checked', modes.includes($(this).val().replace("filter-button-mode-", "")));
     });

static/js/ui-ham.js

@@ -323,121 +323,3 @@ function bandToContrastColor(band) {
     const lum = 0.2126 * rgb[0] + 0.7152 * rgb[1] + 0.0722 * rgb[2];
     return (lum > 128) ? "#000000" : "#ffffff";
 }
-
-const MODE_TYPE_COLOR_SCHEMES = {
-    "CW": "red",
-    "PHONE": "green",
-    "DATA": "blue"
-}
-
-// Mode type (CW, PHONE, DATA) to colour. If the mode type is unknown, black will be returned.
-function modeTypeToColor(modeType) {
-    let col = (modeType != null) ? MODE_TYPE_COLOR_SCHEMES[modeType.toUpperCase()] : null;
-    if (col) {
-        return col;
-    } else {
-        return "#000000";
-    }
-}
-
-const SIG_ICONS = {
-    "POTA": "fa-tree",
-    "SOTA": "fa-mountain-sun",
-    "WWFF": "fa-seedling",
-    "GMA": "fa-person-hiking",
-    "GMA Island": "fa-person-hiking",
-    "WWBOTA": "fa-radiation",
-    "HEMA": "fa-mound",
-    "IOTA": "fa-book-atlas",
-    "MOTA": "fa-fan",
-    "ARLHS": "fa-house-flood-water",
-    "ILLW": "fa-house-flood-water",
-    "SIOTA": "fa-wheat-awn",
-    "WCA": "fa-chess-rook",
-    "ZLOTA": "fa-kiwi-bird",
-    "WOTA": "fa-w",
-    "BOTA": "fa-umbrella-beach",
-    "KRMNPA": "fa-earth-oceania",
-    "SANPCPA": "fa-earth-oceania",
-    "LLOTA": "fa-water",
-    "Towers": "fa-tower-observation",
-    "WAB": "fa-table-cells-large",
-    "WAI": "fa-table-cells-large",
-    "DME": "fa-building",
-    "Tiles": "fa-square",
-    "Toilets": "fa-toilet"
-}
-
-const SIG_NAMES = {
-    "POTA": "Parks on the Air",
-    "SOTA": "Summits on the Air",
-    "WWFF": "Worldwide Flora & Fauna",
-    "GMA": "Global Mountain Activity",
-    "GMA Island": "Global Mountain Activity - Islands",
-    "WWBOTA": "Bunkers on the Air",
-    "HEMA": "Humps Excluding Marilyns Award",
-    "IOTA": "Islands on the Air",
-    "MOTA": "Mills on the Air",
-    "ARLHS": "Amateur Radio Lighthouse Society",
-    "ILLW": "International Lighthouse Lightship Weekend",
-    "SIOTA": "Silos on the Air",
-    "WCA": "World Castles Award",
-    "ZLOTA": "New Zealand on the Air",
-    "WOTA": "Wainwrights on the Air",
-    "BOTA": "Beaches on the Air",
-    "KRMNPA": "Keith Roget Memorial National Parks Award",
-    "SANPCPA": "South Australia National Parks and Conservation Parks Award",
-    "LLOTA": "Lagos y Lagunas on the Air",
-    "WWTOTA": "Towers on the Air",
-    "WAB": "Worked All Britain",
-    "WAI": "Worked All Ireland",
-    "Tiles": "Tiles on the Air",
-    "TOTA": "Toilets on the Air"
-}
-
-// Get the Font Awesome icon for a given SIG. If the SIG is unknown, the provided default symbol will be returned
-function sigToIcon(sig, defaultIcon) {
-    let col = (sig != null) ? SIG_ICONS[sig] : null;
-    if (col) {
-        return col;
-    } else {
-        let col = (sig != null) ? SIG_ICONS[sig.toUpperCase()] : null;
-        if (col) {
-            return col;
-        } else {
-            return defaultIcon;
-        }
-    }
-}
-
-// Get the full name for a given SIG abbreviation. If the SIG is unknown, an empty string will be returned.
-function sigToName(sig) {
-    let col = (sig != null) ? SIG_NAMES[sig] : null;
-    if (col) {
-        return col;
-    } else {
-        let col = (sig != null) ? SIG_NAMES[sig.toUpperCase()] : null;
-        if (col) {
-            return col;
-        } else {
-            return "";
-        }
-    }
-}
-
-// Get the list of known SIGs
-function getKnownSIGs() {
-    return Array.from(Object.keys(SIG_ICONS));
-}
-
-// Format a Maidenhead grid with alternating alphabetic blocks in lower case
-function formatGrid(grid) {
-    grid = grid.toUpperCase();
-    if (grid.length >= 6) {
-        grid = grid.substring(0, 4) + grid.substring(4, 6).toLowerCase() + grid.substring(6);
-    }
-    if (grid.length >= 12) {
-        grid = grid.substring(0, 10) + grid.substring(10, 12).toLowerCase() + grid.substring(14);
-    }
-    return grid;
-}

static/vendor/js/leaflet-providers-2.0.0.js

@@ -801,12 +801,13 @@
 			}
 		},
 		CartoDB: {
-			url: 'https://{s}.basemaps.cartocdn.com/{variant}/{z}/{x}/{y}{r}.png',
+			url: 'https://{s}.basemaps.cartocdn.com/{variant}/{z}/{x}/{y}{r}.png?key={key}',
 			options: {
 				attribution: '{attribution.OpenStreetMap} &copy; <a href="https://carto.com/attributions">CARTO</a>',
 				subdomains: 'abcd',
 				maxZoom: 20,
-				variant: 'light_all'
+				variant: 'light_all',
+				key: '<insert your API key here>'
 			},
 			variants: {
 				Positron: 'light_all',

templates/about.html

@@ -101,12 +101,13 @@
         (MOTA), the Amateur Radio Lighthouse Society (ARLHS), International Lighthouse Lightship Weekend (ILLW), Silos
         on the Air (SIOTA), World Castles Award (WCA), New Zealand on the Air (ZLOTA), Keith Roget Memorial National
         Parks Award (KRMNPA), South Australia National Parks and Conservation Parks Award (SANPCPA), Wainwrights on the
-        Air (WOTA), Beaches on the Air (BOTA), Lagos y Lagunas On the Air (LLOTA), Towers on the Air (WWTOTA), Tiles on
+        Air (WOTA), Beaches on the Air (BOTA), Lagos y Lagunas On the Air (LLOTA), Towers on the Air, Tiles on
-        the Air, Worked All Britain (WAB), Worked All Ireland (WAI), el Diploma Municipios de España (DME) and Toilets
+        the Air, Worked All Britain (WAB), Worked All Ireland (WAI), el Diploma Municipios de España (DME), el Diploma
-        on the Air (TOTA).</p>
+        Faros de España (FEA), il Diploma Teatri Musei e Belle Arti (DTMBA), British Inland Waterways on the Air
-    <p>As of the time of writing in June 2026, I think Spothole captures essentially all outdoor radio programmes
+        (BIWOTA), Castles on the Air (COTA), Polish Gmina Award (PGA), and Toilets on the Air.</p>
-        that have a defined reference list, and almost certainly those that have a spotting/alerting API. If you know of
+    <p>As of the time of writing in August 2026, I think Spothole captures most outdoor radio programmes that have a
-        one I've missed, please let me know!</p>
+        defined, downloadable reference list, and almost certainly those that have a spotting/alerting API. If you know
+        of one I've missed, please let me know!</p>
     <h4 class="mt-4">Why can I filter spots by both SIG and Source? Isn't that basically the same thing?</h4>
     <p>Mostly, but not quite. While POTA spots generally come from the POTA source and so on, there are a few
         exceptions:</p>

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1787900350"></script>
+<script src="/static/js/add-spot.js?v=1788209684"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -84,7 +84,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1787900351"></script>
+<script src="/static/js/alerts.js?v=1788209684"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -76,8 +76,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1787900350"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1788209684"></script>
-<script src="/static/js/bands.js?v=1787900350"></script>
+<script src="/static/js/bands.js?v=1788209684"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1787900350" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1788209684" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -15,10 +15,10 @@
   window.fetchEventSource = fetchEventSource;
 </script>
 
-<script src="/static/js/utils.js?v=1787900350"></script>
+<script src="/static/js/utils.js?v=1788209684"></script>
-<script src="/static/js/ui-ham.js?v=1787900350"></script>
+<script src="/static/js/ui-ham.js?v=1788209684"></script>
-<script src="/static/js/geo.js?v=1787900350"></script>
+<script src="/static/js/geo.js?v=1788209684"></script>
-<script src="/static/js/common.js?v=1787900350"></script>
+<script src="/static/js/common.js?v=1788209684"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1787900350"></script>
+<script src="/static/js/conditions.js?v=1788209684"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -109,8 +109,12 @@
 <script src="/static/vendor/js/leaflet-cqzones.js"></script>
 <script src="/static/vendor/js/leaflet-workedallbritainireland.js" type="module"></script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1787900351"></script>
+<script>
-<script src="/static/js/map.js?v=1787900351"></script>
+    const CARTODB_API_KEY = "{{ web_ui_options.get('cartodb_api_key', '') }}";
+</script>
+
+<script src="/static/js/spotsbandsandmap.js?v=1788209683"></script>
+<script src="/static/js/map.js?v=1788209683"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -113,8 +113,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1787900350"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1788209683"></script>
-<script src="/static/js/spots.js?v=1787900350"></script>
+<script src="/static/js/spots.js?v=1788209683"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -86,7 +86,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1787900350"></script>
+<script src="/static/js/status.js?v=1788209684"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");