Ensure "RTT" as a mode is understood as "RTTY" and similar.

config-example.yml

@@ -90,15 +90,22 @@ spot-providers:
     name: "RBN CW/RTTY"
     enabled: false
     port: 7000
+    # This setting doesn't affect the spot provider itself, or anything in the back-end of Spothole, just the web UI.
+    # By default spots from all enabled providers will be shown in the web UI. However, you might want RBN data to be
+    # received by Spothole but not shown on the web UI unless the user explicitly turns it on. For that behaviour,
+    # set enabled to true, but enabled-by-default-in-web-ui to false.
+    enabled-by-default-in-web-ui: false
   -
     class: "RBN"
     name: "RBN FT8"
     enabled: false
     port: 7001
+    enabled-by-default-in-web-ui: false
   -
     class: "UKPacketNet"
     name: "UK Packet Radio Net"
     enabled: false
+    enabled-by-default-in-web-ui: false
   -
     class: "XOTA"
     name: "39C3 TOTA"

core/config.py

@@ -5,7 +5,8 @@ import yaml
 
 # Check you have a config file
 if not os.path.isfile("config.yml"):
-    logging.error("Your config file is missing. Ensure you have copied config-example.yml to config.yml and updated it according to your needs.")
+    logging.error(
+        "Your config file is missing. Ensure you have copied config-example.yml to config.yml and updated it according to your needs.")
     exit()
 
 # Load config
@@ -17,4 +18,9 @@ MAX_ALERT_AGE = config["max-alert-age-sec"]
 SERVER_OWNER_CALLSIGN = config["server-owner-callsign"]
 WEB_SERVER_PORT = config["web-server-port"]
 ALLOW_SPOTTING = config["allow-spotting"]
-WEB_UI_OPTIONS = config["web-ui-options"]
+WEB_UI_OPTIONS = config["web-ui-options"]
+
+# For ease of config, each spot provider owns its own config about whether it should be enabled by default in the web UI
+# but for consistency we provide this to the front-end in web-ui-options because it has no impact outside of the web UI.
+WEB_UI_OPTIONS["spot-providers-enabled-by-default"] = [p["name"] for p in config["spot-providers"] if p["enabled"] and (
+            "enabled-by-default-in-web-ui" not in p or p["enabled-by-default-in-web-ui"] == True)]

core/constants.py

@@ -4,7 +4,7 @@ from data.sig import SIG
 
 # General software
 SOFTWARE_NAME = "Spothole by M0TRT"
-SOFTWARE_VERSION = "1.1-pre"
+SOFTWARE_VERSION = "1.1.1"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + ", v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
@@ -36,10 +36,25 @@ SIGS = [
 # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
 CW_MODES = ["CW"]
 PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "M17"]
-DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "BPSK", "PSK", "PSK31", "BPSK31", "OLIVIA", "MFSK", "MFSK32", "PKT", "MSK144"]
+DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "PSK", "OLIVIA", "PKT", "MSK144"]
 ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
 MODE_TYPES = ["CW", "PHONE", "DATA"]
 
+# Mode aliases. Sometimes we get spots with a mode described in a different way that is effectively the same as a mode
+# we already know, or we want to normalise things for consistency. The lookup table for this is here. Incoming spots
+# that match a key in this table will be converted to the corresponding value, so only the modes above will actually be
+# present in the spots.
+MODE_ALIASES = {
+    "RTT": "RTTY",
+    "BPSK": "PSK",
+    "PSK31": "PSK",
+    "BPSK31": "PSK",
+    "MFSK": "FSK",
+    "MFSK32": "FSK",
+    "DIGI": "DATA",
+    "DIGITAL": "DATA"
+}
+
 # Band definitions
 BANDS = [
     Band(name="2200m", start_freq=135700, end_freq=137800),

core/lookup_helper.py

@@ -16,7 +16,7 @@ from requests_cache import CachedSession
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.config import config
 from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
-    HTTP_HEADERS, HAMQTH_PRG
+    HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
 
 
 # Singleton class that provides lookup functionality.
@@ -160,6 +160,9 @@ class LookupHelper:
         for mode in ALL_MODES:
             if mode in comment.upper():
                 return mode
+        for mode in MODE_ALIASES.keys():
+            if mode in comment.upper():
+                return MODE_ALIASES[mode]
         return None
 
     # Infer a "mode family" from a mode.

data/spot.py

@@ -10,6 +10,7 @@ import pytz
 from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 
 from core.config import MAX_SPOT_AGE
+from core.constants import MODE_ALIASES
 from core.lookup_helper import lookup_helper
 from core.sig_utils import populate_sig_ref_info, ANY_SIG_REGEX, get_ref_regex_for_sig
 from data.sig_ref import SIGRef
@@ -213,10 +214,9 @@ class Spot:
             self.mode = lookup_helper.infer_mode_from_frequency(self.freq)
             self.mode_source = "BANDPLAN"
 
-        # Normalise "generic digital" modes. "DIGITAL", "DIGI" and "DATA" are just the same thing with no extra
+        # Normalise mode if necessary.
-        # information, so standardise on "DATA"
+        if self.mode in MODE_ALIASES:
-        if self.mode == "DIGI" or self.mode == "DIGITAL":
+            self.mode = MODE_ALIASES[self.mode]
-            self.mode = "DATA"
 
         # Mode type from mode
         if self.mode and not self.mode_type:

server/handlers/api/options.py

@@ -40,6 +40,7 @@ class APIOptionsHandler(tornado.web.RequestHandler):
         # one of our proviers.
         if ALLOW_SPOTTING:
             options["spot_sources"].append("API")
+            options["web-ui-options"]["spot-providers-enabled-by-default"].append("API")
 
         self.write(json.dumps(options, default=serialize_everything))
         self.set_status(200)

spotproviders/sota.py

@@ -46,7 +46,7 @@ class SOTA(HTTPSpotProvider):
                             comment=source_spot["comments"],
                             sig="SOTA",
                             sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA", name=source_spot["summitName"], activation_score=source_spot["points"])],
-                            time=datetime.fromisoformat(source_spot["timeStamp"]).timestamp())
+                            time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp())
 
                 # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
                 # that for us.

spotproviders/wwbota.py

@@ -30,7 +30,7 @@ class WWBOTA(SSESpotProvider):
                     comment=source_spot["comment"],
                     sig="WWBOTA",
                     sig_refs=refs,
-                    time=datetime.fromisoformat(source_spot["time"]).timestamp(),
+                    time=datetime.fromisoformat(source_spot["time"].replace("Z", "+00:00")).timestamp(),
                     # WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
                     # now, we will just pick the first one to use as our grid, latitude and longitude.
                     dx_grid=source_spot["references"][0]["locator"],

spotproviders/zlota.py

@@ -35,7 +35,7 @@ class ZLOTA(HTTPSpotProvider):
                         comment=source_spot["comments"],
                         sig="ZLOTA",
                         sig_refs=[SIGRef(id=source_spot["reference"], sig="ZLOTA", name=source_spot["name"])],
-                        time=datetime.fromisoformat(source_spot["referenced_time"]).astimezone(pytz.UTC).timestamp())
+                        time=datetime.fromisoformat(source_spot["referenced_time"].replace("Z", "+00:00")).astimezone(pytz.UTC).timestamp())
 
             new_spots.append(spot)
         return new_spots

templates/about.html

@@ -63,7 +63,7 @@
     <p>This software is dedicated to the memory of Tom G1PJB, SK, a friend and colleague who sadly passed away around the time I started writing it in Autumn 2025. I was looking forward to showing it to you when it was done.</p>
 </div>
 
-<script src="/js/common.js?v=5"></script>
+<script src="/js/common.js?v=6"></script>
 <script>$(document).ready(function() { $("#nav-link-about").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/add_spot.html

@@ -69,8 +69,8 @@
 
 </div>
 
-<script src="/js/common.js?v=5"></script>
+<script src="/js/common.js?v=6"></script>
-<script src="/js/add-spot.js?v=5"></script>
+<script src="/js/add-spot.js?v=6"></script>
 <script>$(document).ready(function() { $("#nav-link-add-spot").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/alerts.html

@@ -168,8 +168,8 @@
 
 </div>
 
-<script src="/js/common.js?v=5"></script>
+<script src="/js/common.js?v=6"></script>
-<script src="/js/alerts.js?v=5"></script>
+<script src="/js/alerts.js?v=6"></script>
 <script>$(document).ready(function() { $("#nav-link-alerts").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/bands.html

@@ -134,9 +134,9 @@
 
 </div>
 
-<script src="/js/common.js?v=5"></script>
+<script src="/js/common.js?v=6"></script>
-<script src="/js/spotsbandsandmap.js?v=5"></script>
+<script src="/js/spotsbandsandmap.js?v=6"></script>
-<script src="/js/bands.js?v=5"></script>
+<script src="/js/bands.js?v=6"></script>
 <script>$(document).ready(function() { $("#nav-link-bands").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/base.html

@@ -46,10 +46,10 @@
             crossorigin="anonymous"></script>
     <script src="https://cdn.jsdelivr.net/npm/tinycolor2@1.6.0/cjs/tinycolor.min.js"></script>
 
-    <script src="https://misc.ianrenton.com/jsutils/utils.js?v=5"></script>
+    <script src="https://misc.ianrenton.com/jsutils/utils.js?v=6"></script>
-    <script src="https://misc.ianrenton.com/jsutils/storage.js?v=5"></script>
+    <script src="https://misc.ianrenton.com/jsutils/storage.js?v=6"></script>
-    <script src="https://misc.ianrenton.com/jsutils/ui-ham.js?v=5"></script>
+    <script src="https://misc.ianrenton.com/jsutils/ui-ham.js?v=6"></script>
-    <script src="https://misc.ianrenton.com/jsutils/geo.js?v=5"></script>
+    <script src="https://misc.ianrenton.com/jsutils/geo.js?v=6"></script>
 
 </head>
 <body>

templates/map.html

@@ -152,9 +152,9 @@
 <script src="https://cdn.jsdelivr.net/npm/leaflet.geodesic"></script>
 <script src="https://cdn.jsdelivr.net/npm/@joergdietrich/leaflet.terminator@1.1.0/L.Terminator.min.js"></script>
 
-<script src="/js/common.js?v=5"></script>
+<script src="/js/common.js?v=6"></script>
-<script src="/js/spotsbandsandmap.js?v=5"></script>
+<script src="/js/spotsbandsandmap.js?v=6"></script>
-<script src="/js/map.js?v=5"></script>
+<script src="/js/map.js?v=6"></script>
 <script>$(document).ready(function() { $("#nav-link-map").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/spots.html

@@ -223,9 +223,9 @@
 
 </div>
 
-<script src="/js/common.js?v=5"></script>
+<script src="/js/common.js?v=6"></script>
-<script src="/js/spotsbandsandmap.js?v=5"></script>
+<script src="/js/spotsbandsandmap.js?v=6"></script>
-<script src="/js/spots.js?v=5"></script>
+<script src="/js/spots.js?v=6"></script>
 <script>$(document).ready(function() { $("#nav-link-spots").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/status.html

@@ -3,8 +3,8 @@
 
 <div id="status-container" class="row row-cols-1 row-cols-md-4 g-4 mt-4"></div>
 
-<script src="/js/common.js?v=5"></script>
+<script src="/js/common.js?v=6"></script>
-<script src="/js/status.js?v=5"></script>
+<script src="/js/status.js?v=6"></script>
 <script>$(document).ready(function() { $("#nav-link-status").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

webassets/.idea/.gitignore

@@ -0,0 +1,10 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Ignored default folder with query files
+/queries/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/

webassets/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$/.." vcs="Git" />
+  </component>
+</project>

webassets/apidocs/openapi.yml

@@ -1,4 +1,4 @@
-openapi: 3.0.4
+openapi: 3.1.0
 info:
   title: Spothole API
   description: |-
@@ -557,6 +557,12 @@ paths:
                         type: integer
                         example: 30
                         description: The suggested default "maximum spot age" that the web UI should retrieve from the API
+                      spot-providers-enabled-by-default:
+                        type: array
+                        description: A list of the spot providers that should be enabled in the web UI on first load, if the user hasn't already got a localStorage setting that sets their preference. This is to allow some high-volume providers like RBN to be enabled in Spothole's back-end and displayable in the web UI if the user wants, but by default the experience will not include them.
+                        items:
+                          type: string
+                          example: "POTA"
                       alert-count:
                         type: array
                         description: An array of suggested "alert counts" that the web UI can retrieve from the API
@@ -866,12 +872,9 @@ components:
         - SSTV
         - JS8
         - HELL
-        - BPSK
-        - PSK
-        - BPSK31
         - OLIVIA
-        - MFSK
+        - PSK
-        - MFSK32
+        - FSK
         - PKT
         - MSK144
     example: SSB

webassets/js/bands.js

@@ -252,7 +252,7 @@ function loadOptions() {
         generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
         generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
         generateMultiToggleFilterCard("#mode-options", "mode_type", options["mode_types"]);
-        generateMultiToggleFilterCard("#source-options", "source", options["spot_sources"]);
+        generateSourcesMultiToggleFilterCard(options["spot_sources"], options["web-ui-options"]["spot-providers-enabled-by-default"]);
 
         // Load URL params. These may select things from the various filter & display options, so the function needs
         // to be called after these are set up, but if the URL params ask for "embedded mode", this will suppress

webassets/js/map.js

@@ -184,7 +184,7 @@ function loadOptions() {
         generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
         generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
         generateMultiToggleFilterCard("#mode-options", "mode_type", options["mode_types"]);
-        generateMultiToggleFilterCard("#source-options", "source", options["spot_sources"]);
+        generateSourcesMultiToggleFilterCard(options["spot_sources"], options["web-ui-options"]["spot-providers-enabled-by-default"]);
 
         // Load URL params. These may select things from the various filter & display options, so the function needs
         // to be called after these are set up, but if the URL params ask for "embedded mode", this will suppress

webassets/js/spots.js

@@ -422,7 +422,7 @@ function loadOptions() {
         generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
         generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
         generateMultiToggleFilterCard("#mode-options", "mode_type", options["mode_types"]);
-        generateMultiToggleFilterCard("#source-options", "source", options["spot_sources"]);
+        generateSourcesMultiToggleFilterCard(options["spot_sources"], options["web-ui-options"]["spot-providers-enabled-by-default"]);
 
         // Load URL params. These may select things from the various filter & display options, so the function needs
         // to be called after these are set up, but if the URL params ask for "embedded mode", this will suppress

webassets/js/spotsbandsandmap.js

@@ -6,10 +6,9 @@ var spots = []
 function addBandToggleColourCSS(band_options) {
     var $style = $('<style>');
     band_options.forEach(o => {
-        // CSS doesn't like IDs with decimal points in, so we need to replace that
+        var domSafeName = o["name"].replace(/^[^A-Za-z0-9]+|[^\w]+/gi, "");
-        var cssFormattedBandName = o['name'] ? o['name'].replace('.', 'p') : "unknown";
+        $style.append(`#filter-button-label-band-${domSafeName} { border-color: ${bandToColor(o['name'])}; color: var(--bs-primary);}`);
-        $style.append(`#filter-button-label-band-${cssFormattedBandName} { border-color: ${bandToColor(o['name'])}; color: var(--bs-primary);}`);
+        $style.append(`.btn-check:checked + #filter-button-label-band-${domSafeName} { background-color: ${bandToColor(o['name'])}; color: ${bandToContrastColor(o['name'])};}`);
-        $style.append(`.btn-check:checked + #filter-button-label-band-${cssFormattedBandName} { background-color: ${bandToColor(o['name'])}; color: ${bandToContrastColor(o['name'])};}`);
     });
     $('html > head').append($style);
 }
@@ -18,10 +17,8 @@ function addBandToggleColourCSS(band_options) {
 function generateBandsMultiToggleFilterCard(band_options) {
     // Create a button for each option
     band_options.forEach(o => {
-        // CSS doesn't like IDs with decimal points in, so we need to replace that in the same way as when we originally
+        var domSafeName = o["name"].replace(/^[^A-Za-z0-9]+|[^\w]+/gi, "");
-        // queried the options endpoint and set our CSS.
+        $("#band-options").append(`<input type="checkbox" class="btn-check filter-button-band storeable-checkbox" name="options" id="filter-button-band-${domSafeName}" value="${o['name']}" autocomplete="off" onClick="filtersUpdated()" checked><label class="btn btn-outline" id="filter-button-label-band-${domSafeName}" for="filter-button-band-${domSafeName}">${o['name']}</label> `);
-        var cssFormattedBandName = o['name'] ? o['name'].replace('.', 'p') : "unknown";
-        $("#band-options").append(`<input type="checkbox" class="btn-check filter-button-band storeable-checkbox" name="options" id="filter-button-band-${cssFormattedBandName}" value="${o['name']}" autocomplete="off" onClick="filtersUpdated()" checked><label class="btn btn-outline" id="filter-button-label-band-${cssFormattedBandName}" for="filter-button-band-${cssFormattedBandName}">${o['name']}</label> `);
     });
     // Create All/None/Ham HF buttons
     $("#band-options").append(` <span style="display: inline-block"><button id="filter-button-band-all" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('band', true);">All</button> <button id="filter-button-band-none" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('band', false);">None</button> <button id="filter-button-band-none" type="button" class="btn btn-outline-secondary" onclick="setHamHFBandToggles();">Ham HF</button></span>`);
@@ -41,7 +38,8 @@ function setHamHFBandToggles() {
 function generateSIGsMultiToggleFilterCard(sig_options) {
     // Create a button for each option
     sig_options.forEach(o => {
-        $("#sig-options").append(`<input type="checkbox" class="btn-check filter-button-sig storeable-checkbox" name="options" id="filter-button-sig-${o['name']}" value="${o['name']}" autocomplete="off" onClick="filtersUpdated()" checked><label class="btn btn-outline-primary" id="filter-button-label-sig-${o['name']}" for="filter-button-sig-${o['name']}" title="${o['description']}"><i class="fa-solid ${sigToIcon(o['name'], 'fa-tower-cell')}"></i> ${o['name']}</label> `);
+        var domSafeName = o["name"].replace(/^[^A-Za-z0-9]+|[^\w]+/gi, "");
+        $("#sig-options").append(`<input type="checkbox" class="btn-check filter-button-sig storeable-checkbox" name="options" id="filter-button-sig-${domSafeName}" value="${o['name']}" autocomplete="off" onClick="filtersUpdated()" checked><label class="btn btn-outline-primary" id="filter-button-label-sig-${domSafeName}" for="filter-button-sig-${domSafeName}" title="${o['description']}"><i class="fa-solid ${sigToIcon(o['name'], 'fa-tower-cell')}"></i> ${o['name']}</label> `);
     });
     // Create a bonus "NO_SIG" / "General DX" option
     $("#sig-options").append(`<input type="checkbox" class="btn-check filter-button-sig storeable-checkbox" name="options" id="filter-button-sig-NO_SIG" value="NO_SIG" autocomplete="off" onClick="filtersUpdated()" checked><label class="btn btn-outline-primary" id="filter-button-label-sig-NO_SIG" for="filter-button-sig-NO_SIG"><i class="fa-solid fa-tower-cell"></i> General DX</label> `);
@@ -49,6 +47,20 @@ function generateSIGsMultiToggleFilterCard(sig_options) {
     $("#sig-options").append(` <span style="display: inline-block"><button id="filter-button-sig-all" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('sig', true);">All</button>&nbsp;<button id="filter-button-sig-none" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('sig', false);">None</button></span>`);
 }
 
+// Generate Sources filter card. This one is a minor special case as we create the buttons in the normal way, but then
+// set which ones are enabled by default based on config rather than having them all enabled by default. We also sanitise
+// names here for HTML elements.
+function generateSourcesMultiToggleFilterCard(source_options, sources_enabled_by_default) {
+    // Create a button for each option
+    source_options.forEach(o => {
+        var enable = sources_enabled_by_default.includes(o);
+        var domSafeName = o.replace(/^[^A-Za-z0-9]+|[^\w]+/gi, "");
+        $("#source-options").append(`<input type="checkbox" class="btn-check filter-button-source storeable-checkbox" name="options" id="filter-button-source-${domSafeName}" value="${o}" autocomplete="off" onClick="filtersUpdated()" ${enable ? "checked" : ""}><label class="btn btn-outline-primary" for="filter-button-source-${domSafeName}">${o}</label> `);
+    });
+    // Create All/None buttons
+    $("#source-options").append(` <span style="display: inline-block"><button id="filter-button-source-all" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('source', true);">All</button>&nbsp;<button id="filter-button-source-none" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('source', false);">None</button></span>`);
+}
+
 // Method called when any filter is changed to reload the spots and persist the filter settings.
 function filtersUpdated() {
     loadSpots();