Fix a bug where the "last updated time"/"last spot time" of providers that have never updated would be sent as a large negative number and represented on the web UI as e.g. "2026 years ago".

alertproviders/bota.py

@@ -20,7 +20,9 @@ class BOTA(HTTPAlertProvider):
         new_alerts = []
         # Find the table of upcoming alerts
         bs = BeautifulSoup(http_response.content.decode(), features="lxml")
-        tbody = bs.body.find('div', attrs={'class': 'view-activations-public'}).find('table', attrs={'class': 'views-table'}).find('tbody')
+        forthcoming_activations_div = bs.body.find('div', attrs={'class': 'view-activations-public'})
+        if forthcoming_activations_div:
+            tbody = forthcoming_activations_div.find('table', attrs={'class': 'views-table'}).find('tbody')
             for row in tbody.find_all('tr'):
                 cells = row.find_all('td')
                 first_cell_text = str(cells[0].find('a').contents[0]).strip()

core/constants.py

@@ -4,7 +4,7 @@ from data.sig import SIG
 
 # General software
 SOFTWARE_NAME = "Spothole by M0TRT"
-SOFTWARE_VERSION = "1.1"
+SOFTWARE_VERSION = "1.2-pre"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + ", v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
@@ -36,10 +36,25 @@ SIGS = [
 # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
 CW_MODES = ["CW"]
 PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "M17"]
-DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "BPSK", "PSK", "PSK31", "BPSK31", "OLIVIA", "MFSK", "MFSK32", "PKT", "MSK144"]
+DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "PSK", "OLIVIA", "PKT", "MSK144"]
 ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
 MODE_TYPES = ["CW", "PHONE", "DATA"]
 
+# Mode aliases. Sometimes we get spots with a mode described in a different way that is effectively the same as a mode
+# we already know, or we want to normalise things for consistency. The lookup table for this is here. Incoming spots
+# that match a key in this table will be converted to the corresponding value, so only the modes above will actually be
+# present in the spots.
+MODE_ALIASES = {
+    "RTT": "RTTY",
+    "BPSK": "PSK",
+    "PSK31": "PSK",
+    "BPSK31": "PSK",
+    "MFSK": "FSK",
+    "MFSK32": "FSK",
+    "DIGI": "DATA",
+    "DIGITAL": "DATA"
+}
+
 # Band definitions
 BANDS = [
     Band(name="2200m", start_freq=135700, end_freq=137800),

core/lookup_helper.py

@@ -16,7 +16,7 @@ from requests_cache import CachedSession
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.config import config
 from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
-    HTTP_HEADERS, HAMQTH_PRG
+    HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
 
 
 # Singleton class that provides lookup functionality.
@@ -160,6 +160,9 @@ class LookupHelper:
         for mode in ALL_MODES:
             if mode in comment.upper():
                 return mode
+        for mode in MODE_ALIASES.keys():
+            if mode in comment.upper():
+                return MODE_ALIASES[mode]
         return None
 
     # Infer a "mode family" from a mode.
@@ -413,7 +416,12 @@ class LookupHelper:
     # Infer a grid locator from a callsign (using DXCC, probably very inaccurate)
     def infer_grid_from_callsign_dxcc(self, call):
         latlon = self.infer_latlon_from_callsign_dxcc(call)
-        return latlong_to_locator(latlon[0], latlon[1], 8)
+        grid = None
+        try:
+            grid = latlong_to_locator(latlon[0], latlon[1], 8)
+        except:
+            logging.debug("Invalid lat/lon received for DXCC")
+        return grid
 
     # Infer a mode from the frequency (in Hz) according to the band plan. Just a guess really.
     def infer_mode_from_frequency(self, freq):

core/sig_utils.py

@@ -73,9 +73,9 @@ def populate_sig_ref_info(sig_ref):
                 if row["reference"] == ref_id:
                     sig_ref.name = row["name"] if "name" in row else None
                     sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
-                    sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row else None
+                    sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
-                    sig_ref.latitude = float(row["latitude"]) if "latitude" in row else None
+                    sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
-                    sig_ref.longitude = float(row["longitude"]) if "longitude" in row else None
+                    sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row["longitude"] != "-" else None
                     break
         elif sig.upper() == "SIOTA":
             siota_csv_data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
@@ -112,7 +112,10 @@ def populate_sig_ref_info(sig_ref):
                     if asset["code"] == ref_id:
                         sig_ref.name = asset["name"]
                         sig_ref.url = "https://ontheair.nz/assets/ZLI_OT-030" + ref_id.replace("/", "_")
+                        try:
                             sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
+                        except:
+                            logging.debug("Invalid lat/lon received for reference")
                             sig_ref.latitude = asset["y"]
                             sig_ref.longitude = asset["x"]
                         break
@@ -124,9 +127,12 @@ def populate_sig_ref_info(sig_ref):
             ll = wab_wai_square_to_lat_lon(ref_id)
             if ll:
                 sig_ref.name = ref_id
+                try:
                     sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
                     sig_ref.latitude = ll[0]
                     sig_ref.longitude = ll[1]
+                except:
+                    logging.debug("Invalid lat/lon received for reference")
     except:
         logging.warn("Failed to look up sig_ref info for " + sig + " ref " + ref_id + ".")
     return sig_ref

core/status_reporter.py

@@ -47,13 +47,13 @@ class StatusReporter:
         self.status_data["spot_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time else 0,
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
                            "last_spot": p.last_spot_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_spot_time else 0}, self.spot_providers))
+                               tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0}, self.spot_providers))
         self.status_data["alert_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time else 0},
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
                 self.alert_providers))
         self.status_data["cleanup"] = {"status": self.cleanup_timer.status,
                                        "last_ran": self.cleanup_timer.last_cleanup_time.replace(

data/spot.py

@@ -10,6 +10,7 @@ import pytz
 from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 
 from core.config import MAX_SPOT_AGE
+from core.constants import MODE_ALIASES
 from core.lookup_helper import lookup_helper
 from core.sig_utils import populate_sig_ref_info, ANY_SIG_REGEX, get_ref_regex_for_sig
 from data.sig_ref import SIGRef
@@ -213,10 +214,9 @@ class Spot:
             self.mode = lookup_helper.infer_mode_from_frequency(self.freq)
             self.mode_source = "BANDPLAN"
 
-        # Normalise "generic digital" modes. "DIGITAL", "DIGI" and "DATA" are just the same thing with no extra
+        # Normalise mode if necessary.
-        # information, so standardise on "DATA"
+        if self.mode in MODE_ALIASES:
-        if self.mode == "DIGI" or self.mode == "DIGITAL":
+            self.mode = MODE_ALIASES[self.mode]
-            self.mode = "DATA"
 
         # Mode type from mode
         if self.mode and not self.mode_type:
@@ -284,9 +284,13 @@ class Spot:
 
         # DX Grid to lat/lon and vice versa in case one is missing
         if self.dx_grid and not self.dx_latitude:
+            try:
+                print(json.dumps(self))
                 ll = locator_to_latlong(self.dx_grid)
                 self.dx_latitude = ll[0]
                 self.dx_longitude = ll[1]
+            except:
+                logging.debug("Invalid grid received for spot")
         if self.dx_latitude and self.dx_longitude and not self.dx_grid:
             try:
                 self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8)

spotproviders/sota.py

@@ -46,7 +46,7 @@ class SOTA(HTTPSpotProvider):
                             comment=source_spot["comments"],
                             sig="SOTA",
                             sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA", name=source_spot["summitName"], activation_score=source_spot["points"])],
-                            time=datetime.fromisoformat(source_spot["timeStamp"]).timestamp())
+                            time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp())
 
                 # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
                 # that for us.

spotproviders/wwbota.py

@@ -30,7 +30,7 @@ class WWBOTA(SSESpotProvider):
                     comment=source_spot["comment"],
                     sig="WWBOTA",
                     sig_refs=refs,
-                    time=datetime.fromisoformat(source_spot["time"]).timestamp(),
+                    time=datetime.fromisoformat(source_spot["time"].replace("Z", "+00:00")).timestamp(),
                     # WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
                     # now, we will just pick the first one to use as our grid, latitude and longitude.
                     dx_grid=source_spot["references"][0]["locator"],

spotproviders/zlota.py

@@ -35,7 +35,7 @@ class ZLOTA(HTTPSpotProvider):
                         comment=source_spot["comments"],
                         sig="ZLOTA",
                         sig_refs=[SIGRef(id=source_spot["reference"], sig="ZLOTA", name=source_spot["name"])],
-                        time=datetime.fromisoformat(source_spot["referenced_time"]).astimezone(pytz.UTC).timestamp())
+                        time=datetime.fromisoformat(source_spot["referenced_time"].replace("Z", "+00:00")).astimezone(pytz.UTC).timestamp())
 
             new_spots.append(spot)
         return new_spots

templates/spots.html

@@ -17,17 +17,17 @@
             <p class="d-inline-flex gap-1">
                 <span class="btn-group" role="group">
                   <input type="radio" class="btn-check" name="runPause" id="runButton" autocomplete="off" checked>
-                  <label class="btn btn-outline-primary" for="runButton"><i class="fa-solid fa-play"></i> Run</label>
+                    <label class="btn btn-outline-primary" for="runButton"><i class="fa-solid fa-play"></i><span class="hideonmobile"> Run</span></label>
 
                   <input type="radio" class="btn-check" name="runPause" id="pauseButton" autocomplete="off">
-                  <label class="btn btn-outline-primary" for="pauseButton"><i class="fa-solid fa-pause"></i> Pause</label>
+                    <label class="btn btn-outline-primary" for="pauseButton"><i class="fa-solid fa-pause"></i><span class="hideonmobile"> Pause</span></label>
                 </span>
-                <span class="hideonmobile" style="position: relative;">
+                <span style="position: relative;">
                     <i id="searchicon" class="fa-solid fa-magnifying-glass"></i>
                     <input id="search" type="search" class="form-control" oninput="filtersUpdated();" placeholder="Search">
                 </span>
-                <button id="filters-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleFiltersPanel();"><i class="fa-solid fa-filter"></i> Filters</button>
+                <button id="filters-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleFiltersPanel();"><i class="fa-solid fa-filter"></i><span class="hideonmobile"> Filters</span></button>
-                <button id="display-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleDisplayPanel();"><i class="fa-solid fa-desktop"></i> Display</button>
+                <button id="display-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleDisplayPanel();"><i class="fa-solid fa-desktop"></i><span class="hideonmobile"> Display</span></button>
             </p>
         </div>
     </div>

webassets/apidocs/openapi.yml

@@ -1,3 +1,4 @@
+$schema: "https://spec.openapis.org/oas/3.1.0"
 openapi: 3.1.0
 info:
   title: Spothole API
@@ -864,7 +865,6 @@ components:
         - DSTAR
         - C4FM
         - M17
-        - DIGI
         - DATA
         - FT8
         - FT4
@@ -872,12 +872,9 @@ components:
         - SSTV
         - JS8
         - HELL
-        - BPSK
-        - PSK
-        - BPSK31
         - OLIVIA
-        - MFSK
+        - PSK
-        - MFSK32
+        - FSK
         - PKT
         - MSK144
     example: SSB
@@ -1237,11 +1234,11 @@ components:
           example: OK
         last_updated:
           type: number
-          description: The last time at which this provider received data, UTC seconds since UNIX epoch.
+          description: The last time at which this provider received data, UTC seconds since UNIX epoch. If this is zero, the spot provider has never updated.
           example: 1759579508
         last_spot:
           type: number
-          description: The time of the latest spot received by this provider, UTC seconds since UNIX epoch.
+          description: The time of the latest spot received by this provider, UTC seconds since UNIX epoch. If this is zero, the spot provider has never received a spot that was accepted by the system.
           example: 1759579508
 
     AlertProviderStatus:
@@ -1260,7 +1257,7 @@ components:
           example: OK
         last_updated:
           type: number
-          description: The last time at which this provider received data, UTC seconds since UNIX epoch.
+          description: The last time at which this provider received data, UTC seconds since UNIX epoch. If this is zero, the alert provider has never updated.
           example: 1759579508
 
     Band:

webassets/css/style.css

@@ -349,6 +349,9 @@ div.band-spot:hover span.band-spot-info {
         max-height: 26em;
         overflow: scroll;
     }
+    input#search {
+        max-width: 7em;
+    }
 }
 
 @media (min-width: 992px) {

webassets/js/bands.js

@@ -26,7 +26,7 @@ function loadSpots() {
 // Build a query string for the API, based on the filters that the user has selected.
 function buildQueryString() {
     var str =  "?";
-    ["dx_continent", "de_continent", "mode_type", "source", "band", "sig"].forEach(fn => {
+    ["dx_continent", "de_continent", "mode", "source", "band", "sig"].forEach(fn => {
         if (!allFilterOptionsSelected(fn)) {
             str = str + getQueryStringFor(fn) + "&";
         }
@@ -251,7 +251,7 @@ function loadOptions() {
         generateSIGsMultiToggleFilterCard(options["sigs"]);
         generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
         generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
-        generateMultiToggleFilterCard("#mode-options", "mode_type", options["mode_types"]);
+        generateModesMultiToggleFilterCard(options["modes"]);
         generateSourcesMultiToggleFilterCard(options["spot_sources"], options["web-ui-options"]["spot-providers-enabled-by-default"]);
 
         // Load URL params. These may select things from the various filter & display options, so the function needs

webassets/js/common.js

@@ -28,7 +28,7 @@ function loadURLParams() {
     updateFilterFromParam(params, "band", "band");
     updateFilterFromParam(params, "sig", "sig");
     updateFilterFromParam(params, "source", "source");
-    updateFilterFromParam(params, "mode_type", "mode_type");
+    updateFilterFromParam(params, "mode", "mode");
     updateFilterFromParam(params, "dx_continent", "dx_continent");
     updateFilterFromParam(params, "de_continent", "de_continent");
 }

webassets/js/map.js

@@ -20,7 +20,7 @@ function loadSpots() {
 // Build a query string for the API, based on the filters that the user has selected.
 function buildQueryString() {
     var str =  "?";
-    ["dx_continent", "de_continent", "mode_type", "source", "band", "sig"].forEach(fn => {
+    ["dx_continent", "de_continent", "mode", "source", "band", "sig"].forEach(fn => {
         if (!allFilterOptionsSelected(fn)) {
             str = str + getQueryStringFor(fn) + "&";
         }
@@ -183,7 +183,7 @@ function loadOptions() {
         generateSIGsMultiToggleFilterCard(options["sigs"]);
         generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
         generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
-        generateMultiToggleFilterCard("#mode-options", "mode_type", options["mode_types"]);
+        generateModesMultiToggleFilterCard(options["modes"]);
         generateSourcesMultiToggleFilterCard(options["spot_sources"], options["web-ui-options"]["spot-providers-enabled-by-default"]);
 
         // Load URL params. These may select things from the various filter & display options, so the function needs

webassets/js/spots.js

@@ -87,7 +87,7 @@ function updateTimingDisplayRunPause() {
 // Build a query string for the API, based on the filters that the user has selected.
 function buildQueryString() {
     var str =  "?";
-    ["dx_continent", "de_continent", "mode_type", "source", "band", "sig"].forEach(fn => {
+    ["dx_continent", "de_continent", "mode", "source", "band", "sig"].forEach(fn => {
         if (!allFilterOptionsSelected(fn)) {
             str = str + getQueryStringFor(fn) + "&";
         }
@@ -421,7 +421,7 @@ function loadOptions() {
         generateSIGsMultiToggleFilterCard(options["sigs"]);
         generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
         generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
-        generateMultiToggleFilterCard("#mode-options", "mode_type", options["mode_types"]);
+        generateModesMultiToggleFilterCard(options["modes"]);
         generateSourcesMultiToggleFilterCard(options["spot_sources"], options["web-ui-options"]["spot-providers-enabled-by-default"]);
 
         // Load URL params. These may select things from the various filter & display options, so the function needs

webassets/js/spotsbandsandmap.js

@@ -47,6 +47,49 @@ function generateSIGsMultiToggleFilterCard(sig_options) {
     $("#sig-options").append(` <span style="display: inline-block"><button id="filter-button-sig-all" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('sig', true);">All</button>&nbsp;<button id="filter-button-sig-none" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('sig', false);">None</button></span>`);
 }
 
+// Generate modes filter card. This one is also a special case.
+function generateModesMultiToggleFilterCard(mode_options) {
+    // Create a button for each option
+    mode_options.forEach(o => {
+        var domSafeName = o.replace(/^[^A-Za-z0-9]+|[^\w]+/gi, "");
+        $("#mode-options").append(`<input type="checkbox" class="btn-check filter-button-mode storeable-checkbox" name="options" id="filter-button-mode-${domSafeName}" value="${o}" autocomplete="off" onClick="filtersUpdated()" checked><label class="btn btn-outline-primary" id="filter-button-label-mode-${domSafeName}" for="filter-button-mode-${domSafeName}">${o}</label> `);
+    });
+    // Create All/None buttons
+    $("#mode-options").append(` <span style="display: inline-block"><button id="filter-button-mode-all" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('mode', true);">All</button>&nbsp;<button id="filter-button-mode-none" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('mode', false);">None</button></span>`);
+    // Create category buttons
+    $("#mode-options").append(` <button id="filter-button-mode-av" type="button" class="btn btn-outline-secondary" onclick="toggleAnalogVoiceModeToggles();">Analog Voice</button>&nbsp;<button id="filter-button-mode-dv" type="button" class="btn btn-outline-secondary" onclick="toggleDigitalVoiceModeToggles();">Digital Voice</button>&nbsp;<button id="filter-button-mode-digi" type="button" class="btn btn-outline-secondary" onclick="toggleDigiModeToggles();">Digimodes</button></span>`);
+}
+
+// Toggle the mode toggles that relate to Analog Voice.
+function toggleAnalogVoiceModeToggles() {
+    toggleToggles("mode", ["PHONE", "SSB", "LSB", "USB", "AM", "FM"]);
+}
+
+// Toggle the mode toggles that relate to Digital Voice.
+function toggleDigitalVoiceModeToggles() {
+    toggleToggles("mode", ["DV", "DMR", "DSTAR", "C4FM", "M17"]);
+}
+
+// Toggle the mode toggles that relate to Digimodes.
+function toggleDigiModeToggles() {
+    toggleToggles("mode", ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "PSK", "OLIVIA", "PKT", "MSK144"]);
+}
+
+// Toggle the a set of toggles of the given type (e.g. "mode") that match the given values (e.g. ["SSB", "AM", "FM"]).
+function toggleToggles(type, values) {
+    let toggle = null;
+    $(".filter-button-" + type).each(function() {
+        console.log($(this));
+        if (values.includes($(this).val().replace("filter-button-" + type, ""))) {
+            if (toggle == null) {
+                toggle = !$(this).prop('checked');
+            }
+            $(this).prop('checked', toggle);
+        }
+    });
+    filtersUpdated();
+}
+
 // Generate Sources filter card. This one is a minor special case as we create the buttons in the normal way, but then
 // set which ones are enabled by default based on config rather than having them all enabled by default. We also sanitise
 // names here for HTML elements.

webassets/js/status.js

@@ -22,14 +22,14 @@ function loadStatus() {
         jsonData["spot_providers"].forEach(p => {
             $("#status-container").append(generateStatusCard("Spot Provider: " + p["name"], [
                 `Status: ${p["status"]}`,
-                `Last Updated: ${p["enabled"] ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}`,
+                `Last Updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}`,
-                `Latest Spot: ${p["enabled"] ? moment.unix(p["last_spot"]).utc().fromNow() : "N/A"}`
+                `Latest Spot: ${(p["enabled"] && p["last_spot"] > 0) ? moment.unix(p["last_spot"]).utc().fromNow() : "N/A"}`
             ]));
         });
         jsonData["alert_providers"].forEach(p => {
             $("#status-container").append(generateStatusCard("Alert Provider: " + p["name"], [
                 `Status: ${p["status"]}`,
-                `Last Updated: ${p["enabled"] ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}`
+                `Last Updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}`
             ]));
         });
     });