Ham HF band toggle preset and prevent some multiple-SSE shenanigans when searching and typing letters quickly

README.md

@@ -30,7 +30,7 @@ URL parameters can be used to trigger an "embedded" mode which hides the headers
 
 Setting `embedded` to true is important for the rest of the settings to be applied; otherwise, the user's defaults will be used in preference to the URL params.
 
-These are supplied with the URL to the page you want to embed, for example for an embedded version of the band map in dark mode, use `https://spothole.com/bands?embedded=true&dark-mode=true`. For an embedded version of the main spots/home page in the system light/dark mode, use `https://spothole.com/?embedded=true`. For dark mode showing 70cm TOTA spots only, use `https://spothole.com/?embedded=true&dark-mode=true&filter-sigs=TOTA&filter-bands=70cm`. Providing no URL params causes the page to be loaded in the normal way it would when accessed directly in the user's browser.
+These are supplied with the URL to the page you want to embed, for example for an embedded version of the band map in dark mode, use `https://spothole.app/bands?embedded=true&dark-mode=true`. For an embedded version of the main spots/home page in the system light/dark mode, use `https://spothole.app/?embedded=true`. For dark mode showing 70cm TOTA spots only, use `https://spothole.app/?embedded=true&dark-mode=true&sig=TOTA&band=70cm`. Providing no URL params causes the page to be loaded in the normal way it would when accessed directly in the user's browser.
 
 The supported parameters are as follows. Generally these match the equivalent parameters in the real Spothole API, where a mapping exists.
 
@@ -157,6 +157,8 @@ server {
     
     location / {
         add_header Access-Control-Allow-Origin $xssorigin;
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
         proxy_pass http://127.0.0.1:8080;
     }
 }
@@ -219,6 +221,7 @@ To navigate your way around the source code, this list may help.
 
 * `/` - Main script (`spothole.py`), pip `requirements.txt`, config, README, etc.
 * `/images` - Image sources
+* `/datafiles` - Local data sources (differentiated from the majority of data files which are loaded from URLs and cached in `/cache`)
 * `/cache` - Directory where static-ish data downloaded from the internet is cached to avoid rapid re-requests, and where spot/alert data is cached so that it survives a software restart. Created on first run.
 
 ### Extending the server

config-example.yml

@@ -86,12 +86,11 @@ spot-providers:
     name: "39C3 TOTA"
     enabled: false
     url: "wss://dev.39c3.totawatch.de/api/spot/live"
-    # Fixed SIG/latitude/longitude for all spots from a provider is currently only a feature for the "XOTA" provider,
+    # Fixed SIG for all spots from a provider & location CSV are currently only a feature for the "XOTA" provider,
     # the software found at https://github.com/nischu/xOTA/. This is because this is a generic backend for xOTA
     # programmes and so different URLs provide different programmes.
     sig: "TOTA"
-    latitude: 53.5622678
+    locations-csv: "datafiles/39c3-tota.csv"
-    longitude: 9.9855205
 
 
 # Alert providers to use. Same setup as the spot providers list above.

datafiles/39c3-tota.csv

@@ -0,0 +1,18 @@
+ref,lat,lon
+T-01,53.56278090617755,9.984341869295505
+T-02,53.562383404176416,9.98551893027115
+T-03,53.56170184391514,9.985416035619778
+T-04,53.562026534393176,9.986372919078974
+T-11,53.56284641242506,9.98475590239655
+T-12,53.562431705517035,9.98551675702443
+T-13,53.56223704898424,9.985774520335664
+T-14,53.5617893512591,9.986344302837976
+T-21,53.56284641242506,9.98475590239655
+T-22,53.56245816412497,9.985456089490567
+T-23,53.56199560857136,9.985636761412673
+T-24,53.5617893512591,9.986344302837976
+T-31,53.56247470064887,9.985611427551902
+T-32,53.5617893512591,9.986344302837976
+T-41,53.56245039134992,9.985486136112701
+T-91,53.56147934973529,9.984626806439744
+T-92,53.561396810300735,9.987553052152899

server/handlers/api/alerts.py

@@ -53,6 +53,11 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
         self.sse_alert_queues = sse_alert_queues
         self.web_server_metrics = web_server_metrics
 
+    # Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data
+    def custom_headers(self):
+        return {"Cache-Control": "no-store",
+                "X-Accel-Buffering": "no"}
+
     def open(self):
         try:
             # Metrics

server/handlers/api/spots.py

@@ -54,6 +54,11 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
         self.sse_spot_queues = sse_spot_queues
         self.web_server_metrics = web_server_metrics
 
+    # Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data
+    def custom_headers(self):
+        return {"Cache-Control": "no-store",
+                "X-Accel-Buffering": "no"}
+
     # Called once on the client opening a connection, set things up
     def open(self):
         try:

server/handlers/metrics.py

@@ -1,5 +1,5 @@
 import tornado
-from prometheus_client.openmetrics.exposition import CONTENT_TYPE_LATEST
+from prometheus_client import CONTENT_TYPE_LATEST
 
 from core.prometheus_metrics_handler import get_metrics
 

spotproviders/xota.py

@@ -1,4 +1,6 @@
+import csv
 import json
+import logging
 from datetime import datetime
 
 import pytz
@@ -9,31 +11,45 @@ from spotproviders.websocket_spot_provider import WebsocketSpotProvider
 
 
 # Spot provider for servers based on the "xOTA" software at https://github.com/nischu/xOTA/
-# The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides this information. This
+# The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG and a reference
-# functionality is implemented for TOTA events.
+# to a local CSV file with location information. This functionality is implemented for TOTA events, of which there are
+# several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA and hence which server
+# supplied the data, which is why the CSV location lookup is here and not in sig_utils.
 class XOTA(WebsocketSpotProvider):
-    FIXED_LATITUDE = None
+    LOCATION_DATA = {}
-    FIXED_LONGITUDE = None
     SIG = None
 
     def __init__(self, provider_config):
         super().__init__(provider_config, provider_config["url"])
-        self.FIXED_LATITUDE = provider_config["latitude"] if "latitude" in provider_config else None
+        locations_csv = provider_config["locations-csv"] if "locations-csv" in provider_config else None
-        self.FIXED_LONGITUDE = provider_config["longitude"] if "longitude" in provider_config else None
         self.SIG = provider_config["sig"] if "sig" in provider_config else None
 
+        # Load location data
+        if locations_csv:
+            try:
+                f = open(locations_csv)
+                csv_data = f.read()
+                dr = csv.DictReader(csv_data.splitlines())
+                for row in dr:
+                    self.LOCATION_DATA[row["ref"]] = {"lat": row["lat"], "lon": row["lon"]}
+            except:
+                logging.exception("Could not look up location data for XOTA source.")
+
     def ws_message_to_spot(self, bytes):
         string = bytes.decode("utf-8")
         source_spot = json.loads(string)
+        ref_id = source_spot["reference"]["title"]
+        lat = float(self.LOCATION_DATA[ref_id]["lat"]) if ref_id in self.LOCATION_DATA else None
+        lon = float(self.LOCATION_DATA[ref_id]["lon"]) if ref_id in self.LOCATION_DATA else None
         spot = Spot(source=self.name,
                     source_id=source_spot["id"],
                     dx_call=source_spot["stationCallSign"].upper(),
                     freq=float(source_spot["freq"]) * 1000,
                     mode=source_spot["mode"].upper(),
                     sig=self.SIG,
-                    sig_refs=[SIGRef(id=source_spot["reference"]["title"], sig=self.SIG, url=source_spot["reference"]["website"])],
+                    sig_refs=[SIGRef(id=ref_id, sig=self.SIG, url=source_spot["reference"]["website"], latitude=lat, longitude=lon)],
                     time=datetime.now(pytz.UTC).timestamp(),
-                    dx_latitude=self.FIXED_LATITUDE,
+                    dx_latitude=lat,
-                    dx_longitude=self.FIXED_LONGITUDE,
+                    dx_longitude=lon,
                     qrt=source_spot["state"] != "active")
         return spot

templates/about.html

@@ -63,7 +63,7 @@
     <p>This software is dedicated to the memory of Tom G1PJB, SK, a friend and colleague who sadly passed away around the time I started writing it in Autumn 2025. I was looking forward to showing it to you when it was done.</p>
 </div>
 
-<script src="/js/common.js?v=2"></script>
+<script src="/js/common.js?v=3"></script>
 <script>$(document).ready(function() { $("#nav-link-about").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/add_spot.html

@@ -69,8 +69,8 @@
 
 </div>
 
-<script src="/js/common.js?v=2"></script>
+<script src="/js/common.js?v=3"></script>
-<script src="/js/add-spot.js?v=2"></script>
+<script src="/js/add-spot.js?v=3"></script>
 <script>$(document).ready(function() { $("#nav-link-add-spot").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/alerts.html

@@ -168,8 +168,8 @@
 
 </div>
 
-<script src="/js/common.js?v=2"></script>
+<script src="/js/common.js?v=3"></script>
-<script src="/js/alerts.js?v=2"></script>
+<script src="/js/alerts.js?v=3"></script>
 <script>$(document).ready(function() { $("#nav-link-alerts").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/bands.html

@@ -129,9 +129,9 @@
 
 </div>
 
-<script src="/js/common.js?v=2"></script>
+<script src="/js/common.js?v=3"></script>
-<script src="/js/spotsbandsandmap.js?v=2"></script>
+<script src="/js/spotsbandsandmap.js?v=3"></script>
-<script src="/js/bands.js?v=2"></script>
+<script src="/js/bands.js?v=3"></script>
 <script>$(document).ready(function() { $("#nav-link-bands").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/map.html

@@ -147,9 +147,9 @@
 <script src="https://cdn.jsdelivr.net/npm/leaflet.geodesic"></script>
 <script src="https://cdn.jsdelivr.net/npm/@joergdietrich/leaflet.terminator@1.1.0/L.Terminator.min.js"></script>
 
-<script src="/js/common.js?v=2"></script>
+<script src="/js/common.js?v=3"></script>
-<script src="/js/spotsbandsandmap.js?v=2"></script>
+<script src="/js/spotsbandsandmap.js?v=3"></script>
-<script src="/js/map.js?v=2"></script>
+<script src="/js/map.js?v=3"></script>
 <script>$(document).ready(function() { $("#nav-link-map").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/spots.html

@@ -218,9 +218,9 @@
 
 </div>
 
-<script src="/js/common.js?v=2"></script>
+<script src="/js/common.js?v=3"></script>
-<script src="/js/spotsbandsandmap.js?v=2"></script>
+<script src="/js/spotsbandsandmap.js?v=3"></script>
-<script src="/js/spots.js?v=2"></script>
+<script src="/js/spots.js?v=4"></script>
 <script>$(document).ready(function() { $("#nav-link-spots").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

templates/status.html

@@ -3,8 +3,8 @@
 
 <div id="status-container" class="row row-cols-1 row-cols-md-4 g-4 mt-4"></div>
 
-<script src="/js/common.js?v=2"></script>
+<script src="/js/common.js?v=3"></script>
-<script src="/js/status.js?v=2"></script>
+<script src="/js/status.js?v=3"></script>
 <script>$(document).ready(function() { $("#nav-link-status").addClass("active"); }); <!-- highlight active page in nav --></script>
 
 {% end %}

webassets/js/alerts.js

@@ -352,3 +352,11 @@ $(document).ready(function() {
     // Update the refresh timing display every second
     setInterval(updateRefreshDisplay, 1000);
 });
+
+// Reload alerts on becoming visible. This forces a refresh when used as a PWA and the user switches back to the PWA
+// after some time has passed with it in the background.
+addEventListener("visibilitychange", (event) => {
+    if (!document.hidden) {
+        loadAlerts();
+    }
+});

webassets/js/spots.js

@@ -1,5 +1,6 @@
 // SSE event source
 let evtSource;
+let restartSSEOnErrorTimeoutId;
 // Table row count, to alternate shading
 let rowCount = 0;
 
@@ -30,6 +31,9 @@ function loadSpots() {
 // Start an SSE connection (closing an existing one if it exists). This will then be used to add to the table on the
 // fly.
 function startSSEConnection() {
+    if (evtSource != null) {
+        evtSource.close();
+    }
     evtSource = new EventSource('/api/v1/spots/stream' + buildQueryString());
 
     evtSource.onmessage = function(event) {
@@ -66,8 +70,11 @@ function startSSEConnection() {
     };
 
     evtSource.onerror = function(err) {
-        evtSource.close();
+        if (evtSource != null) {
-        setTimeout(startSSEConnection, 1000);
+            evtSource.close();
+        }
+        clearTimeout(restartSSEOnErrorTimeoutId)
+        restartSSEOnErrorTimeoutId = setTimeout(startSSEConnection, 1000);
     };
 }
 
@@ -147,8 +154,8 @@ function updateTable() {
         table.find('tbody').append('<tr class="table-danger"><td colspan="100" style="text-align:center;">No spots match your filters.</td></tr>');
     }
 
-    spots.reverse();
+    let spotsNewestFirst = spots.toReversed();
-    spots.forEach(s => addSpotToTopOfTable(s, false));
+    spotsNewestFirst.forEach(s => addSpotToTopOfTable(s, false));
 }
 
 // Add rows corresponding to a new spot to the top of the table
@@ -182,9 +189,6 @@ function createNewTableRowsForSpot(s, highlightNew) {
 
     // Create row
     let $tr = $('<tr>');
-    if (highlightNew) {
-        $tr.addClass("new");
-    }
 
     // Apply striping to the table. We can't just use Bootstrap's table-striped class because we have all sorts of
     // extra faff to deal with, like the mobile view having extra rows, and the On Now / Next 24h / Later banners
@@ -195,7 +199,13 @@ function createNewTableRowsForSpot(s, highlightNew) {
 
     // Show faded out if QRT
     if (s["qrt"] == true) {
-      $tr.addClass("table-faded");
+        $tr.addClass("table-faded");
+    }
+
+    // If we are asked to highlight new rows (i.e. this row is being added "live" via the SSE client and not as a bulk
+    // reload of the whole table)
+    if (highlightNew) {
+        $tr.addClass("new");
     }
 
     // Format a UTC or local time for display
@@ -277,9 +287,9 @@ function createNewTableRowsForSpot(s, highlightNew) {
         var items = []
         for (var i = 0; i < s["sig_refs"].length; i++) {
             if (s["sig_refs"][i]["url"] != null) {
-                items[i] = `<a href='${s["sig_refs"][i]["url"]}' title='${s["sig_refs"][i]["name"]}' target='_new' class='sig-ref-link'>${s["sig_refs"][i]["id"]}</a>`
+                items[i] = `<span style="white-space: nowrap;"><a href='${s["sig_refs"][i]["url"]}' title='${s["sig_refs"][i]["name"]}' target='_new' class='sig-ref-link'>${s["sig_refs"][i]["id"]}</a></span>`
             } else {
-                items[i] = `${s["sig_refs"][i]["id"]}`
+                items[i] = `<span style="white-space: nowrap;">${s["sig_refs"][i]["id"]}</span>`
             }
         }
         sig_refs = items.join(", ");
@@ -315,7 +325,7 @@ function createNewTableRowsForSpot(s, highlightNew) {
         $tr.append(`<td class='nowrap'>${time_formatted}</td>`);
     }
     if (showDX) {
-        $tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${dx_flag}</span><a class='dx-link' href='https://qrz.com/db/${s["dx_call"]}' target='_new' title='${s["dx_name"] != null ? s["dx_name"] : ""}'>${dx_call}</a></td>`);
+        $tr.append(`<td class='nowrap'><span class='flag-wrapper' title='${dx_country}'>${dx_flag}</span><a class='dx-link' href='https://qrz.com/db/${s["dx_call"]}' target='_new' title='${s["dx_name"] != null ? s["dx_name"] : ""}'>${dx_call}</a></td>`);
     }
     if (showFreq) {
         $tr.append(`<td class='nowrap'><span class='band-bullet' title='${bandFullName}' style='${(s["freq"] != null) ? "color: " + s["band_color"] : "display: none;"}'>&#9632;</span>${freq_string}</td>`);
@@ -333,7 +343,7 @@ function createNewTableRowsForSpot(s, highlightNew) {
         $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${typeText}</td>`);
     }
     if (showRef) {
-        $tr.append(`<td class='hideonmobile'>${sig_refs}</td>`);
+        $tr.append(`<td class='hideonmobile' style='max-width: 11em;'>${sig_refs}</td>`);
     }
     if (showDE) {
         $tr.append(`<td class='nowrap hideonmobile'><span class='flag-wrapper' title='${de_country}'>${de_flag}</span>${de_call}</td>`);
@@ -341,24 +351,38 @@ function createNewTableRowsForSpot(s, highlightNew) {
 
     // Second row for mobile view only, containing type, ref & comment
     $tr2 = $("<tr class='hidenotonmobile'>");
+
+    // Apply styles as per the first row
     if (rowCount % 2 == 1) {
         $tr2.addClass("table-active");
     }
     if (s["qrt"] == true) {
-      $tr2.addClass("table-faded");
+        $tr2.addClass("table-faded");
     }
+    if (highlightNew) {
+        $tr2.addClass("new");
+    }
+
     $td2 = $("<td colspan='100'>");
+    $td2floatleft = $(`<div style="float: left;">`);
     if (showType) {
-        $td2.append(`<span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${typeText} `);
+        $td2floatleft.append(`<span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${typeText} `);
     }
     if (showRef) {
-        $td2.append(`${sig_refs} `);
+        $td2floatleft.append(`${sig_refs} `);
     }
+    $td2.append($td2floatleft);
+    $td2floatright = $(`<div style="float: right;">`);
     if (showBearing) {
-        $td2.append(` &nbsp; Bearing: ${bearingText} `);
+        $td2floatright.append(`${bearingText} &nbsp;`);
     }
+    if (showDE) {
+        $td2floatright.append(` de ${de_call} &nbsp;`);
+    }
+    $td2.append($td2floatright);
+    $td2.append(`</div><div style="clear: both;"></div>`);
     if (showComment) {
-        $td2.append(`<br/>${commentText}`);
+        $td2.append(`${commentText}`);
     }
     $tr2.append($td2);
 

webassets/js/spotsbandsandmap.js

@@ -23,8 +23,18 @@ function generateBandsMultiToggleFilterCard(band_options) {
         var cssFormattedBandName = o['name'] ? o['name'].replace('.', 'p') : "unknown";
         $("#band-options").append(`<input type="checkbox" class="btn-check filter-button-band storeable-checkbox" name="options" id="filter-button-band-${cssFormattedBandName}" value="${o['name']}" autocomplete="off" onClick="filtersUpdated()" checked><label class="btn btn-outline" id="filter-button-label-band-${cssFormattedBandName}" for="filter-button-band-${cssFormattedBandName}">${o['name']}</label> `);
     });
-    // Create All/None buttons
+    // Create All/None/Ham HF buttons
-    $("#band-options").append(` <span style="display: inline-block"><button id="filter-button-band-all" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('band', true);">All</button>&nbsp;<button id="filter-button-band-none" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('band', false);">None</button></span>`);
+    $("#band-options").append(` <span style="display: inline-block"><button id="filter-button-band-all" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('band', true);">All</button> <button id="filter-button-band-none" type="button" class="btn btn-outline-secondary" onclick="toggleFilterButtons('band', false);">None</button> <button id="filter-button-band-none" type="button" class="btn btn-outline-secondary" onclick="setHamHFBandToggles();">Ham HF</button></span>`);
+}
+
+// Set the band toggles so that only the amateur radio HF bands are selected. This includes 160m and 6m because that's
+// widely expected by hams to be included. Special case of toggleFilterButtons().
+function setHamHFBandToggles() {
+    const hamHFBands = ["160m", "80m", "60m", "40m", "30m", "20m", "17m", "15m", "12m", "10m", "6m"];
+    $(".filter-button-band").each(function() {
+        $(this).prop('checked', hamHFBands.includes($(this).attr('id').replace("filter-button-band-", "")));
+    });
+    filtersUpdated();
 }
 
 // Generate SIGs filter card. This one is also a special case.
@@ -50,3 +60,11 @@ function toggleDarkMode() {
     enableDarkMode($("#darkMode")[0].checked);
     saveSettings();
 }
+
+// Reload spots on becoming visible. This forces a refresh when used as a PWA and the user switches back to the PWA
+// after some time has passed with it in the background.
+addEventListener("visibilitychange", (event) => {
+    if (!document.hidden) {
+        loadSpots();
+    }
+});