Support Packet (PKT) as a digi mode

README.md

@@ -14,6 +14,8 @@ Supported data sources include DX Clusters, the Reverse Beacon Network (RBN), th
 
 ![Screenshot](/images/screenshot2.png)
 
+![Screenshot](/images/screenshot3.png)
+
 ### Accessing the public version
 
 You can access the public version's web interface at [https://spothole.app](https://spothole.app), and see [https://spothole.app/apidocs](https://spothole.app/apidocs) for the API details.

alertproviders/parksnpeaks.py

@@ -49,7 +49,8 @@ class ParksNPeaks(HTTPAlertProvider):
                 logging.warn("PNP alert found with sig " + alert.sig + ", developer needs to add support for this!")
 
             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
-            # the alert list.
+            # the alert list. Note that while ZLOTA has its own spots API, it doesn't have its own alerts API. So that
+            # means the PnP *spot* provider rejects ZLOTA spots here, but the PnP *alerts* provider here allows ZLOTA.
             if alert.sig not in ["POTA", "SOTA", "WWFF"]:
                 new_alerts.append(alert)
         return new_alerts

alertproviders/wota.py

@@ -0,0 +1,64 @@
+from datetime import datetime
+
+import pytz
+from rss_parser import RSSParser
+
+from alertproviders.http_alert_provider import HTTPAlertProvider
+from core.sig_utils import get_icon_for_sig
+from data.alert import Alert
+
+
+# Alert provider for Wainwrights on the Air
+class WOTA(HTTPAlertProvider):
+    POLL_INTERVAL_SEC = 3600
+    ALERTS_URL = "https://www.wota.org.uk/alerts_rss.php"
+    RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
+
+    def __init__(self, provider_config):
+        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def http_response_to_alerts(self, http_response):
+        new_alerts = []
+        rss = RSSParser.parse(http_response.content.decode())
+        # Iterate through source data
+        for source_alert in rss.channel.items:
+
+            # Reject GUID missing or zero
+            if not source_alert.guid or not source_alert.guid.content or source_alert.guid.content == "http://www.wota.org.uk/alerts/0":
+                continue
+
+            # Pick apart the title
+            title_split = source_alert.title.split(" on ")
+            dx_call = title_split[0]
+            ref = None
+            ref_name = None
+            if len(title_split) > 1:
+                ref_split = title_split[1].split(" - ")
+                ref = ref_split[0]
+                if len(ref_split) > 1:
+                    ref_name = ref_split[1]
+
+            # Pick apart the description
+            desc_split = source_alert.description.split(". ")
+            freqs_modes = desc_split[0].replace("Frequencies/modes:", "").strip()
+            comment = None
+            if len(desc_split) > 1:
+                comment = desc_split[1].strip()
+
+            time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert.guid.content,
+                          dx_calls=[dx_call],
+                          freqs_modes=freqs_modes,
+                          comment=comment,
+                          sig="WOTA",
+                          sig_refs=[ref] if ref else [],
+                          sig_refs_names=[ref_name] if ref_name else [],
+                          icon=get_icon_for_sig("WOTA"),
+                          start_time=time.timestamp())
+
+            # Add to our list.
+            new_alerts.append(alert)
+        return new_alerts

config-example.yml

@@ -41,6 +41,14 @@ spot-providers:
     class: "ParksNPeaks"
     name: "ParksNPeaks"
     enabled: true
+  -
+    class: "ZLOTA"
+    name: "ZLOTA"
+    enabled: true
+  -
+    class: "WOTA"
+    name: "WOTA"
+    enabled: true
   -
     class: "APRSIS"
     name: "APRS-IS"
@@ -88,6 +96,10 @@ alert-providers:
     class: "ParksNPeaks"
     name: "ParksNPeaks"
     enabled: true
+  -
+    class: "WOTA"
+    name: "WOTA"
+    enabled: true
   -
     class: "NG3K"
     name: "NG3K"

core/constants.py

@@ -26,13 +26,14 @@ SIGS = [
    SIG(name="ZLOTA", description="New Zealand on the Air", icon="kiwi-bird", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d+"),
    SIG(name="KRMNPA", description="Keith Roget Memorial National Parks Award", icon="earth-oceania", ref_regex=r""),
    SIG(name="WAB", description="Worked All Britain", icon="table-cells-large", ref_regex=r"[A-Z]{1,2}[0-9]{2}"),
-   SIG(name="WAI", description="Worked All Ireland", icon="table-cells-large", ref_regex=r"[A-Z][0-9]{2}")
+   SIG(name="WAI", description="Worked All Ireland", icon="table-cells-large", ref_regex=r"[A-Z][0-9]{2}"),
+   SIG(name="WOTA", description="Wainwrights on the Air", icon="w", ref_regex=r"[A-Z]{3}-[0-9]{2}")
 ]
 
 # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
 CW_MODES = ["CW"]
 PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "M17"]
-DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "BPSK", "PSK", "PSK31", "BPSK31", "OLIVIA"]
+DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "BPSK", "PSK", "PSK31", "BPSK31", "OLIVIA", "MFSK", "MFSK32", "PKT"]
 ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
 MODE_TYPES = ["CW", "PHONE", "DATA"]
 

core/lookup_helper.py

@@ -3,7 +3,7 @@ import logging
 from datetime import timedelta
 
 from diskcache import Cache
-from pyhamtools import LookupLib, Callinfo
+from pyhamtools import LookupLib, Callinfo, callinfo
 from pyhamtools.exceptions import APIKeyMissingError
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
@@ -266,36 +266,46 @@ class LookupHelper:
     # Utility method to get QRZ.com data from cache if possible, if not get it from the API and cache it
     def get_qrz_data_for_callsign(self, call):
         # Fetch from cache if we can, otherwise fetch from the API and cache it
-        qrz_data = self.QRZ_CALLSIGN_DATA_CACHE.get(call)
-        if qrz_data:
-            return qrz_data
+        if call in self.QRZ_CALLSIGN_DATA_CACHE:
+            return self.QRZ_CALLSIGN_DATA_CACHE.get(call)
         elif self.QRZ_AVAILABLE:
             try:
                 data = self.LOOKUP_LIB_QRZ.lookup_callsign(callsign=call)
                 self.QRZ_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
                 return data
             except KeyError:
-                # QRZ had no info for the call, that's OK. Cache a None so we don't try to look this up again
-                self.QRZ_CALLSIGN_DATA_CACHE.add(call, None, expire=604800)  # 1 week in seconds
-                return None
+                # QRZ had no info for the call, but maybe it had prefixes or suffixes. Try again with the base call.
+                try:
+                    data = self.LOOKUP_LIB_QRZ.lookup_callsign(callsign=callinfo.Callinfo.get_homecall(call))
+                    self.QRZ_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
+                    return data
+                except KeyError:
+                    # QRZ had no info for the call, that's OK. Cache a None so we don't try to look this up again
+                    self.QRZ_CALLSIGN_DATA_CACHE.add(call, None, expire=604800)  # 1 week in seconds
+                    return None
         else:
             return None
 
     # Utility method to get Clublog API data from cache if possible, if not get it from the API and cache it
     def get_clublog_api_data_for_callsign(self, call):
         # Fetch from cache if we can, otherwise fetch from the API and cache it
-        clublog_data = self.CLUBLOG_CALLSIGN_DATA_CACHE.get(call)
-        if clublog_data:
-            return clublog_data
+        if call in self.CLUBLOG_CALLSIGN_DATA_CACHE:
+            return self.CLUBLOG_CALLSIGN_DATA_CACHE.get(call)
         elif self.CLUBLOG_API_AVAILABLE:
             try:
                 data = self.LOOKUP_LIB_CLUBLOG_API.lookup_callsign(callsign=call)
                 self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
                 return data
             except KeyError:
-                # Clublog had no info for the call, that's OK. Cache a None so we don't try to look this up again
-                self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, None, expire=604800)  # 1 week in seconds
-                return None
+                # Clublog had no info for the call, but maybe it had prefixes or suffixes. Try again with the base call.
+                try:
+                    data = self.LOOKUP_LIB_CLUBLOG_API.lookup_callsign(callsign=callinfo.Callinfo.get_homecall(call))
+                    self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
+                    return data
+                except KeyError:
+                    # Clublog had no info for the call, that's OK. Cache a None so we don't try to look this up again
+                    self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, None, expire=604800)  # 1 week in seconds
+                    return None
             except APIKeyMissingError:
                 # User API key was wrong, warn
                 logging.error("Could not look up via Clublog API, key " + self.CLUBLOG_API_KEY + " was rejected.")

data/alert.py

@@ -111,11 +111,6 @@ class Alert:
         if self.dx_calls and not self.dx_names:
             self.dx_names = list(map(lambda c: lookup_helper.infer_name_from_callsign(c), self.dx_calls))
 
-        # Clean up comments
-        if self.comment:
-            comment = re.sub(r"\(de [A-Za-z0-9]*\)", "", self.comment)
-            self.comment = comment.strip()
-
         # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
         # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
         # apart from that they were retrieved from the API at different times. Note that the simple Python hash()

data/spot.py

@@ -177,6 +177,20 @@ class Spot:
         if self.de_call and "-" in self.de_call:
             self.de_call = self.de_call.split("-")[0]
 
+        # If we have a spotter of "RBNHOLE", we should have the actual spotter callsign in the comment, so extract it.
+        # RBNHole posts come from a number of providers, so it's dealt with here in the generic spot handling code.
+        if self.de_call == "RBNHOLE" and self.comment:
+            rbnhole_call_match = re.search(r"\Wat ([a-z0-9/]+)\W", self.comment, re.IGNORECASE)
+            if rbnhole_call_match:
+                self.de_call = rbnhole_call_match.group(1).upper()
+
+        # If we have a spotter of "SOTAMAT", we might have the actual spotter callsign in the comment, if so extract it.
+        # SOTAMAT can do POTA as well as SOTA, so it's dealt with here in the generic spot handling code.
+        if self.de_call == "SOTAMAT" and self.comment:
+            sotamat_call_match = re.search(r"\Wfrom ([a-z0-9/]+)]", self.comment, re.IGNORECASE)
+            if sotamat_call_match:
+                self.de_call = sotamat_call_match.group(1).upper()
+
         # Spotter country, continent, zones etc. from callsign.
         # DE of "RBNHOLE" and "SOTAMAT" are not things we can look up location for
         if self.de_call != "RBNHOLE" and self.de_call != "SOTAMAT":
@@ -249,14 +263,6 @@ class Spot:
         if self.comment and not self.qrt:
             self.qrt = "QRT" in self.comment.upper()
 
-        # Clean up comments
-        if self.comment:
-            comment = re.sub(r"\(de [A-Za-z0-9]*\)", "", self.comment)
-            comment = re.sub(r"\[.*]:", "", comment)
-            comment = re.sub(r"\[.*]", "", comment)
-            comment = re.sub(r"\"\"", "", comment)
-            self.comment = comment.strip()
-
         # DX operator details lookup, using QRZ.com. This should be the last resort compared to taking the data from
         # the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of
         # the one from the park reference they're at.
@@ -284,8 +290,8 @@ class Spot:
         self.dx_location_good = self.dx_location_source == "SPOT" or self.dx_location_source == "WAB/WAI GRID" or (
                 self.dx_location_source == "QRZ" and not "/" in self.dx_call)
 
-        # DE of "RBNHOLE", "SOTAMAT" and "ZLOTA" are not things we can look up location for
-        if self.de_call != "RBNHOLE" and self.de_call != "SOTAMAT" and self.de_call != "ZLOTA":
+        # DE of "RBNHOLE" and "SOTAMAT" are not things we can look up location for
+        if self.de_call != "RBNHOLE" and self.de_call != "SOTAMAT":
             # DE operator position lookup, using QRZ.com.
             if self.de_call and not self.de_latitude:
                 latlon = lookup_helper.infer_latlon_from_callsign_qrz(self.de_call)

server/webserver.py

@@ -39,6 +39,7 @@ class WebServer:
         # Routes for templated pages
         bottle.get("/")(lambda: self.serve_template('webpage_spots'))
         bottle.get("/map")(lambda: self.serve_template('webpage_map'))
+        bottle.get("/bands")(lambda: self.serve_template('webpage_bands'))
         bottle.get("/alerts")(lambda: self.serve_template('webpage_alerts'))
         bottle.get("/status")(lambda: self.serve_template('webpage_status'))
         bottle.get("/about")(lambda: self.serve_template('webpage_about'))
@@ -206,6 +207,11 @@ class WebServer:
                     needs_sig = query.get(k).upper() == "TRUE"
                     if needs_sig:
                         spots = [s for s in spots if s.sig]
+                case "needs_sig_ref":
+                    # If true, at least one sig ref is required, regardless of what it is, it just can't be missing.
+                    needs_sig_ref = query.get(k).upper() == "TRUE"
+                    if needs_sig_ref:
+                        spots = [s for s in spots if s.sig_refs and len(s.sig_refs) > 0]
                 case "band":
                     bands = query.get(k).split(",")
                     spots = [s for s in spots if s.band and s.band in bands]
@@ -224,6 +230,16 @@ class WebServer:
                 case "comment_includes":
                     comment_includes = query.get(k).strip()
                     spots = [s for s in spots if s.comment and comment_includes.upper() in s.comment.upper()]
+                case "allow_qrt":
+                    # If false, spots that are flagged as QRT are not returned.
+                    prevent_qrt = query.get(k).upper() == "FALSE"
+                    if prevent_qrt:
+                        spots = [s for s in spots if not s.qrt or s.qrt == False]
+                case "needs_good_location":
+                    # If true, spots require a "good" location to be returned
+                    needs_good_location = query.get(k).upper() == "TRUE"
+                    if needs_good_location:
+                        spots = [s for s in spots if s.dx_location_good]
                 case "dedupe":
                     # Ensure only the latest spot of each callsign is present in the list. This relies on the list being
                     # in reverse time order, so if any future change allows re-ordering the list, that should be done

spotproviders/dxcluster.py

@@ -19,9 +19,9 @@ class DXCluster(SpotProvider):
     # Note the callsign pattern deliberately excludes calls ending in "-#", which are from RBN and can be enabled by
     # default on some clusters. If you want RBN spots, there is a separate provider for that.
     CALLSIGN_PATTERN = "([a-z|0-9|/]+)"
-    FREQUENCY_PATTERM = "([0-9|.]+)"
+    FREQUENCY_PATTERN = "([0-9|.]+)"
     LINE_PATTERN = re.compile(
-        "^DX de " + CALLSIGN_PATTERN + ":\\s+" + FREQUENCY_PATTERM + "\\s+" + CALLSIGN_PATTERN + "\\s+(.*)\\s+(\\d{4}Z)",
+        "^DX de " + CALLSIGN_PATTERN + ":\\s+" + FREQUENCY_PATTERN + "\\s+" + CALLSIGN_PATTERN + "\\s+(.*)\\s+(\\d{4}Z)",
         re.IGNORECASE)
 
     # Constructor requires hostname and port

spotproviders/parksnpeaks.py

@@ -19,9 +19,6 @@ class ParksNPeaks(HTTPSpotProvider):
     SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
     SIOTA_LIST_CACHE_TIME_DAYS = 30
     SIOTA_LIST_CACHE = CachedSession("cache/siota_data_cache", expire_after=timedelta(days=SIOTA_LIST_CACHE_TIME_DAYS))
-    ZLOTA_LIST_URL = "https://ontheair.nz/assets/assets.json"
-    ZLOTA_LIST_CACHE_TIME_DAYS = 30
-    ZLOTA_LIST_CACHE = CachedSession("cache/zlota_data_cache", expire_after=timedelta(days=ZLOTA_LIST_CACHE_TIME_DAYS))
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -52,7 +49,7 @@ class ParksNPeaks(HTTPSpotProvider):
 
             # Extract a de_call if it's in the comment but not in the "actSpoter" field
             m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment)
-            if (not spot.de_call or spot.de_call == "ZLOTA") and m:
+            if not spot.de_call and m:
                 spot.de_call = m.group(1)
 
             # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
@@ -70,20 +67,10 @@ class ParksNPeaks(HTTPSpotProvider):
                         spot.dx_grid = row["LOCATOR"]
                         break
 
-            # ZLOTA name/lat/lon lookup
-            if spot.sig == "ZLOTA":
-                zlota_data = self.ZLOTA_LIST_CACHE.get(self.ZLOTA_LIST_URL, headers=HTTP_HEADERS).json()
-                for asset in zlota_data:
-                    if asset["code"] == spot.sig_refs[0]:
-                        spot.sig_refs_names = [asset["name"]]
-                        spot.dx_latitude = asset["y"]
-                        spot.dx_longitude = asset["x"]
-                        break
-
             # Note there is currently no support for KRMNPA location lookup, see issue #61.
 
-            # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
-            # the spot list.
-            if spot.sig not in ["POTA", "SOTA", "WWFF"]:
+            # If this is POTA, SOTA, WWFF or ZLOTA data we already have it through other means, so ignore. Otherwise,
+            # add to the spot list.
+            if spot.sig not in ["POTA", "SOTA", "WWFF", "ZLOTA"]:
                 new_spots.append(spot)
         return new_spots

spotproviders/sse_spot_provider.py

@@ -34,11 +34,19 @@ class SSESpotProvider(SpotProvider):
         if self.thread:
             self.thread.join()
 
+    def _on_open(self):
+        self.status = "Waiting for Data"
+
+    def _on_error(self):
+        self.status = "Connecting"
+
     def run(self):
         while not self.stopped:
             try:
                 logging.debug("Connecting to " + self.name + " spot API...")
-                with EventSource(self.url, headers=HTTP_HEADERS, latest_event_id=self.last_event_id, timeout=30) as event_source:
+                self.status = "Connecting"
+                with EventSource(self.url, headers=HTTP_HEADERS, latest_event_id=self.last_event_id, timeout=30,
+                                 on_open=self._on_open, on_error=self._on_error) as event_source:
                     self.event_source = event_source
                     for event in self.event_source:
                         if event.type == 'message':
@@ -58,6 +66,8 @@ class SSESpotProvider(SpotProvider):
             except Exception as e:
                 self.status = "Error"
                 logging.exception("Exception in SSE Spot Provider (" + self.name + ")")
+            else:
+                self.status = "Disconnected"
             sleep(5)  # Wait before trying to reconnect
 
     # Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass

spotproviders/wota.py

@@ -0,0 +1,88 @@
+from datetime import timedelta, datetime
+
+import pytz
+from requests_cache import CachedSession
+from rss_parser import RSSParser
+
+from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
+from data.spot import Spot
+from spotproviders.http_spot_provider import HTTPSpotProvider
+
+
+# Spot provider for Wainwrights on the Air
+class WOTA(HTTPSpotProvider):
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://www.wota.org.uk/spots_rss.php"
+    LIST_URL = "https://www.wota.org.uk/mapping/data/summits.json"
+    LIST_CACHE_TIME_DAYS = 30
+    LIST_CACHE = CachedSession("cache/wota_data_cache", expire_after=timedelta(days=LIST_CACHE_TIME_DAYS))
+    RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
+
+    def __init__(self, provider_config):
+        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def http_response_to_spots(self, http_response):
+        new_spots = []
+        rss = RSSParser.parse(http_response.content.decode())
+        # Iterate through source data
+        for source_spot in rss.channel.items:
+
+            # Reject GUID missing or zero
+            if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0":
+                continue
+
+            # Pick apart the title
+            title_split = source_spot.title.split(" on ")
+            dx_call = title_split[0]
+            ref = None
+            ref_name = None
+            if len(title_split) > 1:
+                ref_split = title_split[1].split(" - ")
+                ref = ref_split[0]
+                if len(ref_split) > 1:
+                    ref_name = ref_split[1]
+
+            # Pick apart the description
+            desc_split = source_spot.description.split(". ")
+            freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
+            freq_mode_split = freq_mode.split("-")
+            freq_hz = float(freq_mode_split[0]) * 1000000
+            mode = freq_mode_split[1]
+
+            comment = None
+            if len(desc_split) > 1:
+                comment = desc_split[1].strip()
+            spotter = None
+            if len(desc_split) > 2:
+                spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").strip()
+
+            time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+
+            # Convert to our spot format
+            spot = Spot(source=self.name,
+                        source_id=source_spot.guid.content,
+                        dx_call=dx_call,
+                        de_call=spotter,
+                        freq=freq_hz,
+                        mode=mode,
+                        comment=comment,
+                        sig="WOTA",
+                        sig_refs=[ref] if ref else [],
+                        sig_refs_names=[ref_name] if ref_name else [],
+                        sig_refs_urls="https://www.wota.org.uk/MM_" + ref if ref else [],
+                        icon=get_icon_for_sig("WOTA"),
+                        time=time.timestamp())
+
+            # WOTA name/lat/lon lookup
+            wota_data = self.LIST_CACHE.get(self.LIST_URL, headers=HTTP_HEADERS).json()
+            for feature in wota_data["features"]:
+                if feature["properties"]["wotaId"] == spot.sig_refs[0]:
+                    spot.sig_refs_names = [feature["properties"]["title"]]
+                    spot.dx_latitude = feature["geometry"]["coordinates"][1]
+                    spot.dx_longitude = feature["geometry"]["coordinates"][0]
+                    spot.dx_grid = feature["properties"]["qthLocator"]
+                    break
+
+            new_spots.append(spot)
+        return new_spots

spotproviders/zlota.py

@@ -0,0 +1,59 @@
+import csv
+import logging
+import re
+from datetime import datetime, timedelta
+
+import pytz
+from requests_cache import CachedSession
+
+from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
+from data.spot import Spot
+from spotproviders.http_spot_provider import HTTPSpotProvider
+
+
+# Spot provider for ZLOTA
+class ZLOTA(HTTPSpotProvider):
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://ontheair.nz/api/spots?zlota_only=true"
+    LIST_URL = "https://ontheair.nz/assets/assets.json"
+    LIST_CACHE_TIME_DAYS = 30
+    LIST_CACHE = CachedSession("cache/zlota_data_cache", expire_after=timedelta(days=LIST_CACHE_TIME_DAYS))
+
+    def __init__(self, provider_config):
+        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def http_response_to_spots(self, http_response):
+        new_spots = []
+        # Iterate through source data
+        for source_spot in http_response.json():
+            # Frequency is often inconsistent as to whether it's in Hz or kHz. Make a guess.
+            freq_hz = float(source_spot["frequency"])
+            if freq_hz < 1000000:
+                freq_hz = freq_hz * 1000
+
+            # Convert to our spot format
+            spot = Spot(source=self.name,
+                        source_id=source_spot["id"],
+                        dx_call=source_spot["activator"].upper(),
+                        de_call=source_spot["spotter"].upper(),
+                        freq=freq_hz,
+                        mode=source_spot["mode"].upper().strip(),
+                        comment=source_spot["comments"],
+                        sig="ZLOTA",
+                        sig_refs=[source_spot["reference"]],
+                        sig_refs_names=[source_spot["name"]],
+                        icon=get_icon_for_sig("ZLOTA"),
+                        time=datetime.fromisoformat(source_spot["referenced_time"]).astimezone(pytz.UTC).timestamp())
+
+            # ZLOTA name/lat/lon lookup
+            zlota_data = self.LIST_CACHE.get(self.LIST_URL, headers=HTTP_HEADERS).json()
+            for asset in zlota_data:
+                if asset["code"] == spot.sig_refs[0]:
+                    spot.sig_refs_names = [asset["name"]]
+                    spot.dx_latitude = asset["y"]
+                    spot.dx_longitude = asset["x"]
+                    break
+
+            new_spots.append(spot)
+        return new_spots

views/webpage_about.tpl

@@ -24,6 +24,8 @@
     <h4 class="mt-4">Why does this website ask me if I want to install it?</h4>
     <p>Spothole is a Progressive Web App, which means you can install it on an Android or iOS device by opening the site in Chrome or Safari respectively, and clicking "Install" on the pop-up panel. It'll only prompt you once, so if you dismiss the prompt and change your mind, you'll find an Install / Add to Home Screen option on your browser's menu.</p>
     <p>Installing Spothole on your phone is completely optional, the website works exactly the same way as the "app" does.</p>
+    <h4 class="mt-4">Why hasn't my spot/alert shown up yet?</h4>
+    <p>To avoid putting too much load on the various servers that Spothole connects to, the Spothole server only polls them once every two minutes for spots, and once every hour for alerts. (Some sources, such as DX clusters, RBN, APRS-IS and WWBOTA use a non-polling mechanism, and their updates will therefore arrive more quickly.) Then if you are using the web interface, that has its own rate at which it reloads the data from Spothole, which is once a minute for spots or 30 minutes for alerts. So you could be waiting around three minutes to see a newly added spot, or 90 minutes to see a newly added alert.</p>
     <h4 class="mt-4">What licence does Spothole use?</h4>
     <p>Spothole's source code is licenced under the Public Domain. You can write a Spothole client, run your own server, modify it however you like, you can claim you wrote it and charge people £1000 for a copy, I don't really mind. (Please don't do the last one. But if you're using my code for something cool, it would be nice to hear from you!)</p>
     <h2 id="privacy" class="mt-4">Privacy</h2>

views/webpage_bands.tpl

@@ -0,0 +1,117 @@
+% rebase('webpage_base.tpl')
+
+<div class="mt-3">
+    <div class="row">
+        <div class="col-auto me-auto pt-3">
+            <p id="timing-container">Loading...</p>
+        </div>
+        <div class="col-auto">
+            <p class="d-inline-flex gap-1">
+                <button id="filters-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleFiltersPanel();"><i class="fa-solid fa-filter"></i> Filters</button>
+                <button id="display-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleDisplayPanel();"><i class="fa-solid fa-desktop"></i> Display</button>
+            </p>
+        </div>
+    </div>
+
+    <div id="filters-area" class="appearing-panel card mb-3">
+        <div class="card-header text-white bg-primary">
+            <div class="row">
+                <div class="col-auto me-auto">
+                    Filters
+                </div>
+                <div class="col-auto d-inline-flex">
+                    <button id="close-filters-button" type="button" class="btn-close btn-close-white" aria-label="Close" onclick="closeFiltersPanel();"></button>
+                </div>
+            </div>
+
+        </div>
+        <div class="card-body">
+            <div class="row row-cols-1 g-4 mb-4">
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Bands</h5>
+                            <p id="band-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+            <div class="row row-cols-1 row-cols-md-4 g-4">
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">DX Continent</h5>
+                            <p id="dx-continent-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">DE Continent</h5>
+                            <p id="de-continent-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Modes</h5>
+                            <p id="mode-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Sources</h5>
+                            <p id="source-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div id="display-area" class="appearing-panel card mb-3">
+        <div class="card-header text-white bg-primary">
+            <div class="row">
+                <div class="col-auto me-auto">
+                    Display
+                </div>
+                <div class="col-auto d-inline-flex">
+                    <button id="close-display-button" type="button" class="btn-close btn-close-white" aria-label="Close" onclick="closeDisplayPanel();"></button>
+                </div>
+            </div>
+
+        </div>
+        <div class="card-body">
+            <div id="display-container" class="row row-cols-1 row-cols-md-4 g-4">
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Spot Age</h5>
+                            <p class="card-text spothole-card-text">Last
+                                <select id="max-spot-age" class="storeable-select form-select ms-2 me-2 d-inline-block" oninput="filtersUpdated();" style="width: 5em; display: inline-block;">
+                                    <option value="300">5</option>
+                                    <option value="600">10</option>
+                                    <option value="1800" selected>30</option>
+                                    <option value="3600">60</option>
+                                </select>
+                                minutes
+                            </p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div id="bands-container"></div>
+
+</div>
+
+<script src="/js/common.js"></script>
+<script src="/js/spotandmap.js"></script>
+<script src="/js/bands.js"></script>
+<script>$(document).ready(function() { $("#nav-link-bands").addClass("active"); }); <!-- highlight active page in nav --></script>

views/webpage_base.tpl

@@ -31,6 +31,7 @@
     <link href="/fa/css/solid.min.css" rel="stylesheet" />
 
     <link rel="icon" type="image/png" href="/img/icon-512.png">
+    <link rel="apple-touch-icon" href="img/icon-512-pwa.png">
     <link rel="alternate icon" type="image/png" href="/img/icon-192.png">
     <link rel="alternate icon" type="image/png" href="/img/icon-32.png">
     <link rel="alternate icon" type="image/png" href="/img/icon-16.png">
@@ -59,6 +60,7 @@
                 <ul class="navbar-nav me-auto mb-2 mb-lg-0">
                     <li class="nav-item ms-4"><a href="/" class="nav-link" id="nav-link-spots"><i class="fa-solid fa-tower-cell"></i> Spots</a></li>
                     <li class="nav-item ms-4"><a href="/map" class="nav-link" id="nav-link-map"><i class="fa-solid fa-map"></i> Map</a></li>
+                    <li class="nav-item ms-4"><a href="/bands" class="nav-link" id="nav-link-bands"><i class="fa-solid fa-ruler-vertical"></i> Bands</a></li>
                     <li class="nav-item ms-4"><a href="/alerts" class="nav-link" id="nav-link-alerts"><i class="fa-solid fa-bell"></i> Alerts</a></li>
                     <li class="nav-item ms-4"><a href="/status" class="nav-link" id="nav-link-status"><i class="fa-solid fa-chart-simple"></i> Status</a></li>
                     <li class="nav-item ms-4"><a href="/about" class="nav-link" id="nav-link-about"><i class="fa-solid fa-circle-info"></i> About</a></li>

webassets/apidocs/openapi.yml

@@ -85,7 +85,14 @@ paths:
               - IOTA
         - name: needs_sig
           in: query
-          description: "Limit the spots to only ones from a Special Interest Grous such as POTA. Because supplying all known SIGs as a `sigs` parameter is unwieldy, and leaving `sigs` blank will also return spots with *no* SIG, this parameter can be set true to return only spots with a SIG, regardless of what it is, so long as it's not blank. This is what Field Spotter uses to exclude generic cluster spots and only retrieve xOTA things."
+          description: "Limit the spots to only ones with a Special Interest Group such as POTA. Because supplying all known SIGs as a `sigs` parameter is unwieldy, and leaving `sigs` blank will also return spots with *no* SIG, this parameter can be set true to return only spots with a SIG, regardless of what it is, so long as it's not blank. This is what Field Spotter uses to exclude generic cluster spots and only retrieve xOTA things."
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: needs_sig_ref
+          in: query
+          description: "Limit the spots to only ones which have at least one reference (e.g. a park reference) for Special Interest Groups such as POTA."
           required: false
           schema:
             type: boolean
@@ -195,6 +202,20 @@ paths:
           required: false
           schema:
             type: string
+        - name: needs_good_location
+          in: query
+          description: "Return only spots with a 'good' location. (See the spot `dx_location_good` parameter for details. Useful for map-based clients, to avoid spots with 'bad' locations e.g. loads of cluster spots ending up in the centre of the DXCC entitity.)"
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: allow_qrt
+          in: query
+          description: Allow spots that are known to be QRT to be returned.
+          required: false
+          schema:
+            type: boolean
+            default: true
       responses:
         '200':
           description: Success
@@ -995,4 +1016,4 @@ components:
         ref_regex:
           type: string
           description: Regex that matches this SIG's reference IDs. Generally for Spothole's own internal use, clients probably won't need this.
-          example: "[A-Z]{2}\-\d+"
+          example: "[A-Z]{2}\\-\\d+"

webassets/css/style.css

@@ -150,13 +150,108 @@ div#map {
 }
 
 
+/* BANDS PANEL */
+
+div#bands-container {
+    min-height: 64em;
+    margin: 0;
+    padding: 0;
+    overflow-x: auto;
+    white-space: nowrap;
+    overscroll-behavior-x: none;
+}
+
+#bands-table {
+    min-width: 100%;
+}
+
+#bands-table th {
+    width: 20%;
+    max-height: 40px;
+    min-width: 12em;
+    padding: 0.5em;
+    text-align: center;
+    font-weight: bold;
+}
+
+#bands-table td {
+    width: 20%;
+    min-width: 12em;
+    height: 62em;
+}
+
+div.band-container {
+    height: 62em;
+    width: 20%;
+    min-width: 12em;
+    position: relative;
+}
+
+div.band-markers {
+    width: 100%;
+    height: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    z-index: 13;
+    border-left: 2px dotted black;
+}
+
+div.band-spots {
+    width: 100%;
+    height: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    z-index: 15;
+}
+
+canvas.band-lines-canvas {
+    width: 5em;
+    height: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    z-index: 11;
+}
+
+div.band-spot {
+    position: absolute;
+    left: 5em;
+    padding: 0 0.25em;
+    background-color: white;
+    border-radius: 3px;
+    cursor: default;
+}
+
+div.band-spot:hover {
+    z-index: 999;
+}
+
+div.band-spot span.band-spot-call {
+    display: inline;
+}
+
+div.band-spot:hover span.band-spot-call {
+    display: none;
+}
+
+div.band-spot span.band-spot-info {
+    display: none;
+}
+
+div.band-spot:hover span.band-spot-info {
+    display: inline;
+}
+
+
 /* GENERAL MOBILE SUPPORT */
 
 @media (max-width: 991.99px) {
     .hideonmobile {
         display: none !important;
     }
-    div#map, div#table-container {
+    div#map, div#table-container, div#bands-container {
         margin-left: -1em;
         margin-right: -1em;
     }

webassets/js/alerts.js

@@ -124,16 +124,12 @@ function addAlertRowsToTable(tbody, alerts) {
         var showRef = $("#tableShowRef")[0].checked;
 
         // Get times for the alert, and convert to local time if necessary.
-        var start_time_unix = moment.unix(a["start_time"]);
-        var start_time = start_time_unix.utc();
-        if (useLocalTime) {
-            start_time = start_time.local();
-        }
-        var end_time_unix = moment.unix(a["end_time"]);
-        var end_time = end_time_unix.utc();
-        if (useLocalTime) {
-            end_time = end_time.local();
-        }
+        var start_time_utc = moment.unix(a["start_time"]).utc();
+        var start_time_local = start_time_utc.clone().local();
+        start_time = useLocalTime ? start_time_local : start_time_utc;
+        var end_time_utc = moment.unix(a["end_time"]).utc();
+        var end_time_local = end_time_utc.clone().local();
+        end_time = useLocalTime ? end_time_local : end_time_utc;
 
         // Format the times for display. Start time is displayed as e.g. 7 Oct 12:34 unless the time is in a
         // different year to the current year, in which case the year is inserted between month and hour.
@@ -143,8 +139,8 @@ function addAlertRowsToTable(tbody, alerts) {
         // Overriding all of that, if the start time is 00:00 and the end time is 23:59 when considered in UTC, the
         // hours and minutes are stripped out from the display, as we assume the server is just giving us full days.
         // Finally, if there is no end date set, "---" is displayed.
-        var whole_days = start_time_unix.utc().format("HH:mm") == "00:00" &&
-            (end_time_unix != null || end_time_unix > 0 || end_time_unix.utc().format("HH:mm") == "23:59");
+        var whole_days = start_time_utc.format("HH:mm") == "00:00" &&
+            (end_time_utc != null || end_time_utc > 0 || end_time_utc.format("HH:mm") == "23:59");
         var hours_minutes_format = whole_days ? "" : " HH:mm";
         var start_time_formatted = start_time.format("D MMM" + hours_minutes_format);
         if (start_time.format("YYYY") != moment().format("YYYY")) {
@@ -153,11 +149,13 @@ function addAlertRowsToTable(tbody, alerts) {
             start_time_formatted = start_time.format("[Today]" + hours_minutes_format);
         }
         var end_time_formatted = "---";
-        if (end_time_unix != null && end_time_unix > 0 && end_time != null) {
+        if (end_time_utc != null && end_time_utc > 0 && end_time != null) {
             var end_time_formatted = whole_days ? start_time_formatted : end_time.format("HH:mm");
             if (end_time.format("D MMM") != start_time.format("D MMM")) {
                 if (end_time.format("YYYY") != moment().format("YYYY")) {
                     end_time_formatted = end_time.format("D MMM YYYY" + hours_minutes_format);
+                } else if (useLocalTime && end_time.format("D MMM YYYY") == moment().format("D MMM YYYY")) {
+                    end_time_formatted = end_time.format("[Today]" + hours_minutes_format);
                 } else {
                     end_time_formatted = end_time.format("D MMM" + hours_minutes_format);
                 }

webassets/js/bands.js

@@ -0,0 +1,286 @@
+// A couple of constants that must match what's in CSS. We need to know them before the content actually renders, so we
+// can't just ask the elements themselves for their dimensions.
+BAND_COLUMN_HEIGHT_EM = 62;
+BAND_COLUMN_CANVAS_WIDTH_EM = 4;
+BAND_COLUMN_FONT_SIZE = 16;
+BAND_COLUMN_HEIGHT_PX = BAND_COLUMN_HEIGHT_EM * BAND_COLUMN_FONT_SIZE;
+BAND_COLUMN_CANVAS_WIDTH_PX = BAND_COLUMN_CANVAS_WIDTH_EM * BAND_COLUMN_FONT_SIZE;
+BAND_COLUMN_SPOT_DIV_HEIGHT_PX = BAND_COLUMN_FONT_SIZE * 1.6;
+
+// Load spots and populate the bands display.
+function loadSpots() {
+    $.getJSON('/api/v1/spots' + buildQueryString(), function(jsonData) {
+        // Store last updated time
+        lastUpdateTime = moment.utc();
+        updateRefreshDisplay();
+        // Store data
+        spots = jsonData;
+        // Update bands display
+        updateBands();
+    });
+}
+
+// Build a query string for the API, based on the filters that the user has selected.
+function buildQueryString() {
+    var str =  "?";
+    ["dx_continent", "de_continent", "mode_type", "source", "band"].forEach(fn => {
+        if (!allFilterOptionsSelected(fn)) {
+            str = str + getQueryStringFor(fn) + "&";
+        }
+    });
+    str = str + "max_age=" + $("#max-spot-age option:selected").val();
+    // Additional filters for the bands view: No dupes, no QRT
+    str = str + "&dedupe=true&allow_qrt=false";
+    return str;
+}
+
+// Update the bands display
+function updateBands() {
+    // Stop here if nothing to display
+    var bandsContainer = $("#bands-container");
+    if (spots.length === 0) {
+        bandsContainer.html("<div class='alert alert-danger' role='alert'>No spots match your filters.</div>");
+        return;
+    }
+
+    // Do some harsher de-duping. Because we only display callsign, frequency and mode here, the previous
+    // de-duplication could have let some through that don't look like dupes on the map, but would do here.
+    // Typically that's a person activating two programs at the same time, e.g. POTA & WWFF.
+    spotList = removeDuplicatesForBandPanel(spots);
+
+    // Convert to a map of band names to the spots on that band. Bands with no
+    // spots in view will not be present.
+    const bandToSpots = new Map();
+    options["bands"].forEach(function (band) {
+        const matchingSpots = spotList.filter(function (s) {
+            return s.band === band.name;
+        });
+        if (matchingSpots.length > 0) {
+            bandToSpots.set(band.name, matchingSpots);
+        }
+    });
+
+    // Track if any columns end up taller than expected, so we can resize the container and avoid vertical scroll.
+    var maxHeightBand = 0;
+
+    // Build up table content for each band
+    var table = $('<table id="bands-table">').append('<thead><tr></tr></thead><tbody><tr></tr></tbody>');
+    bandToSpots.forEach(function (spotList, bandName) {
+        // Get the colours for the band from the first spot, and prepare the header
+        table.find('thead tr').append(`<th style='background-color:${spotList[0].band_color}; color:${spotList[0].band_contrast_color}'>${spotList[0].band}</th>`);
+
+        // Get the band data to fetch start and end frequencies
+        let band = options["bands"].filter(function (b) {
+            return b.name === bandName;
+        })[0];
+
+        // Print the frequency band markers. This is 41 steps to divide the band evenly into 40 markers. One in every
+        // four will show the actual frequency, the others will just be dashes.
+        bandMarkersDiv = $('<div class="band-markers">');
+        const freqStep = (band.end_freq - band.start_freq) / 40.0;
+        for (let i = 0; i <= 40; i++) {
+            if (i % 4 === 0) {
+                bandMarkersDiv.append("&mdash;" + ((band.start_freq + i * freqStep)/1000000).toFixed(3) + "<br/>");
+            } else if (i % 4 === 2) {
+                bandMarkersDiv.append("&ndash;<br/>");
+            } else {
+                bandMarkersDiv.append("-<br/>");
+            }
+        }
+
+        // Prepare the spots list
+        var bandSpotsDiv = $("<div class='band-spots'>");
+        var lastSpotPxDownBand = -999;
+        // Sort by frequency so have a consistent order in which to plan where they will appear on the band div.
+        spotList.sort(function(a, b) { return a.freq - b.freq; });
+        // First calculate how we should be displaying the spots. There are three "modes" to try to place them in a
+        // visually appealing way:
+        // 1) Spaced normally, not going over the end of the band, so we populate them forwards.
+        // 2) Would go over the end, but the spots don't fill the band, so we populate them backwards.
+        // 3) Spots totally fill the band (or more), so we space them evenly starting at the top.
+        // In each case, we don't add anything to the DOM yet, we just calculate "pxDownBandLabel" (how far the *top* of
+        // the label is from the top of the div) and add that as a property to the spot for later use.
+        if (spotList.length >= BAND_COLUMN_HEIGHT_PX / BAND_COLUMN_SPOT_DIV_HEIGHT_PX) {
+            // Mode 3.
+            // Just lay out all spots simply, starting at 0px offset and working down with each one touching.
+            lastSpotPxDownBand = 0 - BAND_COLUMN_SPOT_DIV_HEIGHT_PX;
+            spotList.forEach(s => {
+                lastSpotPxDownBand = lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX;
+                s["pxDownBandLabel"] = lastSpotPxDownBand;
+            });
+        } else {
+            // Mode 1 or 2. Run through adding things to the list forwards as a test.
+            spotList.forEach(s => {
+                // Work out how far down the div to draw it
+                var percentDownBand = (s.freq - band.start_freq) / (band.end_freq - band.start_freq) * 0.97; // not 100% due to fudge, the first and last dashes are not exactly at the top and bottom of the div as some space is needed for text
+                var pxDownBand = percentDownBand * BAND_COLUMN_HEIGHT_PX;
+                if (pxDownBand < lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX) {
+                    pxDownBand = lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX; // Prevent overlap
+                }
+                s["pxDownBandLabel"] = pxDownBand;
+                lastSpotPxDownBand = pxDownBand;
+            });
+
+            // Work out if we overflowed the end.
+            if (lastSpotPxDownBand <= BAND_COLUMN_HEIGHT_PX) {
+                // Mode 1. Current positions are fine and there's nothing to do.
+            } else {
+                // Mode 2. Repeat the process but backwards, starting at the end and working upwards.
+                lastSpotPxDownBand = 999999;
+                spotList.reverse().forEach(s => {
+                    // Work out how far down the div to draw it
+                    var percentDownBand = (s.freq - band.start_freq) / (band.end_freq - band.start_freq) * 0.97; // not 100% due to fudge, the first and last dashes are not exactly at the top and bottom of the div as some space is needed for text
+                    var pxDownBand = percentDownBand * BAND_COLUMN_HEIGHT_PX;
+                    if (pxDownBand > lastSpotPxDownBand - BAND_COLUMN_SPOT_DIV_HEIGHT_PX) {
+                        pxDownBand = lastSpotPxDownBand - BAND_COLUMN_SPOT_DIV_HEIGHT_PX; // Prevent overlap
+                    }
+                    s["pxDownBandLabel"] = pxDownBand;
+                    lastSpotPxDownBand = pxDownBand;
+                });
+            }
+        }
+
+        // Now each spot is tagged with how far down the div it should go, add them to the DOM.
+        spotList.forEach(s => {
+            bandSpotsDiv.append(`<div class="band-spot" style="top: ${s['pxDownBandLabel']}px; border-top: 1px solid ${s.band_color}; border-left: 5px solid ${s.band_color}; border-bottom: 1px solid ${s.band_color}; border-right: 1px solid ${s.band_color};"><span class="band-spot-call">${s.dx_call}</span><span class="band-spot-info">${s.dx_call} ${(s.freq/1000000).toFixed(3)} ${s.mode}</span></div>`);
+        });
+
+        // Work out how tall the canvas should be. Normally this is matching the normal band column height, but if some
+        // spots have gone off the end of the band markers and stretched their div, we need to resize the canvas to
+        // match, otherwise we have nowhere to draw their connecting lines.
+        var canvasHeight = Math.max(BAND_COLUMN_HEIGHT_PX, lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX);
+        maxHeightBand = Math.max(maxHeightBand, canvasHeight);
+
+        // Draw horizontal or diagonal lines to join up the "real" frequency with where the spot div ended up
+        var bandLinesCanvas = $(`<canvas class='band-lines-canvas' width='${BAND_COLUMN_CANVAS_WIDTH_PX}px' height='${canvasHeight}px' style='height:${canvasHeight}px !important;'>`);
+        spotList.forEach(s => {
+            // Work out how far down the div to draw it
+            var percentDownBand = (s.freq - band.start_freq) / (band.end_freq - band.start_freq) * 0.97; // not 100% due to fudge, the first and last dashes are not exactly at the top and bottom of the div as some space is needed for text
+            var pxDownBandFreq = (percentDownBand + 0.015) * BAND_COLUMN_HEIGHT_PX; // same fudge but add half to put the left end of the line in the right place
+            var pxDownBandLabel = s["pxDownBandLabel"] + (BAND_COLUMN_SPOT_DIV_HEIGHT_PX / 1.75); // line should be to the vertical text-centre spot, not to the top corner
+
+            // Draw the line on the canvas
+            var ctx = bandLinesCanvas[0].getContext('2d');
+            ctx.beginPath();
+            ctx.lineWidth = 2;
+            ctx.lineCap = "round";
+            ctx.strokeStyle = s.band_color;
+            ctx.moveTo(0, pxDownBandFreq);
+            ctx.lineTo(BAND_COLUMN_CANVAS_WIDTH_PX, pxDownBandLabel);
+            ctx.stroke();
+        });
+
+        // Assemble the table cell
+        td = $("<td>");
+        container = $("<div class='band-container'>");
+        container.append(bandLinesCanvas);
+        container.append(bandMarkersDiv);
+        container.append(bandSpotsDiv);
+        td.append(container);
+        table.find('tbody tr').append(td);
+    });
+
+    // Update the DOM with the band HTML
+    bandsContainer.html(table);
+
+    // Increase the height of the bands container so we don't have any vertical scroll bars except the browser ones
+    bandsContainer.css("min-height", `${maxHeightBand + 42}px`);
+
+    // Desktop mouse wheel to scroll bands horizontally if used on the headers
+    table.find('thead tr').on("wheel", () => {
+        bandsContainer.scrollLeft(bandsContainer.scrollLeft() + event.deltaY / 10.0);
+        return false;
+    });
+}
+
+// Iterate through a temporary list of spots, merging duplicates in a way suitable for the band panel. If two or more
+// spots with the activator, mode and frequency are found, these will be merged and reduced until only one remains,
+// with the best data. Note that unlike removeDuplicates(), which operates on the main spot map, this operates only
+// on the temporary array of spots provided as an argument, and returns the output, for use when constructing the
+// band panel.
+function removeDuplicatesForBandPanel(spotList) {
+    const spotsToRemove = [];
+    spotList.forEach(function (check) {
+        spotList.forEach(function (s) {
+            if (s !== check) {
+                if (s.dx_call === check.dx_call && s.freq === check.freq && s.mode === check.mode) {
+                    // Find which one to keep and which to delete
+                    const checkSpotNewer = check.time > s.time;
+                    const keepSpot = checkSpotNewer ? check : s;
+                    const deleteSpot = checkSpotNewer ? s : check;
+                    // Aggregate list of spots to remove
+                    spotsToRemove.push(deleteSpot.uid);
+                }
+            }
+        });
+    });
+    // Perform the removal
+    return spotList.filter(s => !spotsToRemove.includes(s.uid));
+}
+
+// Load server options. Once a successful callback is made from this, we then query spots and set up the timer to query
+// spots repeatedly.
+function loadOptions() {
+    $.getJSON('/api/v1/options', function(jsonData) {
+        // Store options
+        options = jsonData;
+
+        // Add CSS for band toggle buttons
+        addBandToggleColourCSS(options["bands"]);
+
+        // Populate the filters panel
+        generateBandsMultiToggleFilterCard(options["bands"]);
+        generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
+        generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
+        generateMultiToggleFilterCard("#mode-options", "mode_type", options["mode_types"]);
+        generateMultiToggleFilterCard("#source-options", "source", options["spot_sources"]);
+
+        // Load settings from settings storage now all the controls are available
+        loadSettings();
+
+        // Load spots and set up the timer
+        loadSpots();
+        setInterval(loadSpots, REFRESH_INTERVAL_SEC * 1000);
+    });
+}
+
+// Method called when any display property is changed to reload the map and persist the display settings.
+function displayUpdated() {
+    updateMap();
+    saveSettings();
+}
+
+// React to toggling/closing panels
+function toggleFiltersPanel() {
+    // If we are going to show the filters panel, hide the display panel
+    if (!$("#filters-area").is(":visible") && $("#display-area").is(":visible")) {
+        $("#display-area").hide();
+        $("#display-button").button("toggle");
+    }
+    $("#filters-area").toggle();
+}
+function closeFiltersPanel() {
+    $("#filters-button").button("toggle");
+    $("#filters-area").hide();
+}
+
+function toggleDisplayPanel() {
+    // If we are going to show the display panel, hide the filters panel
+    if (!$("#display-area").is(":visible") && $("#filters-area").is(":visible")) {
+        $("#filters-area").hide();
+        $("#filters-button").button("toggle");
+    }
+    $("#display-area").toggle();
+}
+function closeDisplayPanel() {
+    $("#display-button").button("toggle");
+    $("#display-area").hide();
+}
+
+// Startup
+$(document).ready(function() {
+    // Call loadOptions(), this will then trigger loading spots and setting up timers.
+    loadOptions();
+    // Update the refresh timing display every second
+    setInterval(updateRefreshDisplay, 1000);
+});

webassets/js/map.js

@@ -3,7 +3,7 @@ var markersLayer;
 var geodesicsLayer;
 var terminator;
 
-// Load spots and populate the table.
+// Load spots and populate the map.
 function loadSpots() {
     $.getJSON('/api/v1/spots' + buildQueryString(), function(jsonData) {
         // Store data
@@ -23,6 +23,8 @@ function buildQueryString() {
         }
     });
     str = str + "max_age=" + $("#max-spot-age option:selected").val();
+    // Additional filters for the map view: No dupes, no QRT, only spots with good locations
+    str = str + "&dedupe=true&allow_qrt=false&needs_good_location=true";
     return str;
 }
 
@@ -32,29 +34,20 @@ function updateMap() {
     markersLayer.clearLayers();
     geodesicsLayer.clearLayers();
 
-    // Make new markers for all spots with a good location, not QRT, and not a duplicate spot within the data set.
-    var callsAlreadyDisplayed = [];
+    // Make new markers for all spots that match the filter
     spots.forEach(function (s) {
-        if (s["dx_location_good"] && (s["qrt"] == null || s["qrt"] == false)) {
-            if (!callsAlreadyDisplayed.includes(s["dx_call"])) {
+        var m = L.marker([s["dx_latitude"], s["dx_longitude"]], {icon: getIcon(s)});
+        m.bindPopup(getTooltipText(s));
+        markersLayer.addLayer(m);
 
-                // OK, create the marker
-                var m = L.marker([s["dx_latitude"], s["dx_longitude"]], {icon: getIcon(s)});
-                m.bindPopup(getTooltipText(s));
-                markersLayer.addLayer(m);
-
-                // Create geodesics if required
-                if ($("#mapShowGeodesics")[0].checked && s["de_latitude"] != null && s["de_longitude"] != null) {
-                    var geodesic = L.geodesic([[s["de_latitude"], s["de_longitude"]], m.getLatLng()], {
-                        color: s["band_color"],
-                        wrap: false,
-                        steps: 5
-                    });
-                    geodesicsLayer.addLayer(geodesic);
-                }
-
-            }
-            callsAlreadyDisplayed.push(s["dx_call"]);
+        // Create geodesics if required
+        if ($("#mapShowGeodesics")[0].checked && s["de_latitude"] != null && s["de_longitude"] != null) {
+            var geodesic = L.geodesic([[s["de_latitude"], s["de_longitude"]], m.getLatLng()], {
+                color: s["band_color"],
+                wrap: false,
+                steps: 5
+            });
+            geodesicsLayer.addLayer(geodesic);
         }
     });
 }

webassets/js/spots.js

@@ -88,7 +88,7 @@ function updateTable() {
         // Format a UTC or local time for display
         var time = moment.unix(s["time"]).utc();
         if (useLocalTime) {
-            time = time.local();
+            time.local();
         }
         var time_formatted = time.format("HH:mm");
 
@@ -185,7 +185,7 @@ function updateTable() {
             $tr.append(`<td class='nowrap'>${time_formatted}</td>`);
         }
         if (showDX) {
-            $tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${dx_flag}</span><a class='dx-link' href='https://qrz.com/db/${s["dx_call"]}' target='_new'>${s["dx_call"]}</a></td>`);
+            $tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${dx_flag}</span><a class='dx-link' href='https://qrz.com/db/${s["dx_call"]}' target='_new' title='${s["dx_name"] != null ? s["dx_name"] : ""}'>${s["dx_call"]}</a></td>`);
         }
         if (showFreq) {
             $tr.append(`<td class='nowrap'><span class='band-bullet' title='${bandFullName}' style='color: ${s["band_color"]}'>&#9632;</span>${freq_string}</td>`);