FAQ

README.md

@@ -14,6 +14,8 @@ Supported data sources include DX Clusters, the Reverse Beacon Network (RBN), th
 
 ![Screenshot](/images/screenshot2.png)
 
+![Screenshot](/images/screenshot3.png)
+
 ### Accessing the public version
 
 You can access the public version's web interface at [https://spothole.app](https://spothole.app), and see [https://spothole.app/apidocs](https://spothole.app/apidocs) for the API details.

alertproviders/parksnpeaks.py

@@ -0,0 +1,55 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from alertproviders.http_alert_provider import HTTPAlertProvider
+from core.sig_utils import get_icon_for_sig
+from data.alert import Alert
+
+
+# Alert provider for Parks n Peaks
+class ParksNPeaks(HTTPAlertProvider):
+    POLL_INTERVAL_SEC = 3600
+    ALERTS_URL = "http://parksnpeaks.org/api/ALERTS/"
+
+    def __init__(self, provider_config):
+        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json():
+            # Calculate some things
+            if " - " in source_alert["Location"]:
+                split = source_alert["Location"].split(" - ")
+                sig_ref = split[0]
+                sig_ref_name = split[1]
+            else:
+                sig_ref = source_alert["WWFFID"]
+                sig_ref_name = source_alert["Location"]
+            start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(
+                tzinfo=pytz.UTC).timestamp()
+
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert["alID"],
+                          dx_calls=[source_alert["CallSign"].upper()],
+                          freqs_modes=source_alert["Freq"] + " " + source_alert["MODE"],
+                          comment=source_alert["Comments"],
+                          sig=source_alert["Class"],
+                          sig_refs=[sig_ref],
+                          sig_refs_names=[sig_ref_name],
+                          icon=get_icon_for_sig(source_alert["Class"]),
+                          start_time=start_time,
+                          is_dxpedition=False)
+
+            # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
+            if alert.sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA"]:
+                logging.warn("PNP alert found with sig " + alert.sig + ", developer needs to add support for this!")
+
+            # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
+            # the alert list.
+            if alert.sig not in ["POTA", "SOTA", "WWFF"]:
+                new_alerts.append(alert)
+        return new_alerts

alertproviders/pota.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import pytz
 
 from alertproviders.http_alert_provider import HTTPAlertProvider
+from core.sig_utils import get_icon_for_sig
 from data.alert import Alert
 
 
@@ -27,7 +28,7 @@ class POTA(HTTPAlertProvider):
                           sig="POTA",
                           sig_refs=[source_alert["reference"]],
                           sig_refs_names=[source_alert["name"]],
-                          icon="tree",
+                          icon=get_icon_for_sig("POTA"),
                           start_time=datetime.strptime(source_alert["startDate"] + source_alert["startTime"],
                                                        "%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
                           end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"],

alertproviders/sota.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import pytz
 
 from alertproviders.http_alert_provider import HTTPAlertProvider
+from core.sig_utils import get_icon_for_sig
 from data.alert import Alert
 
 
@@ -28,7 +29,7 @@ class SOTA(HTTPAlertProvider):
                           sig="SOTA",
                           sig_refs=[source_alert["associationCode"] + "/" + source_alert["summitCode"]],
                           sig_refs_names=[source_alert["summitDetails"]],
-                          icon="mountain-sun",
+                          icon=get_icon_for_sig("SOTA"),
                           start_time=datetime.strptime(source_alert["dateActivated"],
                                                        "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=pytz.UTC).timestamp(),
                           is_dxpedition=False)

alertproviders/wwff.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import pytz
 
 from alertproviders.http_alert_provider import HTTPAlertProvider
+from core.sig_utils import get_icon_for_sig
 from data.alert import Alert
 
 
@@ -26,7 +27,7 @@ class WWFF(HTTPAlertProvider):
                           comment=source_alert["remarks"],
                           sig="WWFF",
                           sig_refs=[source_alert["reference"]],
-                          icon="seedling",
+                          icon=get_icon_for_sig("WWFF"),
                           start_time=datetime.strptime(source_alert["utc_start"],
                                                        "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
                           end_time=datetime.strptime(source_alert["utc_end"],

config-example.yml

@@ -84,6 +84,10 @@ alert-providers:
     class: "WWFF"
     name: "WWFF"
     enabled: true
+  -
+    class: "ParksNPeaks"
+    name: "ParksNPeaks"
+    enabled: true
   -
     class: "NG3K"
     name: "NG3K"

core/constants.py

@@ -1,20 +1,38 @@
 from core.config import SERVER_OWNER_CALLSIGN
 from data.band import Band
+from data.sig import SIG
 
 # General software
 SOFTWARE_NAME = "Spothole by M0TRT"
 SOFTWARE_VERSION = "0.1"
 
 # HTTP headers used for spot providers that use HTTP
-HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + " " + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
+HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + ", v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
 
 # Special Interest Groups
-SIGS = ["POTA", "SOTA", "WWFF", "GMA", "WWBOTA", "HEMA", "MOTA", "ARLHS", "SiOTA", "WCA"]
+SIGS = [
+   SIG(name="POTA", description="Parks on the Air", icon="tree", ref_regex=r"[A-Z]{2}\-\d+"),
+   SIG(name="SOTA", description="Summits on the Air", icon="mountain-sun", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d+"),
+   SIG(name="WWFF", description="World Wide Flora & Fauna", icon="seedling", ref_regex=r"[A-Z0-9]{1,3}FF\-\d+"),
+   SIG(name="GMA", description="Global Mountain Activity", icon="person-hiking", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d+"),
+   SIG(name="WWBOTA", description="Worldwide Bunkers on the Air", icon="radiation", ref_regex=r"B\/[A-Z0-9]{1,3}\-\d+"),
+   SIG(name="HEMA", description="HuMPs Excluding Marilyns Award", icon="mound", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d+"),
+   SIG(name="IOTA", description="Islands on the Air", icon="umbrella-beach", ref_regex=r"[A-Z]{2}\-\d+"),
+   SIG(name="MOTA", description="Mills on the Air", icon="fan", ref_regex=r"X\d{4-6}"),
+   SIG(name="ARLHS", description="Amateur Radio Lighthouse Society", icon="tower-observation", ref_regex=r"[A-Z]{3}\-\d+"),
+   SIG(name="ILLW", description="International Lighthouse & Lightship Weekend", icon="tower-observation", ref_regex=r"[A-Z]{2}\d{4}"),
+   SIG(name="SIOTA", description="Silos on the Air", icon="wheat-awn", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
+   SIG(name="WCA", description="World Castles Award", icon="chess-rook", ref_regex=r"[A-Z0-9]{1,3}\-\d+"),
+   SIG(name="ZLOTA", description="New Zealand on the Air", icon="kiwi-bird", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d+"),
+   SIG(name="KRMNPA", description="Keith Roget Memorial National Parks Award", icon="earth-oceania", ref_regex=r""),
+   SIG(name="WAB", description="Worked All Britain", icon="table-cells-large", ref_regex=r"[A-Z]{1,2}[0-9]{2}"),
+   SIG(name="WAI", description="Worked All Ireland", icon="table-cells-large", ref_regex=r"[A-Z][0-9]{2}")
+]
 
 # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
 CW_MODES = ["CW"]
 PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "M17"]
-DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "BPSK", "PSK", "PSK31", "BPSK31", "OLIVIA"]
+DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "BPSK", "PSK", "PSK31", "BPSK31", "OLIVIA", "MFSK", "MFSK32"]
 ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
 MODE_TYPES = ["CW", "PHONE", "DATA"]
 

core/geo_utils.py

@@ -0,0 +1,103 @@
+import logging
+import re
+from math import floor
+
+from pyproj import Transformer
+
+TRANSFORMER_OS_GRID_TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326")
+TRANSFORMER_IRISH_GRID_TO_WGS84 = Transformer.from_crs("EPSG:29903", "EPSG:4326")
+TRANSFORMER_CI_UTM_GRID_TO_WGS84 = Transformer.from_crs("+proj=utm +zone=30 +ellps=WGS84", "EPSG:4326")
+
+
+# Convert a Worked All Britain or Worked All Ireland reference to a lat/lon point.
+def wab_wai_square_to_lat_lon(ref):
+    # First check we have a valid grid square, and based on what it looks like, use either the Ordnance Survey, Irish,
+    # or UTM grid systems to perform the conversion.
+    if re.match(r"^[HNOST][ABCDEFGHJKLMNOPQRSTUVWXYZ][0-9]{2}$", ref):
+        return os_grid_square_to_lat_lon(ref)
+    elif re.match(r"^[ABCDEFGHJKLMNOPQRSTUVWXYZ][0-9]{2}$", ref):
+        return irish_grid_square_to_lat_lon(ref)
+    elif re.match(r"^W[AV][0-9]{2}$", ref):
+        return utm_grid_square_to_lat_lon(ref)
+    else:
+        logging.warn("Invalid WAB/WAI square: " + ref)
+        return None
+
+
+# Get a lat/lon point for the centre of an Ordnance Survey grid square
+def os_grid_square_to_lat_lon(ref):
+    # Convert the letters into multipliers for the 500km squares and 100km squares
+    offset_500km_multiplier = ord(ref[0]) - 65
+    offset_100km_multiplier = ord(ref[1]) - 65
+
+    # The letter "I" is not used in the grid, so any offset of 8 or more needs to be reduced by 1.
+    if offset_500km_multiplier >= 8:
+        offset_500km_multiplier = offset_500km_multiplier - 1
+    if offset_100km_multiplier >= 8:
+        offset_100km_multiplier = offset_100km_multiplier - 1
+
+    # Convert the offsets into increments of 100km from the false origin (grid square SV):
+    easting_100km = ((offset_500km_multiplier - 2) % 5) * 5 + (offset_100km_multiplier % 5)
+    northing_100km = (19 - floor(offset_500km_multiplier / 5) * 5) - floor(offset_100km_multiplier / 5)
+
+    # Take the numeric parts of the grid square and multiply by 10000 to get metres, then combine with the 100km
+    # box offsets
+    easting = int(ref[2]) * 10000 + easting_100km * 100000
+    northing = int(ref[3]) * 10000 + northing_100km * 100000
+
+    # Add 5000m to each value to get the middle of the box rather than the south-west corner
+    easting = easting + 5000
+    northing = northing + 5000
+
+    # Reproject to WGS84 lat/lon
+    lat, lon = TRANSFORMER_OS_GRID_TO_WGS84.transform(easting, northing)
+    return lat, lon
+
+
+# Get a lat/lon point for the centre of an Irish Grid square.
+def irish_grid_square_to_lat_lon(ref):
+    # Convert the letters into multipliers for the 100km squares
+    offset_100km_multiplier = ord(ref[0]) - 65
+
+    # The letter "I" is not used in the grid, so any offset of 8 or more needs to be reduced by 1.
+    if offset_100km_multiplier >= 8:
+        offset_100km_multiplier = offset_100km_multiplier - 1
+
+    # Convert the offsets into increments of 100km from the false origin:
+    easting_100km = offset_100km_multiplier % 5
+    northing_100km = 4 - floor(offset_100km_multiplier / 5)
+
+    # Take the numeric parts of the grid square and multiply by 10000 to get metres, then combine with the 100km
+    # box offsets
+    easting = int(ref[1]) * 10000 + easting_100km * 100000
+    northing = int(ref[2]) * 10000 + northing_100km * 100000
+
+    # Add 5000m to each value to get the middle of the box rather than the south-west corner
+    easting = easting + 5000
+    northing = northing + 5000
+
+    # Reproject to WGS84 lat/lon
+    lat, lon = TRANSFORMER_IRISH_GRID_TO_WGS84.transform(easting, northing)
+    return lat, lon
+
+
+# Get a lat/lon point for the centre of a UTM grid square (supports only squares WA & WV for the Channel Islands, nothing else implemented)
+def utm_grid_square_to_lat_lon(ref):
+    # Take the numeric parts of the grid square and multiply by 10000 to get metres from the corner of the letter-based grid square
+    easting = int(ref[2]) * 10000
+    northing = int(ref[3]) * 10000
+
+    # Apply the appropriate offset based on whether the square is WA or WV
+    easting = easting + 500000
+    if ref[1] == "A":
+        northing = northing + 5500000
+    else:
+        northing = northing + 5400000
+
+    # Add 5000m to each value to get the middle of the box rather than the south-west corner
+    easting = easting + 5000
+    northing = northing + 5000
+
+    # Reproject to WGS84 lat/lon
+    lat, lon = TRANSFORMER_CI_UTM_GRID_TO_WGS84.transform(easting, northing)
+    return lat, lon

core/sig_utils.py

@@ -0,0 +1,21 @@
+from core.constants import SIGS
+
+# Utility function to get the icon for a named SIG. If no match is found, the "circle-question" icon will be returned.
+def get_icon_for_sig(sig):
+    for s in SIGS:
+        if s.name == sig:
+            return s.icon
+    return "circle-question"
+
+# Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned.
+def get_ref_regex_for_sig(sig):
+    for s in SIGS:
+        if s.name == sig:
+            return s.ref_regex
+    return None
+
+# Regex matching any SIG
+ANY_SIG_REGEX = r"(" + r"|".join(list(map(lambda p: p.name, SIGS))) + r")"
+
+# Regex matching any SIG reference
+ANY_XOTA_SIG_REF_REGEX = r"[\w\/]+\-\d+"

data/alert.py

@@ -1,6 +1,7 @@
 import copy
 import hashlib
 import json
+import re
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 
@@ -8,6 +9,7 @@ import pytz
 
 from core.constants import DXCC_FLAGS
 from core.lookup_helper import lookup_helper
+from core.sig_utils import get_icon_for_sig
 
 
 # Data class that defines an alert.
@@ -58,7 +60,7 @@ class Alert:
     # Activation score. SOTA only
     activation_score: int = None
     # Icon, from the Font Awesome set. This is fairly opinionated but is here to help the alerthole web UI and Field alertter. Does not include the "fa-" prefix.
-    icon: str = "question"
+    icon: str = None
     # Whether this alert is for a DXpedition, as opposed to e.g. an xOTA programme.
     is_dxpedition: bool = False
     # Where we got the alert from, e.g. "POTA", "SOTA"...
@@ -99,12 +101,21 @@ class Alert:
         if self.dx_dxcc_id and self.dx_dxcc_id in DXCC_FLAGS and not self.dx_flag:
             self.dx_flag = DXCC_FLAGS[self.dx_dxcc_id]
 
+        # Icon from SIG
+        if self.sig and not self.icon:
+            self.icon = get_icon_for_sig(self.sig)
+
         # DX operator details lookup, using QRZ.com. This should be the last resort compared to taking the data from
         # the actual alertting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of
         # the one from the park reference they're at.
         if self.dx_calls and not self.dx_names:
             self.dx_names = list(map(lambda c: lookup_helper.infer_name_from_callsign(c), self.dx_calls))
 
+        # Clean up comments
+        if self.comment:
+            comment = re.sub(r"\(de [A-Za-z0-9]*\)", "", self.comment)
+            self.comment = comment.strip()
+
         # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
         # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
         # apart from that they were retrieved from the API at different times. Note that the simple Python hash()

data/sig.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+
+# Data class that defines a Special Interest Group.
+@dataclass
+class SIG:
+    # SIG name, e.g. "POTA"
+    name: str
+    # Description, e.g. "Parks on the Air"
+    description: str
+    # Icon to use for it, from the Font Awesome set. This is fairly opinionated but is here to help the Spothole web UI
+    # and Field Spotter. Does not include the "fa-" prefix.
+    icon: str
+    # Regex matcher for references, e.g. for POTA r"[A-Z]{2}\-\d+".
+    ref_regex: str

data/spot.py

@@ -2,6 +2,7 @@ import copy
 import hashlib
 import json
 import logging
+import re
 from dataclasses import dataclass
 from datetime import datetime
 
@@ -9,7 +10,9 @@ import pytz
 from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 
 from core.constants import DXCC_FLAGS
+from core.geo_utils import wab_wai_square_to_lat_lon
 from core.lookup_helper import lookup_helper
+from core.sig_utils import get_icon_for_sig
 
 
 # Data class that defines a spot.
@@ -18,7 +21,6 @@ class Spot:
     # Unique identifier for the spot
     id: str = None
 
-
     # DX (spotted) operator info
 
     # Callsign of the operator that has been spotted
@@ -49,12 +51,13 @@ class Spot:
     # lookup
     dx_latitude: float = None
     dx_longitude: float = None
-    # DX Location source. Indicates how accurate the location might be. Values: "SPOT", "QRZ, "DXCC", "NONE"
+    # DX Location source. Indicates how accurate the location might be. Values: "SPOT", "WAB/WAI GRID", "QRZ", "DXCC", "NONE"
     dx_location_source: str = "NONE"
-    # DX Location good. Indicates that the software thinks the location data is good enough to plot on a map.
+    # DX Location good. Indicates that the software thinks the location data is good enough to plot on a map. This is
+    # true if the location source is "SPOT" or "WAB/WAI GRID", or if the location source is "QRZ" and the DX callsign
+    # doesn't have a suffix like /P.
     dx_location_good: bool = False
 
-
     # DE (Spotter) info
 
     # Callsign of the spotter
@@ -75,7 +78,6 @@ class Spot:
     de_latitude: float = None
     de_longitude: float = None
 
-
     # General QSO info
 
     # Reported mode, such as SSB, PHONE, CW, FT8...
@@ -93,7 +95,6 @@ class Spot:
     # QRT state. Some APIs return spots marked as QRT. Otherwise we can check the comments.
     qrt: bool = False
 
-
     # Special Interest Group info
 
     # Special Interest Group (SIG), e.g. outdoor activity programme such as POTA
@@ -102,21 +103,21 @@ class Spot:
     sig_refs: list = None
     # SIG reference names
     sig_refs_names: list = None
+    # SIG reference URLs
+    sig_refs_urls: list = None
     # Activation score. SOTA only
     activation_score: int = None
 
-
     # Display guidance (optional)
 
     # Icon, from the Font Awesome set. This is fairly opinionated but is here to help the Spothole web UI and Field
     # Spotter. Does not include the "fa-" prefix.
-    icon: str = "question"
+    icon: str = None
     # Colour to represent this spot, if a client chooses to colour spots based on their frequency band, using PSK
     # Reporter's default colours. HTML colour e.g. hex. A contrast colour is also provided which will be black or white.
     band_color: str = None
     band_contrast_color: str = None
 
-
     # Timing info
 
     # Time of the spot, UTC seconds since UNIX epoch
@@ -130,7 +131,6 @@ class Spot:
     # Time that this software received the spot, ISO 8601
     received_time_iso: str = None
 
-
     # Source info
 
     # Where we got the spot from, e.g. "POTA", "Cluster"...
@@ -215,6 +215,10 @@ class Spot:
         if self.mode and not self.mode_type:
             self.mode_type = lookup_helper.infer_mode_type_from_mode(self.mode)
 
+        # Icon from SIG
+        if self.sig and not self.icon:
+            self.icon = get_icon_for_sig(self.sig)
+
         # DX Grid to lat/lon and vice versa
         if self.dx_grid and not self.dx_latitude:
             ll = locator_to_latlong(self.dx_grid)
@@ -228,10 +232,31 @@ class Spot:
         if self.dx_latitude:
             self.dx_location_source = "SPOT"
 
+        # WAB/WAI grid to lat/lon
+        if not self.dx_latitude and self.sig and self.sig_refs and len(self.sig_refs) > 0 and (
+                self.sig == "WAB" or self.sig == "WAI"):
+            ll = wab_wai_square_to_lat_lon(self.sig_refs[0])
+            if ll:
+                self.dx_latitude = ll[0]
+                self.dx_longitude = ll[1]
+                try:
+                    self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8)
+                except:
+                    logging.debug("Invalid lat/lon received from WAB/WAI grid")
+                self.dx_location_source = "WAB/WAI GRID"
+
         # QRT comment detection
         if self.comment and not self.qrt:
             self.qrt = "QRT" in self.comment.upper()
 
+        # Clean up comments
+        if self.comment:
+            comment = re.sub(r"\(de [A-Za-z0-9]*\)", "", self.comment)
+            comment = re.sub(r"\[.*]:", "", comment)
+            comment = re.sub(r"\[.*]", "", comment)
+            comment = re.sub(r"\"\"", "", comment)
+            self.comment = comment.strip()
+
         # DX operator details lookup, using QRZ.com. This should be the last resort compared to taking the data from
         # the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of
         # the one from the park reference they're at.
@@ -256,11 +281,11 @@ class Spot:
 
         # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
         # is likely at home.
-        self.dx_location_good = self.dx_location_source == "SPOT" or (
+        self.dx_location_good = self.dx_location_source == "SPOT" or self.dx_location_source == "WAB/WAI GRID" or (
                 self.dx_location_source == "QRZ" and not "/" in self.dx_call)
 
-        # DE of "RBNHOLE" and "SOTAMAT" are not things we can look up location for
+        # DE of "RBNHOLE", "SOTAMAT" and "ZLOTA" are not things we can look up location for
-        if self.de_call != "RBNHOLE" and self.de_call != "SOTAMAT":
+        if self.de_call != "RBNHOLE" and self.de_call != "SOTAMAT" and self.de_call != "ZLOTA":
             # DE operator position lookup, using QRZ.com.
             if self.de_call and not self.de_latitude:
                 latlon = lookup_helper.infer_latlon_from_callsign_qrz(self.de_call)

requirements.txt

@@ -10,3 +10,4 @@ diskcache~=5.6.3
 psutil~=7.1.0
 requests-sse~=0.5.2
 rss-parser~=2.1.1
+pyproj~=3.7.2

server/webserver.py

@@ -31,14 +31,15 @@ class WebServer:
         bottle.BaseTemplate.defaults['software_version'] = SOFTWARE_VERSION
 
         # Routes for API calls
-        bottle.get("/api/v1/spots")(lambda: self.serve_api(self.get_spot_list_with_filters()))
+        bottle.get("/api/v1/spots")(lambda: self.serve_spots_api())
-        bottle.get("/api/v1/alerts")(lambda: self.serve_api(self.get_alert_list_with_filters()))
+        bottle.get("/api/v1/alerts")(lambda: self.serve_alerts_api())
         bottle.get("/api/v1/options")(lambda: self.serve_api(self.get_options()))
         bottle.get("/api/v1/status")(lambda: self.serve_api(self.status_data))
         bottle.post("/api/v1/spot")(lambda: self.accept_spot())
         # Routes for templated pages
         bottle.get("/")(lambda: self.serve_template('webpage_spots'))
         bottle.get("/map")(lambda: self.serve_template('webpage_map'))
+        bottle.get("/bands")(lambda: self.serve_template('webpage_bands'))
         bottle.get("/alerts")(lambda: self.serve_template('webpage_alerts'))
         bottle.get("/status")(lambda: self.serve_template('webpage_status'))
         bottle.get("/about")(lambda: self.serve_template('webpage_about'))
@@ -56,6 +57,38 @@ class WebServer:
         self.status = "Waiting"
         run(host='localhost', port=self.port)
 
+    # Serve the JSON API /spots endpoint
+    def serve_spots_api(self):
+        try:
+            data = self.get_spot_list_with_filters()
+            return self.serve_api(data)
+        except ValueError as e:
+            logging.error(e)
+            response.content_type = 'application/json'
+            response.status = 400
+            return json.dumps("Bad request - " + str(e), default=serialize_everything)
+        except Exception as e:
+            logging.error(e)
+            response.content_type = 'application/json'
+            response.status = 500
+            return json.dumps("Error - " + str(e), default=serialize_everything)
+
+    # Serve the JSON API /alerts endpoint
+    def serve_alerts_api(self):
+        try:
+            data = self.get_alert_list_with_filters()
+            return self.serve_api(data)
+        except ValueError as e:
+            logging.error(e)
+            response.content_type = 'application/json'
+            response.status = 400
+            return json.dumps("Bad request - " + str(e), default=serialize_everything)
+        except Exception as e:
+            logging.error(e)
+            response.content_type = 'application/json'
+            response.status = 500
+            return json.dumps("Error - " + str(e), default=serialize_everything)
+
     # Serve a JSON API endpoint
     def serve_api(self, data):
         self.last_api_access_time = datetime.now(pytz.UTC)
@@ -109,6 +142,7 @@ class WebServer:
 
             response.content_type = 'application/json'
             response.set_header('Cache-Control', 'no-store')
+            response.status = 201
             return json.dumps("OK", default=serialize_everything)
         except Exception as e:
             logging.error(e)
@@ -137,6 +171,9 @@ class WebServer:
         # in seconds UTC.
         # We can also filter by source, sig, band, mode, dx_continent and de_continent. Each of these accepts a single
         # value or a comma-separated list.
+        # We can filter by comments, accepting a single string, where the API will only return spots where the comment
+        # contains the provided value (case-insensitive).
+        # We can "de-dupe" spots, so only the latest spot will be sent for each callsign.
         # We can provide a "limit" number as well. Spots are always returned newest-first; "limit" limits to only the
         # most recent X spots.
         spot_ids = list(self.spots.iterkeys())
@@ -162,8 +199,19 @@ class WebServer:
                     sources = query.get(k).split(",")
                     spots = [s for s in spots if s.source and s.source in sources]
                 case "sig":
+                    # If a list of sigs is provided, the spot must have a sig and it must match one of them
                     sigs = query.get(k).split(",")
                     spots = [s for s in spots if s.sig and s.sig in sigs]
+                case "needs_sig":
+                    # If true, a sig is required, regardless of what it is, it just can't be missing.
+                    needs_sig = query.get(k).upper() == "TRUE"
+                    if needs_sig:
+                        spots = [s for s in spots if s.sig]
+                case "needs_sig_ref":
+                    # If true, at least one sig ref is required, regardless of what it is, it just can't be missing.
+                    needs_sig_ref = query.get(k).upper() == "TRUE"
+                    if needs_sig_ref:
+                        spots = [s for s in spots if s.sig_refs and len(s.sig_refs) > 0]
                 case "band":
                     bands = query.get(k).split(",")
                     spots = [s for s in spots if s.band and s.band in bands]
@@ -179,6 +227,32 @@ class WebServer:
                 case "de_continent":
                     deconts = query.get(k).split(",")
                     spots = [s for s in spots if s.de_continent and s.de_continent in deconts]
+                case "comment_includes":
+                    comment_includes = query.get(k).strip()
+                    spots = [s for s in spots if s.comment and comment_includes.upper() in s.comment.upper()]
+                case "allow_qrt":
+                    # If false, spots that are flagged as QRT are not returned.
+                    prevent_qrt = query.get(k).upper() == "FALSE"
+                    if prevent_qrt:
+                        spots = [s for s in spots if not s.qrt or s.qrt == False]
+                case "needs_good_location":
+                    # If true, spots require a "good" location to be returned
+                    needs_good_location = query.get(k).upper() == "TRUE"
+                    if needs_good_location:
+                        spots = [s for s in spots if s.dx_location_good]
+                case "dedupe":
+                    # Ensure only the latest spot of each callsign is present in the list. This relies on the list being
+                    # in reverse time order, so if any future change allows re-ordering the list, that should be done
+                    # *after* this.
+                    dedupe = query.get(k).upper() == "TRUE"
+                    if dedupe:
+                        spots_temp = []
+                        already_seen = []
+                        for s in spots:
+                            if s.dx_call not in already_seen:
+                                spots_temp.append(s)
+                                already_seen.append(s.dx_call)
+                        spots = spots_temp
         # If we have a "limit" parameter, we apply that last, regardless of where it appeared in the list of keys.
         if "limit" in query.keys():
             spots = spots[:int(query.get("limit"))]

spotproviders/dxcluster.py

@@ -7,6 +7,8 @@ from time import sleep
 import pytz
 import telnetlib3
 
+from core.constants import SIGS
+from core.sig_utils import ANY_SIG_REGEX, ANY_XOTA_SIG_REF_REGEX, get_icon_for_sig, get_ref_regex_for_sig
 from data.spot import Spot
 from core.config import SERVER_OWNER_CALLSIGN
 from spotproviders.spot_provider import SpotProvider
@@ -75,6 +77,21 @@ class DXCluster(SpotProvider):
                                     icon="desktop",
                                     time=spot_datetime.timestamp())
 
+                        # See if the comment looks like it contains a SIG (and optionally SIG reference). Currently,
+                        # only one sig ref is supported. Note that this code is specifically in the DX Cluster class and
+                        # not in the general "spot" infer_missing() method. Because we only support one SIG per spot
+                        # at the moment (see issue #54), we don't want to risk e.g. a POTA spot with comment "WWFF GFF-0001"
+                        # being converted into a WWFF spot.
+                        sig_match = re.search(r"(^|\W)" + ANY_SIG_REGEX + r"($|\W)", spot.comment, re.IGNORECASE)
+                        if sig_match:
+                            spot.sig = sig_match.group(2).upper()
+                            spot.icon = get_icon_for_sig(spot.sig)
+                            ref_regex = get_ref_regex_for_sig(spot.sig)
+                            if ref_regex:
+                                sig_ref_match = re.search(r"(^|\W)" + spot.sig + r"($|\W)(" + ref_regex + r")($|\W)", spot.comment, re.IGNORECASE)
+                                if sig_ref_match:
+                                    spot.sig_refs = [sig_ref_match.group(3).upper()]
+
                         # Add to our list
                         self.submit(spot)
 

spotproviders/gma.py

@@ -5,6 +5,7 @@ import pytz
 from requests_cache import CachedSession
 
 from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -36,6 +37,7 @@ class GMA(HTTPSpotProvider):
                         comment=source_spot["TEXT"],
                         sig_refs=[source_spot["REF"]],
                         sig_refs_names=[source_spot["NAME"]],
+                        sig_refs_urls=["https://www.cqgma.org/zinfo.php?ref=" + source_spot["REF"]],
                         time=datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
                             tzinfo=pytz.UTC).timestamp(),
                         dx_latitude=float(source_spot["LAT"]) if (source_spot["LAT"] and source_spot["LAT"] != "") else None,
@@ -56,27 +58,21 @@ class GMA(HTTPSpotProvider):
                     match ref_info["reftype"]:
                         case "Summit":
                             spot.sig = "GMA"
-                            spot.icon = "mountain"
                         case "IOTA Island":
                             spot.sig = "IOTA"
-                            spot.icon = "umbrella-beach"
                         case "Lighthouse (ILLW)":
                             spot.sig = "ILLW"
-                            spot.icon = "tower-observation"
                         case "Lighthouse (ARLHS)":
                             spot.sig = "ARLHS"
-                            spot.icon = "tower-observation"
                         case "Castle":
-                            spot.sig = "WCA/COTA"
+                            spot.sig = "WCA"
-                            spot.icon = "chess-rook"
                         case "Mill":
                             spot.sig = "MOTA"
-                            spot.icon = "fan"
                         case _:
                             logging.warn("GMA spot found with ref type " + ref_info[
-                                "reftype"] + ", developer needs to figure out an icon for this!")
+                                "reftype"] + ", developer needs to add support for this!")
                             spot.sig = ref_info["reftype"]
-                            spot.icon = "person-hiking"
+                    spot.icon = get_icon_for_sig(spot.sig)
 
                     # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
                     # that for us.

spotproviders/hema.py

@@ -5,6 +5,7 @@ import pytz
 import requests
 
 from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -54,7 +55,7 @@ class HEMA(HTTPSpotProvider):
                                 sig="HEMA",
                                 sig_refs=[spot_items[3].upper()],
                                 sig_refs_names=[spot_items[4]],
-                                icon="mound",
+                                icon=get_icon_for_sig("HEMA"),
                                 time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(tzinfo=pytz.UTC).timestamp(),
                                 dx_latitude=float(spot_items[7]),
                                 dx_longitude=float(spot_items[8]))

spotproviders/parksnpeaks.py

@@ -1,11 +1,13 @@
 import csv
 import logging
+import re
 from datetime import datetime, timedelta
 
 import pytz
 from requests_cache import CachedSession
 
 from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -32,7 +34,7 @@ class ParksNPeaks(HTTPSpotProvider):
             spot = Spot(source=self.name,
                         source_id=source_spot["actID"],
                         dx_call=source_spot["actCallsign"].upper(),
-                        de_call=source_spot["actSpoter"].upper(),  # typo exists in API
+                        de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,  # typo exists in API
                         freq=float(source_spot["actFreq"].replace(",", "")) * 1000000 if (
                                     source_spot["actFreq"] != "") else None,
                         # Seen PNP spots with empty frequency, and with comma-separated thousands digits
@@ -40,22 +42,22 @@ class ParksNPeaks(HTTPSpotProvider):
                         comment=source_spot["actComments"],
                         sig=source_spot["actClass"],
                         sig_refs=[source_spot["actSiteID"]],
+                        icon=get_icon_for_sig(source_spot["actClass"]),
                         time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S").replace(
                             tzinfo=pytz.UTC).timestamp())
 
-            # PNP supports a bunch of programs which should have different icons
+            # Free text location is not present in all spots, so only add it if it's set
-            if spot.sig == "SiOTA":
+            if "actLocation" in source_spot and source_spot["actLocation"] != "":
-                spot.icon = "wheat-awn"
+                spot.sig_refs_names = [source_spot["actLocation"]]
-            elif spot.sig == "ZLOTA":
+
-                spot.icon = "kiwi-bird"
+            # Extract a de_call if it's in the comment but not in the "actSpoter" field
-            elif spot.sig in ["POTA", "SOTA", "WWFF"]:
+            m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment)
-                # Don't care about an icon as this will be rejected anyway, we have better data from POTA/SOTA/WWFF direct
+            if (not spot.de_call or spot.de_call == "ZLOTA") and m:
-                spot.icon = ""
+                spot.de_call = m.group(1)
-            else:
+
-                # Unknown programme we've never seen before
+            # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-                logging.warn(
+            if spot.sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA"]:
-                    "PNP spot found with sig " + spot.sig + ", developer needs to add support for icon and grid/lat/lon lookup!")
+                logging.warn("PNP spot found with sig " + spot.sig + ", developer needs to add support for this!")
-                spot.icon = "question"
 
             # SiOTA lat/lon/grid lookup
             if spot.sig == "SiOTA":
@@ -76,10 +78,10 @@ class ParksNPeaks(HTTPSpotProvider):
                         spot.sig_refs_names = [asset["name"]]
                         spot.dx_latitude = asset["y"]
                         spot.dx_longitude = asset["x"]
-                        # Junk the "DE call", PNP always returns "ZLOTA" as the spotter for ZLOTA spots
-                        spot.de_call = None
                         break
 
+            # Note there is currently no support for KRMNPA location lookup, see issue #61.
+
             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
             # the spot list.
             if spot.sig not in ["POTA", "SOTA", "WWFF"]:

spotproviders/pota.py

@@ -1,7 +1,11 @@
-from datetime import datetime
+import re
+from datetime import datetime, timedelta
 
 import pytz
+from requests_cache import CachedSession
 
+from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig, get_ref_regex_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -10,6 +14,11 @@ from spotproviders.http_spot_provider import HTTPSpotProvider
 class POTA(HTTPSpotProvider):
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://api.pota.app/spot/activator"
+    # Might need to look up extra park data
+    PARK_URL_ROOT = "https://api.pota.app/park/"
+    PARK_DATA_CACHE_TIME_DAYS = 30
+    PARK_DATA_CACHE = CachedSession("cache/pota_park_data_cache",
+                                    expire_after=timedelta(days=PARK_DATA_CACHE_TIME_DAYS))
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -29,12 +38,27 @@ class POTA(HTTPSpotProvider):
                         sig="POTA",
                         sig_refs=[source_spot["reference"]],
                         sig_refs_names=[source_spot["name"]],
-                        icon="tree",
+                        sig_refs_urls=["https://pota.app/#/park/" + source_spot["reference"]],
-                        time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
+                        icon=get_icon_for_sig("POTA"),
+                        time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(
+                            tzinfo=pytz.UTC).timestamp(),
                         dx_grid=source_spot["grid6"],
                         dx_latitude=source_spot["latitude"],
                         dx_longitude=source_spot["longitude"])
 
+            # Sometimes we can get other refs in the comments for n-fer activations, extract them
+            all_comment_refs = re.findall(get_ref_regex_for_sig("POTA"), spot.comment)
+            for r in all_comment_refs:
+                if r not in spot.sig_refs:
+                    spot.sig_refs.append(r.upper())
+                    spot.sig_refs_urls.append("https://pota.app/#/park/" + r.upper())
+
+                    # Now we need to look up the name of that reference from the API, because the comment won't have it
+                    park_response = self.PARK_DATA_CACHE.get(self.PARK_URL_ROOT + r.upper(), headers=HTTP_HEADERS)
+                    park_data = park_response.json()
+                    if park_data and "name" in park_data:
+                        spot.sig_refs_names.append(park_data["name"])
+
             # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
             # that for us.
             new_spots.append(spot)

spotproviders/sota.py

@@ -5,6 +5,7 @@ import requests
 from requests_cache import CachedSession
 
 from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -50,7 +51,8 @@ class SOTA(HTTPSpotProvider):
                             sig="SOTA",
                             sig_refs=[source_spot["summitCode"]],
                             sig_refs_names=[source_spot["summitName"]],
-                            icon="mountain-sun",
+                            sig_refs_urls=["https://www.sotadata.org.uk/en/summit/" + source_spot["summitCode"]],
+                            icon=get_icon_for_sig("SOTA"),
                             time=datetime.fromisoformat(source_spot["timeStamp"]).timestamp(),
                             activation_score=source_spot["points"])
 

spotproviders/sse_spot_provider.py

@@ -34,11 +34,19 @@ class SSESpotProvider(SpotProvider):
         if self.thread:
             self.thread.join()
 
+    def _on_open(self):
+        self.status = "Waiting for Data"
+
+    def _on_error(self):
+        self.status = "Connecting"
+
     def run(self):
         while not self.stopped:
             try:
                 logging.debug("Connecting to " + self.name + " spot API...")
-                with EventSource(self.url, headers=HTTP_HEADERS, latest_event_id=self.last_event_id, timeout=30) as event_source:
+                self.status = "Connecting"
+                with EventSource(self.url, headers=HTTP_HEADERS, latest_event_id=self.last_event_id, timeout=30,
+                                 on_open=self._on_open, on_error=self._on_error) as event_source:
                     self.event_source = event_source
                     for event in self.event_source:
                         if event.type == 'message':
@@ -58,6 +66,8 @@ class SSESpotProvider(SpotProvider):
             except Exception as e:
                 self.status = "Error"
                 logging.exception("Exception in SSE Spot Provider (" + self.name + ")")
+            else:
+                self.status = "Disconnected"
             sleep(5)  # Wait before trying to reconnect
 
     # Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass

spotproviders/wwbota.py

@@ -1,6 +1,7 @@
 import json
 from datetime import datetime
 
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.sse_spot_provider import SSESpotProvider
 
@@ -18,9 +19,16 @@ class WWBOTA(SSESpotProvider):
         # n-fer activations.
         refs = []
         ref_names = []
+        ref_urls = []
         for ref in source_spot["references"]:
             refs.append(ref["reference"])
             ref_names.append(ref["name"])
+            # Bunkerbase URLs only work for UK bunkers, so only add a URL if we have a B/G prefix. In theory this could
+            # lead to array alignment mismatches if there was e.g. a B/F bunker followed by a B/G one, we'd end up with
+            # the B/G URL in index 0. But in practice there are no overlaps between B/G bunkers and any others, so an
+            # activation will either be entirely B/G or not B/G at all.
+            if ref["reference"].startswith("B/G"):
+                ref_urls.append("https://bunkerwiki.org/?s=" + ref["reference"])
 
         spot = Spot(source=self.name,
                     dx_call=source_spot["call"].upper(),
@@ -31,7 +39,7 @@ class WWBOTA(SSESpotProvider):
                     sig="WWBOTA",
                     sig_refs=refs,
                     sig_refs_names=ref_names,
-                    icon="radiation",
+                    icon=get_icon_for_sig("WWBOTA"),
                     time=datetime.fromisoformat(source_spot["time"]).timestamp(),
                     # WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
                     # now, we will just pick the first one to use as our grid, latitude and longitude.

spotproviders/wwff.py

@@ -2,6 +2,7 @@ from datetime import datetime
 
 import pytz
 
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -29,7 +30,8 @@ class WWFF(HTTPSpotProvider):
                         sig="WWFF",
                         sig_refs=[source_spot["reference"]],
                         sig_refs_names=[source_spot["reference_name"]],
-                        icon="seedling",
+                        sig_refs_urls=["https://wwff.co/directory/?showRef=" + source_spot["reference"]],
+                        icon=get_icon_for_sig("WWFF"),
                         time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
                         dx_latitude=source_spot["latitude"],
                         dx_longitude=source_spot["longitude"])

views/webpage_about.tpl

@@ -24,6 +24,8 @@
     <h4 class="mt-4">Why does this website ask me if I want to install it?</h4>
     <p>Spothole is a Progressive Web App, which means you can install it on an Android or iOS device by opening the site in Chrome or Safari respectively, and clicking "Install" on the pop-up panel. It'll only prompt you once, so if you dismiss the prompt and change your mind, you'll find an Install / Add to Home Screen option on your browser's menu.</p>
     <p>Installing Spothole on your phone is completely optional, the website works exactly the same way as the "app" does.</p>
+    <h4 class="mt-4">Why hasn't my spot/alert shown up yet?</h4>
+    <p>To avoid putting too much load on the various servers that Spothole connects to, the Spothole server only polls them once every two minutes for spots, and once every hour for alerts. (Some sources, such as DX clusters, RBN, APRS-IS and WWBOTA use a non-polling mechanism, and their updates will therefore arrive more quickly.) Then if you are using the web interface, that has its own rate at which it reloads the data from Spothole, which is once a minute for spots or 30 minutes for alerts. So you could be waiting around three minutes to see a newly added spot, or 90 minutes to see a newly added alert.</p>
     <h4 class="mt-4">What licence does Spothole use?</h4>
     <p>Spothole's source code is licenced under the Public Domain. You can write a Spothole client, run your own server, modify it however you like, you can claim you wrote it and charge people £1000 for a copy, I don't really mind. (Please don't do the last one. But if you're using my code for something cool, it would be nice to hear from you!)</p>
     <h2 id="privacy" class="mt-4">Privacy</h2>

views/webpage_bands.tpl

@@ -0,0 +1,117 @@
+% rebase('webpage_base.tpl')
+
+<div class="mt-3">
+    <div class="row">
+        <div class="col-auto me-auto pt-3">
+            <p id="timing-container">Loading...</p>
+        </div>
+        <div class="col-auto">
+            <p class="d-inline-flex gap-1">
+                <button id="filters-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleFiltersPanel();"><i class="fa-solid fa-filter"></i> Filters</button>
+                <button id="display-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleDisplayPanel();"><i class="fa-solid fa-desktop"></i> Display</button>
+            </p>
+        </div>
+    </div>
+
+    <div id="filters-area" class="appearing-panel card mb-3">
+        <div class="card-header text-white bg-primary">
+            <div class="row">
+                <div class="col-auto me-auto">
+                    Filters
+                </div>
+                <div class="col-auto d-inline-flex">
+                    <button id="close-filters-button" type="button" class="btn-close btn-close-white" aria-label="Close" onclick="closeFiltersPanel();"></button>
+                </div>
+            </div>
+
+        </div>
+        <div class="card-body">
+            <div class="row row-cols-1 g-4 mb-4">
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Bands</h5>
+                            <p id="band-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+            <div class="row row-cols-1 row-cols-md-4 g-4">
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">DX Continent</h5>
+                            <p id="dx-continent-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">DE Continent</h5>
+                            <p id="de-continent-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Modes</h5>
+                            <p id="mode-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Sources</h5>
+                            <p id="source-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div id="display-area" class="appearing-panel card mb-3">
+        <div class="card-header text-white bg-primary">
+            <div class="row">
+                <div class="col-auto me-auto">
+                    Display
+                </div>
+                <div class="col-auto d-inline-flex">
+                    <button id="close-display-button" type="button" class="btn-close btn-close-white" aria-label="Close" onclick="closeDisplayPanel();"></button>
+                </div>
+            </div>
+
+        </div>
+        <div class="card-body">
+            <div id="display-container" class="row row-cols-1 row-cols-md-4 g-4">
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Spot Age</h5>
+                            <p class="card-text spothole-card-text">Last
+                                <select id="max-spot-age" class="storeable-select form-select ms-2 me-2 d-inline-block" oninput="filtersUpdated();" style="width: 5em; display: inline-block;">
+                                    <option value="300">5</option>
+                                    <option value="600">10</option>
+                                    <option value="1800" selected>30</option>
+                                    <option value="3600">60</option>
+                                </select>
+                                minutes
+                            </p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div id="bands-container"></div>
+
+</div>
+
+<script src="/js/common.js"></script>
+<script src="/js/spotandmap.js"></script>
+<script src="/js/bands.js"></script>
+<script>$(document).ready(function() { $("#nav-link-bands").addClass("active"); }); <!-- highlight active page in nav --></script>

views/webpage_base.tpl

@@ -57,12 +57,13 @@
             </button>
             <div class="collapse navbar-collapse" id="navbarTogglerDemo02">
                 <ul class="navbar-nav me-auto mb-2 mb-lg-0">
-                    <li class="nav-item ms-4"><a href="/" class="nav-link" id="nav-link-spots">Spots</a></li>
+                    <li class="nav-item ms-4"><a href="/" class="nav-link" id="nav-link-spots"><i class="fa-solid fa-tower-cell"></i> Spots</a></li>
-                    <li class="nav-item ms-4"><a href="/map" class="nav-link" id="nav-link-map">Map</a></li>
+                    <li class="nav-item ms-4"><a href="/map" class="nav-link" id="nav-link-map"><i class="fa-solid fa-map"></i> Map</a></li>
-                    <li class="nav-item ms-4"><a href="/alerts" class="nav-link" id="nav-link-alerts">Alerts</a></li>
+                    <li class="nav-item ms-4"><a href="/bands" class="nav-link" id="nav-link-bands"><i class="fa-solid fa-ruler-vertical"></i> Bands</a></li>
-                    <li class="nav-item ms-4"><a href="/status" class="nav-link" id="nav-link-status">Status</a></li>
+                    <li class="nav-item ms-4"><a href="/alerts" class="nav-link" id="nav-link-alerts"><i class="fa-solid fa-bell"></i> Alerts</a></li>
-                    <li class="nav-item ms-4"><a href="/about" class="nav-link" id="nav-link-about">About</a></li>
+                    <li class="nav-item ms-4"><a href="/status" class="nav-link" id="nav-link-status"><i class="fa-solid fa-chart-simple"></i> Status</a></li>
-                    <li class="nav-item ms-4"><a href="/apidocs" class="nav-link" id="nav-link-api">API</a></li>
+                    <li class="nav-item ms-4"><a href="/about" class="nav-link" id="nav-link-about"><i class="fa-solid fa-circle-info"></i> About</a></li>
+                    <li class="nav-item ms-4"><a href="/apidocs" class="nav-link" id="nav-link-api"><i class="fa-solid fa-gear"></i> API</a></li>
                 </ul>
             </div>
         </div>

views/webpage_spots.tpl

@@ -154,8 +154,8 @@
                                   <label class="form-check-label" for="tableShowBearing">Bearing</label>
                                 </div>
                                 <div class="form-check form-check-inline">
-                                  <input class="form-check-input storeable-checkbox" type="checkbox" id="tableShowSource" value="tableShowSource" oninput="columnsUpdated();" checked>
+                                  <input class="form-check-input storeable-checkbox" type="checkbox" id="tableShowType" value="tableShowType" oninput="columnsUpdated();" checked>
-                                  <label class="form-check-label" for="tableShowSource">Source</label>
+                                  <label class="form-check-label" for="tableShowType">Type</label>
                                 </div>
                                 <div class="form-check form-check-inline">
                                   <input class="form-check-input storeable-checkbox" type="checkbox" id="tableShowRef" value="tableShowRef" oninput="columnsUpdated();" checked>

webassets/apidocs/openapi.yml

@@ -65,7 +65,7 @@ paths:
               - APRS-IS
         - name: sig
           in: query
-          description: "Limit the spots to only ones from one or more Special Interest Groups. To select more than one SIG, supply a comma-separated list."
+          description: "Limit the spots to only ones from one or more Special Interest Groups provided as an argument. To select more than one SIG, supply a comma-separated list."
           required: false
           schema:
             type: string
@@ -76,6 +76,27 @@ paths:
               - WWBOTA
               - GMA
               - HEMA
+              - WCA
+              - MOTA
+              - SiOTA
+              - ARLHS
+              - ILLW
+              - ZLOTA
+              - IOTA
+        - name: needs_sig
+          in: query
+          description: "Limit the spots to only ones with a Special Interest Group such as POTA. Because supplying all known SIGs as a `sigs` parameter is unwieldy, and leaving `sigs` blank will also return spots with *no* SIG, this parameter can be set true to return only spots with a SIG, regardless of what it is, so long as it's not blank. This is what Field Spotter uses to exclude generic cluster spots and only retrieve xOTA things."
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: needs_sig_ref
+          in: query
+          description: "Limit the spots to only ones which have at least one reference (e.g. a park reference) for Special Interest Groups such as POTA."
+          required: false
+          schema:
+            type: boolean
+            default: false
         - name: band
           in: query
           description: "Limit the spots to only ones from one or more bands. To select more than one band, supply a comma-separated list."
@@ -168,6 +189,33 @@ paths:
               - AF
               - OC
               - AN
+        - name: dedupe
+          in: query
+          description: "\"De-duplicate\" the spots, returning only the latest spot for any given callsign."
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: comment_includes
+          in: query
+          description: "Return only spots where the comment includes the provided string (case-insensitive)."
+          required: false
+          schema:
+            type: string
+        - name: needs_good_location
+          in: query
+          description: "Return only spots with a 'good' location. (See the spot `dx_location_good` parameter for details. Useful for map-based clients, to avoid spots with 'bad' locations e.g. loads of cluster spots ending up in the centre of the DXCC entitity.)"
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: allow_qrt
+          in: query
+          description: Allow spots that are known to be QRT to be returned.
+          required: false
+          schema:
+            type: boolean
+            default: true
       responses:
         '200':
           description: Success
@@ -241,6 +289,13 @@ paths:
               - WWBOTA
               - GMA
               - HEMA
+              - WCA
+              - MOTA
+              - SiOTA
+              - ARLHS
+              - ILLW
+              - ZLOTA
+              - IOTA
         - name: dx_continent
           in: query
           description: "Limit the alerts to only ones where the DX (the operator being spotted) is on the given continent(s). To select more than one continent, supply a comma-separated list."
@@ -379,8 +434,7 @@ paths:
                     type: array
                     description: An array of all the supported Special Interest Groups.
                     items:
-                      type: string
+                      $ref: '#/components/schemas/SIG'
-                      example: "POTA"
                   sources:
                     type: array
                     description: An array of all the supported data sources.
@@ -518,13 +572,14 @@ components:
           description: Where we got the DX location (grid/latitude/longitude) from. If this was from the spot itself, it's likely quite accurate, but if we had to fall back to QRZ lookup, or even a location based on the DXCC itself, it will be a lot less accurate.
           enum:
             - SPOT
+            - "WAB/WAI GRID"
             - QRZ
             - DXCC
             - NONE
           example: SPOT
         dx_location_good:
           type: boolean
-          description: Does the software think the location is good enough to put a marker on a map? This is true if the source is "SPOT", or alternatively if the source is "QRZ" and the callsign doesn't have a slash in it (i.e. operator likely at home).
+          description: Does the software think the location is good enough to put a marker on a map? This is true if the source is "SPOT" or "WAB/WAI GRID", or alternatively if the source is "QRZ" and the callsign doesn't have a slash in it (i.e. operator likely at home).
           example: true
         de_call:
           type: string
@@ -667,6 +722,13 @@ components:
             - WWBOTA
             - GMA
             - HEMA
+            - WCA
+            - MOTA
+            - SiOTA
+            - ARLHS
+            - ILLW
+            - ZLOTA
+            - IOTA
           example: POTA
         sig_refs:
           type: array
@@ -680,6 +742,12 @@ components:
             type: string
           description: SIG reference names
           example: Null Country Park
+        sig_refs_urls:
+          type: array
+          items:
+            type: string
+          description: SIG reference URLs, which the user can look up for more information
+          example: "https://pota.app/#/park/GB-0001"
         activation_score:
           type: integer
           description: Activation score. SOTA only
@@ -688,6 +756,14 @@ components:
           type: string
           descripton: Icon, from the Font Awesome set. This is fairly opinionated but is here to help the Spothole web UI and Field Spotter. Does not include the "fa-" prefix.
           example: tree
+        band_color:
+          type: string
+          descripton: Colour to represent this spot, if a client chooses to colour spots based on their frequency band, using PSK Reporter's default colours. HTML colour e.g. hex.
+          example: #ff0000"
+        band_contrast_color:
+          type: string
+          descripton: Black or white, whichever best contrasts with "band_color".
+          example: "white"
         qrt:
           type: boolean
           description: QRT state. Some APIs return spots marked as QRT. Otherwise we can check the comments.
@@ -806,6 +882,13 @@ components:
             - WWBOTA
             - GMA
             - HEMA
+            - WCA
+            - MOTA
+            - SiOTA
+            - ARLHS
+            - ILLW
+            - ZLOTA
+            - IOTA
           example: POTA
         sig_refs:
           type: array
@@ -827,14 +910,6 @@ components:
           type: string
           descripton: Icon, from the Font Awesome set. This is fairly opinionated but is here to help the Spothole web UI and Field Spotter. Does not include the "fa-" prefix.
           example: tree
-        band_color:
-          type: string
-          descripton: Colour to represent this spot, if a client chooses to colour spots based on their frequency band, using PSK Reporter's default colours. HTML colour e.g. hex.
-          example: #ff0000"
-        band_contrast_color:
-          type: string
-          descripton: Black or white, whichever best contrasts with "band_color".
-          example: "white"
         source:
           type: string
           description: Where we got the alert from.
@@ -922,3 +997,23 @@ components:
           type: string
           description: Black or white, whichever provides the best contrast against the band colour.
           example: white
+
+    SIG:
+      type: object
+      properties:
+        name:
+          type: string
+          description: The abbreviated name of the SIG
+          example: POTA
+        description:
+          type: string
+          description: The full name of the SIG
+          example: Parks on the Air
+        icon:
+          type: string
+          description: Icon, from the Font Awesome set. This is fairly opinionated but is here to help the Spothole web UI and Field Spotter. Does not include the "fa-" prefix.
+          example: tree
+        ref_regex:
+          type: string
+          description: Regex that matches this SIG's reference IDs. Generally for Spothole's own internal use, clients probably won't need this.
+          example: "[A-Z]{2}\-\d+"

webassets/css/style.css

@@ -92,10 +92,13 @@ span.icon-wrapper {
 }
 
 span.freq-mhz {
+    font-weight: bold;
+}
+
+span.freq-mhz-pad {
     display: inline-block;
     min-width: 1.7em;
     text-align: right;
-    font-weight: bold;
 }
 
 span.freq-khz {
@@ -117,6 +120,10 @@ a.dx-link {
     text-decoration: none;
     font-weight: bold;
 }
+a.sig-ref-link {
+    color: var(--bs-emphasis-color);
+    text-decoration: none;
+}
 
 /* QRT/faded styles */
 tr.table-faded td {
@@ -142,10 +149,99 @@ div#map {
     font-family: var(--bs-body-font-family) !important;
 }
 
-a.leaflet-popup-callsign-link {
+
-    color: black;
+/* BANDS PANEL */
+
+div#bands-container {
+    min-height: 64em;
+    margin: 0;
+    padding: 0;
+    overflow-x: auto;
+    white-space: nowrap;
+    overscroll-behavior-x: none;
+}
+
+#bands-table {
+    min-width: 100%;
+}
+
+#bands-table th {
+    width: 20%;
+    max-height: 40px;
+    min-width: 12em;
+    padding: 0.5em;
+    text-align: center;
     font-weight: bold;
-    text-decoration: none;
+}
+
+#bands-table td {
+    width: 20%;
+    min-width: 12em;
+    height: 62em;
+}
+
+div.band-container {
+    height: 62em;
+    width: 20%;
+    min-width: 12em;
+    position: relative;
+}
+
+div.band-markers {
+    width: 100%;
+    height: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    z-index: 13;
+    border-left: 2px dotted black;
+}
+
+div.band-spots {
+    width: 100%;
+    height: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    z-index: 15;
+}
+
+canvas.band-lines-canvas {
+    width: 5em;
+    height: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    z-index: 11;
+}
+
+div.band-spot {
+    position: absolute;
+    left: 5em;
+    padding: 0 0.25em;
+    background-color: white;
+    border-radius: 3px;
+    cursor: default;
+}
+
+div.band-spot:hover {
+    z-index: 999;
+}
+
+div.band-spot span.band-spot-call {
+    display: inline;
+}
+
+div.band-spot:hover span.band-spot-call {
+    display: none;
+}
+
+div.band-spot span.band-spot-info {
+    display: none;
+}
+
+div.band-spot:hover span.band-spot-info {
+    display: inline;
 }
 
 
@@ -155,6 +251,10 @@ a.leaflet-popup-callsign-link {
     .hideonmobile {
         display: none !important;
     }
+    div#map, div#table-container, div#bands-container {
+        margin-left: -1em;
+        margin-right: -1em;
+    }
 }
 
 @media (min-width: 992px) {

webassets/js/bands.js

@@ -0,0 +1,286 @@
+// A couple of constants that must match what's in CSS. We need to know them before the content actually renders, so we
+// can't just ask the elements themselves for their dimensions.
+BAND_COLUMN_HEIGHT_EM = 62;
+BAND_COLUMN_CANVAS_WIDTH_EM = 4;
+BAND_COLUMN_FONT_SIZE = 16;
+BAND_COLUMN_HEIGHT_PX = BAND_COLUMN_HEIGHT_EM * BAND_COLUMN_FONT_SIZE;
+BAND_COLUMN_CANVAS_WIDTH_PX = BAND_COLUMN_CANVAS_WIDTH_EM * BAND_COLUMN_FONT_SIZE;
+BAND_COLUMN_SPOT_DIV_HEIGHT_PX = BAND_COLUMN_FONT_SIZE * 1.6;
+
+// Load spots and populate the bands display.
+function loadSpots() {
+    $.getJSON('/api/v1/spots' + buildQueryString(), function(jsonData) {
+        // Store last updated time
+        lastUpdateTime = moment.utc();
+        updateRefreshDisplay();
+        // Store data
+        spots = jsonData;
+        // Update bands display
+        updateBands();
+    });
+}
+
+// Build a query string for the API, based on the filters that the user has selected.
+function buildQueryString() {
+    var str =  "?";
+    ["dx_continent", "de_continent", "mode_type", "source", "band"].forEach(fn => {
+        if (!allFilterOptionsSelected(fn)) {
+            str = str + getQueryStringFor(fn) + "&";
+        }
+    });
+    str = str + "max_age=" + $("#max-spot-age option:selected").val();
+    // Additional filters for the bands view: No dupes, no QRT
+    str = str + "&dedupe=true&allow_qrt=false";
+    return str;
+}
+
+// Update the bands display
+function updateBands() {
+    // Stop here if nothing to display
+    var bandsContainer = $("#bands-container");
+    if (spots.length === 0) {
+        bandsContainer.html("<div class='alert alert-danger' role='alert'>No spots match your filters.</div>");
+        return;
+    }
+
+    // Do some harsher de-duping. Because we only display callsign, frequency and mode here, the previous
+    // de-duplication could have let some through that don't look like dupes on the map, but would do here.
+    // Typically that's a person activating two programs at the same time, e.g. POTA & WWFF.
+    spotList = removeDuplicatesForBandPanel(spots);
+
+    // Convert to a map of band names to the spots on that band. Bands with no
+    // spots in view will not be present.
+    const bandToSpots = new Map();
+    options["bands"].forEach(function (band) {
+        const matchingSpots = spotList.filter(function (s) {
+            return s.band === band.name;
+        });
+        if (matchingSpots.length > 0) {
+            bandToSpots.set(band.name, matchingSpots);
+        }
+    });
+
+    // Track if any columns end up taller than expected, so we can resize the container and avoid vertical scroll.
+    var maxHeightBand = 0;
+
+    // Build up table content for each band
+    var table = $('<table id="bands-table">').append('<thead><tr></tr></thead><tbody><tr></tr></tbody>');
+    bandToSpots.forEach(function (spotList, bandName) {
+        // Get the colours for the band from the first spot, and prepare the header
+        table.find('thead tr').append(`<th style='background-color:${spotList[0].band_color}; color:${spotList[0].band_contrast_color}'>${spotList[0].band}</th>`);
+
+        // Get the band data to fetch start and end frequencies
+        let band = options["bands"].filter(function (b) {
+            return b.name === bandName;
+        })[0];
+
+        // Print the frequency band markers. This is 41 steps to divide the band evenly into 40 markers. One in every
+        // four will show the actual frequency, the others will just be dashes.
+        bandMarkersDiv = $('<div class="band-markers">');
+        const freqStep = (band.end_freq - band.start_freq) / 40.0;
+        for (let i = 0; i <= 40; i++) {
+            if (i % 4 === 0) {
+                bandMarkersDiv.append("&mdash;" + ((band.start_freq + i * freqStep)/1000000).toFixed(3) + "<br/>");
+            } else if (i % 4 === 2) {
+                bandMarkersDiv.append("&ndash;<br/>");
+            } else {
+                bandMarkersDiv.append("-<br/>");
+            }
+        }
+
+        // Prepare the spots list
+        var bandSpotsDiv = $("<div class='band-spots'>");
+        var lastSpotPxDownBand = -999;
+        // Sort by frequency so have a consistent order in which to plan where they will appear on the band div.
+        spotList.sort(function(a, b) { return a.freq - b.freq; });
+        // First calculate how we should be displaying the spots. There are three "modes" to try to place them in a
+        // visually appealing way:
+        // 1) Spaced normally, not going over the end of the band, so we populate them forwards.
+        // 2) Would go over the end, but the spots don't fill the band, so we populate them backwards.
+        // 3) Spots totally fill the band (or more), so we space them evenly starting at the top.
+        // In each case, we don't add anything to the DOM yet, we just calculate "pxDownBandLabel" (how far the *top* of
+        // the label is from the top of the div) and add that as a property to the spot for later use.
+        if (spotList.length >= BAND_COLUMN_HEIGHT_PX / BAND_COLUMN_SPOT_DIV_HEIGHT_PX) {
+            // Mode 3.
+            // Just lay out all spots simply, starting at 0px offset and working down with each one touching.
+            lastSpotPxDownBand = 0 - BAND_COLUMN_SPOT_DIV_HEIGHT_PX;
+            spotList.forEach(s => {
+                lastSpotPxDownBand = lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX;
+                s["pxDownBandLabel"] = lastSpotPxDownBand;
+            });
+        } else {
+            // Mode 1 or 2. Run through adding things to the list forwards as a test.
+            spotList.forEach(s => {
+                // Work out how far down the div to draw it
+                var percentDownBand = (s.freq - band.start_freq) / (band.end_freq - band.start_freq) * 0.97; // not 100% due to fudge, the first and last dashes are not exactly at the top and bottom of the div as some space is needed for text
+                var pxDownBand = percentDownBand * BAND_COLUMN_HEIGHT_PX;
+                if (pxDownBand < lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX) {
+                    pxDownBand = lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX; // Prevent overlap
+                }
+                s["pxDownBandLabel"] = pxDownBand;
+                lastSpotPxDownBand = pxDownBand;
+            });
+
+            // Work out if we overflowed the end.
+            if (lastSpotPxDownBand <= BAND_COLUMN_HEIGHT_PX) {
+                // Mode 1. Current positions are fine and there's nothing to do.
+            } else {
+                // Mode 2. Repeat the process but backwards, starting at the end and working upwards.
+                lastSpotPxDownBand = 999999;
+                spotList.reverse().forEach(s => {
+                    // Work out how far down the div to draw it
+                    var percentDownBand = (s.freq - band.start_freq) / (band.end_freq - band.start_freq) * 0.97; // not 100% due to fudge, the first and last dashes are not exactly at the top and bottom of the div as some space is needed for text
+                    var pxDownBand = percentDownBand * BAND_COLUMN_HEIGHT_PX;
+                    if (pxDownBand > lastSpotPxDownBand - BAND_COLUMN_SPOT_DIV_HEIGHT_PX) {
+                        pxDownBand = lastSpotPxDownBand - BAND_COLUMN_SPOT_DIV_HEIGHT_PX; // Prevent overlap
+                    }
+                    s["pxDownBandLabel"] = pxDownBand;
+                    lastSpotPxDownBand = pxDownBand;
+                });
+            }
+        }
+
+        // Now each spot is tagged with how far down the div it should go, add them to the DOM.
+        spotList.forEach(s => {
+            bandSpotsDiv.append(`<div class="band-spot" style="top: ${s['pxDownBandLabel']}px; border-top: 1px solid ${s.band_color}; border-left: 5px solid ${s.band_color}; border-bottom: 1px solid ${s.band_color}; border-right: 1px solid ${s.band_color};"><span class="band-spot-call">${s.dx_call}</span><span class="band-spot-info">${s.dx_call} ${(s.freq/1000000).toFixed(3)} ${s.mode}</span></div>`);
+        });
+
+        // Work out how tall the canvas should be. Normally this is matching the normal band column height, but if some
+        // spots have gone off the end of the band markers and stretched their div, we need to resize the canvas to
+        // match, otherwise we have nowhere to draw their connecting lines.
+        var canvasHeight = Math.max(BAND_COLUMN_HEIGHT_PX, lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX);
+        maxHeightBand = Math.max(maxHeightBand, canvasHeight);
+
+        // Draw horizontal or diagonal lines to join up the "real" frequency with where the spot div ended up
+        var bandLinesCanvas = $(`<canvas class='band-lines-canvas' width='${BAND_COLUMN_CANVAS_WIDTH_PX}px' height='${canvasHeight}px' style='height:${canvasHeight}px !important;'>`);
+        spotList.forEach(s => {
+            // Work out how far down the div to draw it
+            var percentDownBand = (s.freq - band.start_freq) / (band.end_freq - band.start_freq) * 0.97; // not 100% due to fudge, the first and last dashes are not exactly at the top and bottom of the div as some space is needed for text
+            var pxDownBandFreq = (percentDownBand + 0.015) * BAND_COLUMN_HEIGHT_PX; // same fudge but add half to put the left end of the line in the right place
+            var pxDownBandLabel = s["pxDownBandLabel"] + (BAND_COLUMN_SPOT_DIV_HEIGHT_PX / 1.75); // line should be to the vertical text-centre spot, not to the top corner
+
+            // Draw the line on the canvas
+            var ctx = bandLinesCanvas[0].getContext('2d');
+            ctx.beginPath();
+            ctx.lineWidth = 2;
+            ctx.lineCap = "round";
+            ctx.strokeStyle = s.band_color;
+            ctx.moveTo(0, pxDownBandFreq);
+            ctx.lineTo(BAND_COLUMN_CANVAS_WIDTH_PX, pxDownBandLabel);
+            ctx.stroke();
+        });
+
+        // Assemble the table cell
+        td = $("<td>");
+        container = $("<div class='band-container'>");
+        container.append(bandLinesCanvas);
+        container.append(bandMarkersDiv);
+        container.append(bandSpotsDiv);
+        td.append(container);
+        table.find('tbody tr').append(td);
+    });
+
+    // Update the DOM with the band HTML
+    bandsContainer.html(table);
+
+    // Increase the height of the bands container so we don't have any vertical scroll bars except the browser ones
+    bandsContainer.css("min-height", `${maxHeightBand + 42}px`);
+
+    // Desktop mouse wheel to scroll bands horizontally if used on the headers
+    table.find('thead tr').on("wheel", () => {
+        bandsContainer.scrollLeft(bandsContainer.scrollLeft() + event.deltaY / 10.0);
+        return false;
+    });
+}
+
+// Iterate through a temporary list of spots, merging duplicates in a way suitable for the band panel. If two or more
+// spots with the activator, mode and frequency are found, these will be merged and reduced until only one remains,
+// with the best data. Note that unlike removeDuplicates(), which operates on the main spot map, this operates only
+// on the temporary array of spots provided as an argument, and returns the output, for use when constructing the
+// band panel.
+function removeDuplicatesForBandPanel(spotList) {
+    const spotsToRemove = [];
+    spotList.forEach(function (check) {
+        spotList.forEach(function (s) {
+            if (s !== check) {
+                if (s.dx_call === check.dx_call && s.freq === check.freq && s.mode === check.mode) {
+                    // Find which one to keep and which to delete
+                    const checkSpotNewer = check.time > s.time;
+                    const keepSpot = checkSpotNewer ? check : s;
+                    const deleteSpot = checkSpotNewer ? s : check;
+                    // Aggregate list of spots to remove
+                    spotsToRemove.push(deleteSpot.uid);
+                }
+            }
+        });
+    });
+    // Perform the removal
+    return spotList.filter(s => !spotsToRemove.includes(s.uid));
+}
+
+// Load server options. Once a successful callback is made from this, we then query spots and set up the timer to query
+// spots repeatedly.
+function loadOptions() {
+    $.getJSON('/api/v1/options', function(jsonData) {
+        // Store options
+        options = jsonData;
+
+        // Add CSS for band toggle buttons
+        addBandToggleColourCSS(options["bands"]);
+
+        // Populate the filters panel
+        generateBandsMultiToggleFilterCard(options["bands"]);
+        generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
+        generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
+        generateMultiToggleFilterCard("#mode-options", "mode_type", options["mode_types"]);
+        generateMultiToggleFilterCard("#source-options", "source", options["spot_sources"]);
+
+        // Load settings from settings storage now all the controls are available
+        loadSettings();
+
+        // Load spots and set up the timer
+        loadSpots();
+        setInterval(loadSpots, REFRESH_INTERVAL_SEC * 1000);
+    });
+}
+
+// Method called when any display property is changed to reload the map and persist the display settings.
+function displayUpdated() {
+    updateMap();
+    saveSettings();
+}
+
+// React to toggling/closing panels
+function toggleFiltersPanel() {
+    // If we are going to show the filters panel, hide the display panel
+    if (!$("#filters-area").is(":visible") && $("#display-area").is(":visible")) {
+        $("#display-area").hide();
+        $("#display-button").button("toggle");
+    }
+    $("#filters-area").toggle();
+}
+function closeFiltersPanel() {
+    $("#filters-button").button("toggle");
+    $("#filters-area").hide();
+}
+
+function toggleDisplayPanel() {
+    // If we are going to show the display panel, hide the filters panel
+    if (!$("#display-area").is(":visible") && $("#filters-area").is(":visible")) {
+        $("#filters-area").hide();
+        $("#filters-button").button("toggle");
+    }
+    $("#display-area").toggle();
+}
+function closeDisplayPanel() {
+    $("#display-button").button("toggle");
+    $("#display-area").hide();
+}
+
+// Startup
+$(document).ready(function() {
+    // Call loadOptions(), this will then trigger loading spots and setting up timers.
+    loadOptions();
+    // Update the refresh timing display every second
+    setInterval(updateRefreshDisplay, 1000);
+});

webassets/js/map.js

@@ -3,7 +3,7 @@ var markersLayer;
 var geodesicsLayer;
 var terminator;
 
-// Load spots and populate the table.
+// Load spots and populate the map.
 function loadSpots() {
     $.getJSON('/api/v1/spots' + buildQueryString(), function(jsonData) {
         // Store data
@@ -23,6 +23,8 @@ function buildQueryString() {
         }
     });
     str = str + "max_age=" + $("#max-spot-age option:selected").val();
+    // Additional filters for the map view: No dupes, no QRT, only spots with good locations
+    str = str + "&dedupe=true&allow_qrt=false&needs_good_location=true";
     return str;
 }
 
@@ -32,13 +34,8 @@ function updateMap() {
     markersLayer.clearLayers();
     geodesicsLayer.clearLayers();
 
-    // Make new markers for all spots with a good location, not QRT, and not a duplicate spot within the data set.
+    // Make new markers for all spots that match the filter
-    var callsAlreadyDisplayed = [];
     spots.forEach(function (s) {
-        if (s["dx_location_good"] && (s["qrt"] == null || s["qrt"] == false)) {
-            if (!callsAlreadyDisplayed.includes(s["dx_call"])) {
-
-                // OK, create the marker
         var m = L.marker([s["dx_latitude"], s["dx_longitude"]], {icon: getIcon(s)});
         m.bindPopup(getTooltipText(s));
         markersLayer.addLayer(m);
@@ -52,10 +49,6 @@ function updateMap() {
             });
             geodesicsLayer.addLayer(geodesic);
         }
-
-            }
-            callsAlreadyDisplayed.push(s["dx_call"]);
-        }
     });
 }
 
@@ -100,7 +93,13 @@ function getTooltipText(s) {
 
     // Format sig_refs
     var sig_refs = "";
-    if (s["sig_refs"]) {
+    if (s["sig_refs"] && s["sig_refs_urls"] && s["sig_refs"].length == s["sig_refs_urls"].length) {
+        items = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`)
+        for (var i = 0; i < items.length; i++) {
+            items[i] = `<a href='${s["sig_refs_urls"][i]}' target='_new' class='sig-ref-link'>${items[i]}</a>`
+        }
+        sig_refs = items.join(", ");
+    } else if (s["sig_refs"]) {
         sig_refs = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`).join(", ");
     }
 
@@ -108,10 +107,13 @@ function getTooltipText(s) {
     const shortCall = s["dx_call"].split("/").sort(function (a, b) {
         return b.length - a.length;
     })[0];
-    ttt = `<span class='nowrap'>${dx_flag} <a href='https://www.qrz.com/db/${shortCall}' target='_blank' class="leaflet-popup-callsign-link">${s["dx_call"]}</a></span><br/>`;
+    ttt = `<span class='nowrap'><span class='icon-wrapper'>${dx_flag}</span> <a href='https://www.qrz.com/db/${shortCall}' target='_blank' class="dx-link">${s["dx_call"]}</a></span><br/>`;
 
     // Frequency & band
-    ttt += `<i class='fa-solid fa-walkie-talkie markerPopupIcon'></i>&nbsp;${freq_string} (${s["band"]})`;
+    ttt += `<span class='icon-wrapper'><i class='fa-solid fa-radio markerPopupIcon'></i></span>&nbsp;${freq_string}`;
+    if (s["band"] != null) {
+        ttt += ` (${s["band"]})`;
+    }
     // Mode
     if (s["mode"] != null) {
         ttt += ` &nbsp;&nbsp; <i class='fa-solid fa-wave-square markerPopupIcon'></i>&nbsp;${s["mode"]}`;
@@ -119,14 +121,14 @@ function getTooltipText(s) {
     ttt += "<br/>";
 
     // Source / SIG / Ref
-    ttt += `<span class='nowrap'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i>&nbsp;${sigSourceText} ${sig_refs}</span><br/>`;
+    ttt += `<span class='nowrap'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span>&nbsp;${sigSourceText} ${sig_refs}</span><br/>`;
 
     // Time
-    ttt += `<i class='fa-solid fa-clock markerPopupIcon'></i>&nbsp;${moment.unix(s["time"]).fromNow()}`;
+    ttt += `<span class='icon-wrapper'><i class='fa-solid fa-clock markerPopupIcon'></i></span>&nbsp;${moment.unix(s["time"]).fromNow()}`;
 
     // Comment
     if (commentText.length > 0) {
-        ttt += `<br/><i class='fa-solid fa-comment markerPopupIcon'></i> ${commentText}`;
+        ttt += `<br/><span class='icon-wrapper'><i class='fa-solid fa-comment markerPopupIcon'></i></span> ${commentText}`;
     }
 
     return ttt;

webassets/js/spots.js

@@ -38,7 +38,7 @@ function updateTable() {
     var showMode = $("#tableShowMode")[0].checked;
     var showComment = $("#tableShowComment")[0].checked;
     var showBearing = $("#tableShowBearing")[0].checked && userPos != null;
-    var showSource = $("#tableShowSource")[0].checked;
+    var showType = $("#tableShowType")[0].checked;
     var showRef = $("#tableShowRef")[0].checked;
     var showDE = $("#tableShowDE")[0].checked;
 
@@ -62,8 +62,8 @@ function updateTable() {
     if (showBearing) {
         table.find('thead tr').append(`<th class='hideonmobile'>Bearing</th>`);
     }
-    if (showSource) {
+    if (showType) {
-        table.find('thead tr').append(`<th class='hideonmobile'>Source</th>`);
+        table.find('thead tr').append(`<th class='hideonmobile'>Type</th>`);
     }
     if (showRef) {
         table.find('thead tr').append(`<th class='hideonmobile'>Ref.</th>`);
@@ -109,7 +109,7 @@ function updateTable() {
         var khz = Math.floor((s["freq"] - (mhz * 1000000.0)) / 1000.0);
         var hz = Math.floor(s["freq"] - (mhz * 1000000.0) - (khz * 1000.0));
         var hz_string = (hz > 0) ? hz.toFixed(0)[0] : "";
-        var freq_string = `<span class='freq-mhz'>${mhz.toFixed(0)}</span><span class='freq-khz'>${khz.toFixed(0).padStart(3, '0')}</span><span class='freq-hz hideonmobile'>${hz_string}</span>`
+        var freq_string = `<span class='freq-mhz freq-mhz-pad'>${mhz.toFixed(0)}</span><span class='freq-khz'>${khz.toFixed(0).padStart(3, '0')}</span><span class='freq-hz hideonmobile'>${hz_string}</span>`
 
         // Format the mode
         mode_string = s["mode"];
@@ -140,22 +140,22 @@ function updateTable() {
             }
         }
 
-        // Sig or fallback to source
+        // Format "type" (Sig or fallback to source)
-        var sigSourceText = s["source"];
+        var typeText = s["source"];
         if (s["sig"]) {
-            sigSourceText = s["sig"];
+            typeText = s["sig"];
         }
 
         // Format sig_refs
         var sig_refs = "";
-        if (s["sig_refs"]) {
+        if (s["sig_refs"] && s["sig_refs_urls"] && s["sig_refs"].length == s["sig_refs_urls"].length && s["sig_refs"].length == s["sig_refs_names"].length) {
-            sig_refs = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`).join(", ");
+            items = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`)
+            for (var i = 0; i < items.length; i++) {
+                items[i] = `<a href='${s["sig_refs_urls"][i]}' title='${s["sig_refs_names"][i]}' target='_new' class='sig-ref-link'>${items[i]}</a>`
             }
-
+            sig_refs = items.join(", ");
-        // Format sig_refs title
+        } else if (s["sig_refs"]) {
-        var sig_refs_title_string = "";
+            sig_refs = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`).join(", ");
-        if (s["sig_refs_names"]) {
-            sig_refs_title_string = " title=\"" + s["sig_refs_names"].join(", ") + "\"";
         }
 
         // Format DE flag
@@ -199,25 +199,25 @@ function updateTable() {
         if (showBearing) {
             $tr.append(`<td class='nowrap hideonmobile'>${bearingText}</td>`);
         }
-        if (showSource) {
+        if (showType) {
-            $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${sigSourceText}</td>`);
+            $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${typeText}</td>`);
         }
         if (showRef) {
-            $tr.append(`<td class='hideonmobile'><span ${sig_refs_title_string}>${sig_refs}</span></td>`);
+            $tr.append(`<td class='hideonmobile'>${sig_refs}</td>`);
         }
         if (showDE) {
             $tr.append(`<td class='nowrap hideonmobile'><span class='flag-wrapper' title='${de_country}'>${de_flag}</span>${de_call}</td>`);
         }
         table.find('tbody').append($tr);
 
-        // Second row for mobile view only, containing source, ref & comment
+        // Second row for mobile view only, containing type, ref & comment
         $tr2 = $("<tr class='hidenotonmobile'>");
         if (s["qrt"] == true) {
           $tr2.addClass("table-faded");
         }
         $td2 = $("<td colspan='100'>");
-        if (showSource) {
+        if (showType) {
-            $td2.append(`<span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${sigSourceText} `);
+            $td2.append(`<span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${typeText} `);
         }
         if (showRef) {
             $td2.append(`${sig_refs} `);