Support Packet (PKT) as a digi mode

README.md

@@ -14,6 +14,8 @@ Supported data sources include DX Clusters, the Reverse Beacon Network (RBN), th
 
 ![Screenshot](/images/screenshot2.png)
 
+![Screenshot](/images/screenshot3.png)
+
 ### Accessing the public version
 
 You can access the public version's web interface at [https://spothole.app](https://spothole.app), and see [https://spothole.app/apidocs](https://spothole.app/apidocs) for the API details.

alertproviders/parksnpeaks.py

@@ -0,0 +1,56 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from alertproviders.http_alert_provider import HTTPAlertProvider
+from core.sig_utils import get_icon_for_sig
+from data.alert import Alert
+
+
+# Alert provider for Parks n Peaks
+class ParksNPeaks(HTTPAlertProvider):
+    POLL_INTERVAL_SEC = 3600
+    ALERTS_URL = "http://parksnpeaks.org/api/ALERTS/"
+
+    def __init__(self, provider_config):
+        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json():
+            # Calculate some things
+            if " - " in source_alert["Location"]:
+                split = source_alert["Location"].split(" - ")
+                sig_ref = split[0]
+                sig_ref_name = split[1]
+            else:
+                sig_ref = source_alert["WWFFID"]
+                sig_ref_name = source_alert["Location"]
+            start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(
+                tzinfo=pytz.UTC).timestamp()
+
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert["alID"],
+                          dx_calls=[source_alert["CallSign"].upper()],
+                          freqs_modes=source_alert["Freq"] + " " + source_alert["MODE"],
+                          comment=source_alert["Comments"],
+                          sig=source_alert["Class"],
+                          sig_refs=[sig_ref],
+                          sig_refs_names=[sig_ref_name],
+                          icon=get_icon_for_sig(source_alert["Class"]),
+                          start_time=start_time,
+                          is_dxpedition=False)
+
+            # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
+            if alert.sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA"]:
+                logging.warn("PNP alert found with sig " + alert.sig + ", developer needs to add support for this!")
+
+            # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
+            # the alert list. Note that while ZLOTA has its own spots API, it doesn't have its own alerts API. So that
+            # means the PnP *spot* provider rejects ZLOTA spots here, but the PnP *alerts* provider here allows ZLOTA.
+            if alert.sig not in ["POTA", "SOTA", "WWFF"]:
+                new_alerts.append(alert)
+        return new_alerts

alertproviders/pota.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import pytz
 
 from alertproviders.http_alert_provider import HTTPAlertProvider
+from core.sig_utils import get_icon_for_sig
 from data.alert import Alert
 
 
@@ -27,7 +28,7 @@ class POTA(HTTPAlertProvider):
                           sig="POTA",
                           sig_refs=[source_alert["reference"]],
                           sig_refs_names=[source_alert["name"]],
-                          icon="tree",
+                          icon=get_icon_for_sig("POTA"),
                           start_time=datetime.strptime(source_alert["startDate"] + source_alert["startTime"],
                                                        "%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
                           end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"],

alertproviders/sota.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import pytz
 
 from alertproviders.http_alert_provider import HTTPAlertProvider
+from core.sig_utils import get_icon_for_sig
 from data.alert import Alert
 
 
@@ -28,7 +29,7 @@ class SOTA(HTTPAlertProvider):
                           sig="SOTA",
                           sig_refs=[source_alert["associationCode"] + "/" + source_alert["summitCode"]],
                           sig_refs_names=[source_alert["summitDetails"]],
-                          icon="mountain-sun",
+                          icon=get_icon_for_sig("SOTA"),
                           start_time=datetime.strptime(source_alert["dateActivated"],
                                                        "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=pytz.UTC).timestamp(),
                           is_dxpedition=False)

alertproviders/wota.py

@@ -0,0 +1,64 @@
+from datetime import datetime
+
+import pytz
+from rss_parser import RSSParser
+
+from alertproviders.http_alert_provider import HTTPAlertProvider
+from core.sig_utils import get_icon_for_sig
+from data.alert import Alert
+
+
+# Alert provider for Wainwrights on the Air
+class WOTA(HTTPAlertProvider):
+    POLL_INTERVAL_SEC = 3600
+    ALERTS_URL = "https://www.wota.org.uk/alerts_rss.php"
+    RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
+
+    def __init__(self, provider_config):
+        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def http_response_to_alerts(self, http_response):
+        new_alerts = []
+        rss = RSSParser.parse(http_response.content.decode())
+        # Iterate through source data
+        for source_alert in rss.channel.items:
+
+            # Reject GUID missing or zero
+            if not source_alert.guid or not source_alert.guid.content or source_alert.guid.content == "http://www.wota.org.uk/alerts/0":
+                continue
+
+            # Pick apart the title
+            title_split = source_alert.title.split(" on ")
+            dx_call = title_split[0]
+            ref = None
+            ref_name = None
+            if len(title_split) > 1:
+                ref_split = title_split[1].split(" - ")
+                ref = ref_split[0]
+                if len(ref_split) > 1:
+                    ref_name = ref_split[1]
+
+            # Pick apart the description
+            desc_split = source_alert.description.split(". ")
+            freqs_modes = desc_split[0].replace("Frequencies/modes:", "").strip()
+            comment = None
+            if len(desc_split) > 1:
+                comment = desc_split[1].strip()
+
+            time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert.guid.content,
+                          dx_calls=[dx_call],
+                          freqs_modes=freqs_modes,
+                          comment=comment,
+                          sig="WOTA",
+                          sig_refs=[ref] if ref else [],
+                          sig_refs_names=[ref_name] if ref_name else [],
+                          icon=get_icon_for_sig("WOTA"),
+                          start_time=time.timestamp())
+
+            # Add to our list.
+            new_alerts.append(alert)
+        return new_alerts

alertproviders/wwff.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import pytz
 
 from alertproviders.http_alert_provider import HTTPAlertProvider
+from core.sig_utils import get_icon_for_sig
 from data.alert import Alert
 
 
@@ -26,7 +27,7 @@ class WWFF(HTTPAlertProvider):
                           comment=source_alert["remarks"],
                           sig="WWFF",
                           sig_refs=[source_alert["reference"]],
-                          icon="seedling",
+                          icon=get_icon_for_sig("WWFF"),
                           start_time=datetime.strptime(source_alert["utc_start"],
                                                        "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
                           end_time=datetime.strptime(source_alert["utc_end"],

config-example.yml

@@ -41,6 +41,14 @@ spot-providers:
     class: "ParksNPeaks"
     name: "ParksNPeaks"
     enabled: true
+  -
+    class: "ZLOTA"
+    name: "ZLOTA"
+    enabled: true
+  -
+    class: "WOTA"
+    name: "WOTA"
+    enabled: true
   -
     class: "APRSIS"
     name: "APRS-IS"
@@ -84,6 +92,14 @@ alert-providers:
     class: "WWFF"
     name: "WWFF"
     enabled: true
+  -
+    class: "ParksNPeaks"
+    name: "ParksNPeaks"
+    enabled: true
+  -
+    class: "WOTA"
+    name: "WOTA"
+    enabled: true
   -
     class: "NG3K"
     name: "NG3K"

core/constants.py

@@ -1,20 +1,39 @@
 from core.config import SERVER_OWNER_CALLSIGN
 from data.band import Band
+from data.sig import SIG
 
 # General software
 SOFTWARE_NAME = "Spothole by M0TRT"
 SOFTWARE_VERSION = "0.1"
 
 # HTTP headers used for spot providers that use HTTP
-HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + " " + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
+HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + ", v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
 
 # Special Interest Groups
-SIGS = ["POTA", "SOTA", "WWFF", "GMA", "WWBOTA", "HEMA", "MOTA", "ARLHS", "ILLW", "SiOTA", "WCA", "ZLOTA", "IOTA"]
+SIGS = [
+   SIG(name="POTA", description="Parks on the Air", icon="tree", ref_regex=r"[A-Z]{2}\-\d+"),
+   SIG(name="SOTA", description="Summits on the Air", icon="mountain-sun", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d+"),
+   SIG(name="WWFF", description="World Wide Flora & Fauna", icon="seedling", ref_regex=r"[A-Z0-9]{1,3}FF\-\d+"),
+   SIG(name="GMA", description="Global Mountain Activity", icon="person-hiking", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d+"),
+   SIG(name="WWBOTA", description="Worldwide Bunkers on the Air", icon="radiation", ref_regex=r"B\/[A-Z0-9]{1,3}\-\d+"),
+   SIG(name="HEMA", description="HuMPs Excluding Marilyns Award", icon="mound", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d+"),
+   SIG(name="IOTA", description="Islands on the Air", icon="umbrella-beach", ref_regex=r"[A-Z]{2}\-\d+"),
+   SIG(name="MOTA", description="Mills on the Air", icon="fan", ref_regex=r"X\d{4-6}"),
+   SIG(name="ARLHS", description="Amateur Radio Lighthouse Society", icon="tower-observation", ref_regex=r"[A-Z]{3}\-\d+"),
+   SIG(name="ILLW", description="International Lighthouse & Lightship Weekend", icon="tower-observation", ref_regex=r"[A-Z]{2}\d{4}"),
+   SIG(name="SIOTA", description="Silos on the Air", icon="wheat-awn", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
+   SIG(name="WCA", description="World Castles Award", icon="chess-rook", ref_regex=r"[A-Z0-9]{1,3}\-\d+"),
+   SIG(name="ZLOTA", description="New Zealand on the Air", icon="kiwi-bird", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d+"),
+   SIG(name="KRMNPA", description="Keith Roget Memorial National Parks Award", icon="earth-oceania", ref_regex=r""),
+   SIG(name="WAB", description="Worked All Britain", icon="table-cells-large", ref_regex=r"[A-Z]{1,2}[0-9]{2}"),
+   SIG(name="WAI", description="Worked All Ireland", icon="table-cells-large", ref_regex=r"[A-Z][0-9]{2}"),
+   SIG(name="WOTA", description="Wainwrights on the Air", icon="w", ref_regex=r"[A-Z]{3}-[0-9]{2}")
+]
 
 # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
 CW_MODES = ["CW"]
 PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "M17"]
-DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "BPSK", "PSK", "PSK31", "BPSK31", "OLIVIA"]
+DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "BPSK", "PSK", "PSK31", "BPSK31", "OLIVIA", "MFSK", "MFSK32", "PKT"]
 ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
 MODE_TYPES = ["CW", "PHONE", "DATA"]
 

core/geo_utils.py

@@ -0,0 +1,103 @@
+import logging
+import re
+from math import floor
+
+from pyproj import Transformer
+
+TRANSFORMER_OS_GRID_TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326")
+TRANSFORMER_IRISH_GRID_TO_WGS84 = Transformer.from_crs("EPSG:29903", "EPSG:4326")
+TRANSFORMER_CI_UTM_GRID_TO_WGS84 = Transformer.from_crs("+proj=utm +zone=30 +ellps=WGS84", "EPSG:4326")
+
+
+# Convert a Worked All Britain or Worked All Ireland reference to a lat/lon point.
+def wab_wai_square_to_lat_lon(ref):
+    # First check we have a valid grid square, and based on what it looks like, use either the Ordnance Survey, Irish,
+    # or UTM grid systems to perform the conversion.
+    if re.match(r"^[HNOST][ABCDEFGHJKLMNOPQRSTUVWXYZ][0-9]{2}$", ref):
+        return os_grid_square_to_lat_lon(ref)
+    elif re.match(r"^[ABCDEFGHJKLMNOPQRSTUVWXYZ][0-9]{2}$", ref):
+        return irish_grid_square_to_lat_lon(ref)
+    elif re.match(r"^W[AV][0-9]{2}$", ref):
+        return utm_grid_square_to_lat_lon(ref)
+    else:
+        logging.warn("Invalid WAB/WAI square: " + ref)
+        return None
+
+
+# Get a lat/lon point for the centre of an Ordnance Survey grid square
+def os_grid_square_to_lat_lon(ref):
+    # Convert the letters into multipliers for the 500km squares and 100km squares
+    offset_500km_multiplier = ord(ref[0]) - 65
+    offset_100km_multiplier = ord(ref[1]) - 65
+
+    # The letter "I" is not used in the grid, so any offset of 8 or more needs to be reduced by 1.
+    if offset_500km_multiplier >= 8:
+        offset_500km_multiplier = offset_500km_multiplier - 1
+    if offset_100km_multiplier >= 8:
+        offset_100km_multiplier = offset_100km_multiplier - 1
+
+    # Convert the offsets into increments of 100km from the false origin (grid square SV):
+    easting_100km = ((offset_500km_multiplier - 2) % 5) * 5 + (offset_100km_multiplier % 5)
+    northing_100km = (19 - floor(offset_500km_multiplier / 5) * 5) - floor(offset_100km_multiplier / 5)
+
+    # Take the numeric parts of the grid square and multiply by 10000 to get metres, then combine with the 100km
+    # box offsets
+    easting = int(ref[2]) * 10000 + easting_100km * 100000
+    northing = int(ref[3]) * 10000 + northing_100km * 100000
+
+    # Add 5000m to each value to get the middle of the box rather than the south-west corner
+    easting = easting + 5000
+    northing = northing + 5000
+
+    # Reproject to WGS84 lat/lon
+    lat, lon = TRANSFORMER_OS_GRID_TO_WGS84.transform(easting, northing)
+    return lat, lon
+
+
+# Get a lat/lon point for the centre of an Irish Grid square.
+def irish_grid_square_to_lat_lon(ref):
+    # Convert the letters into multipliers for the 100km squares
+    offset_100km_multiplier = ord(ref[0]) - 65
+
+    # The letter "I" is not used in the grid, so any offset of 8 or more needs to be reduced by 1.
+    if offset_100km_multiplier >= 8:
+        offset_100km_multiplier = offset_100km_multiplier - 1
+
+    # Convert the offsets into increments of 100km from the false origin:
+    easting_100km = offset_100km_multiplier % 5
+    northing_100km = 4 - floor(offset_100km_multiplier / 5)
+
+    # Take the numeric parts of the grid square and multiply by 10000 to get metres, then combine with the 100km
+    # box offsets
+    easting = int(ref[1]) * 10000 + easting_100km * 100000
+    northing = int(ref[2]) * 10000 + northing_100km * 100000
+
+    # Add 5000m to each value to get the middle of the box rather than the south-west corner
+    easting = easting + 5000
+    northing = northing + 5000
+
+    # Reproject to WGS84 lat/lon
+    lat, lon = TRANSFORMER_IRISH_GRID_TO_WGS84.transform(easting, northing)
+    return lat, lon
+
+
+# Get a lat/lon point for the centre of a UTM grid square (supports only squares WA & WV for the Channel Islands, nothing else implemented)
+def utm_grid_square_to_lat_lon(ref):
+    # Take the numeric parts of the grid square and multiply by 10000 to get metres from the corner of the letter-based grid square
+    easting = int(ref[2]) * 10000
+    northing = int(ref[3]) * 10000
+
+    # Apply the appropriate offset based on whether the square is WA or WV
+    easting = easting + 500000
+    if ref[1] == "A":
+        northing = northing + 5500000
+    else:
+        northing = northing + 5400000
+
+    # Add 5000m to each value to get the middle of the box rather than the south-west corner
+    easting = easting + 5000
+    northing = northing + 5000
+
+    # Reproject to WGS84 lat/lon
+    lat, lon = TRANSFORMER_CI_UTM_GRID_TO_WGS84.transform(easting, northing)
+    return lat, lon

core/lookup_helper.py

@@ -3,7 +3,7 @@ import logging
 from datetime import timedelta
 
 from diskcache import Cache
-from pyhamtools import LookupLib, Callinfo
+from pyhamtools import LookupLib, Callinfo, callinfo
 from pyhamtools.exceptions import APIKeyMissingError
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
@@ -266,36 +266,46 @@ class LookupHelper:
     # Utility method to get QRZ.com data from cache if possible, if not get it from the API and cache it
     def get_qrz_data_for_callsign(self, call):
         # Fetch from cache if we can, otherwise fetch from the API and cache it
-        qrz_data = self.QRZ_CALLSIGN_DATA_CACHE.get(call)
+        if call in self.QRZ_CALLSIGN_DATA_CACHE:
-        if qrz_data:
+            return self.QRZ_CALLSIGN_DATA_CACHE.get(call)
-            return qrz_data
         elif self.QRZ_AVAILABLE:
             try:
                 data = self.LOOKUP_LIB_QRZ.lookup_callsign(callsign=call)
                 self.QRZ_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
                 return data
             except KeyError:
-                # QRZ had no info for the call, that's OK. Cache a None so we don't try to look this up again
+                # QRZ had no info for the call, but maybe it had prefixes or suffixes. Try again with the base call.
-                self.QRZ_CALLSIGN_DATA_CACHE.add(call, None, expire=604800)  # 1 week in seconds
+                try:
-                return None
+                    data = self.LOOKUP_LIB_QRZ.lookup_callsign(callsign=callinfo.Callinfo.get_homecall(call))
+                    self.QRZ_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
+                    return data
+                except KeyError:
+                    # QRZ had no info for the call, that's OK. Cache a None so we don't try to look this up again
+                    self.QRZ_CALLSIGN_DATA_CACHE.add(call, None, expire=604800)  # 1 week in seconds
+                    return None
         else:
             return None
 
     # Utility method to get Clublog API data from cache if possible, if not get it from the API and cache it
     def get_clublog_api_data_for_callsign(self, call):
         # Fetch from cache if we can, otherwise fetch from the API and cache it
-        clublog_data = self.CLUBLOG_CALLSIGN_DATA_CACHE.get(call)
+        if call in self.CLUBLOG_CALLSIGN_DATA_CACHE:
-        if clublog_data:
+            return self.CLUBLOG_CALLSIGN_DATA_CACHE.get(call)
-            return clublog_data
         elif self.CLUBLOG_API_AVAILABLE:
             try:
                 data = self.LOOKUP_LIB_CLUBLOG_API.lookup_callsign(callsign=call)
                 self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
                 return data
             except KeyError:
-                # Clublog had no info for the call, that's OK. Cache a None so we don't try to look this up again
+                # Clublog had no info for the call, but maybe it had prefixes or suffixes. Try again with the base call.
-                self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, None, expire=604800)  # 1 week in seconds
+                try:
-                return None
+                    data = self.LOOKUP_LIB_CLUBLOG_API.lookup_callsign(callsign=callinfo.Callinfo.get_homecall(call))
+                    self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
+                    return data
+                except KeyError:
+                    # Clublog had no info for the call, that's OK. Cache a None so we don't try to look this up again
+                    self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, None, expire=604800)  # 1 week in seconds
+                    return None
             except APIKeyMissingError:
                 # User API key was wrong, warn
                 logging.error("Could not look up via Clublog API, key " + self.CLUBLOG_API_KEY + " was rejected.")

core/sig_utils.py

@@ -0,0 +1,21 @@
+from core.constants import SIGS
+
+# Utility function to get the icon for a named SIG. If no match is found, the "circle-question" icon will be returned.
+def get_icon_for_sig(sig):
+    for s in SIGS:
+        if s.name == sig:
+            return s.icon
+    return "circle-question"
+
+# Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned.
+def get_ref_regex_for_sig(sig):
+    for s in SIGS:
+        if s.name == sig:
+            return s.ref_regex
+    return None
+
+# Regex matching any SIG
+ANY_SIG_REGEX = r"(" + r"|".join(list(map(lambda p: p.name, SIGS))) + r")"
+
+# Regex matching any SIG reference
+ANY_XOTA_SIG_REF_REGEX = r"[\w\/]+\-\d+"

data/alert.py

@@ -1,6 +1,7 @@
 import copy
 import hashlib
 import json
+import re
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 
@@ -8,6 +9,7 @@ import pytz
 
 from core.constants import DXCC_FLAGS
 from core.lookup_helper import lookup_helper
+from core.sig_utils import get_icon_for_sig
 
 
 # Data class that defines an alert.
@@ -58,7 +60,7 @@ class Alert:
     # Activation score. SOTA only
     activation_score: int = None
     # Icon, from the Font Awesome set. This is fairly opinionated but is here to help the alerthole web UI and Field alertter. Does not include the "fa-" prefix.
-    icon: str = "question"
+    icon: str = None
     # Whether this alert is for a DXpedition, as opposed to e.g. an xOTA programme.
     is_dxpedition: bool = False
     # Where we got the alert from, e.g. "POTA", "SOTA"...
@@ -99,6 +101,10 @@ class Alert:
         if self.dx_dxcc_id and self.dx_dxcc_id in DXCC_FLAGS and not self.dx_flag:
             self.dx_flag = DXCC_FLAGS[self.dx_dxcc_id]
 
+        # Icon from SIG
+        if self.sig and not self.icon:
+            self.icon = get_icon_for_sig(self.sig)
+
         # DX operator details lookup, using QRZ.com. This should be the last resort compared to taking the data from
         # the actual alertting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of
         # the one from the park reference they're at.

data/sig.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+
+# Data class that defines a Special Interest Group.
+@dataclass
+class SIG:
+    # SIG name, e.g. "POTA"
+    name: str
+    # Description, e.g. "Parks on the Air"
+    description: str
+    # Icon to use for it, from the Font Awesome set. This is fairly opinionated but is here to help the Spothole web UI
+    # and Field Spotter. Does not include the "fa-" prefix.
+    icon: str
+    # Regex matcher for references, e.g. for POTA r"[A-Z]{2}\-\d+".
+    ref_regex: str

data/spot.py

@@ -10,7 +10,9 @@ import pytz
 from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 
 from core.constants import DXCC_FLAGS
+from core.geo_utils import wab_wai_square_to_lat_lon
 from core.lookup_helper import lookup_helper
+from core.sig_utils import get_icon_for_sig
 
 
 # Data class that defines a spot.
@@ -19,7 +21,6 @@ class Spot:
     # Unique identifier for the spot
     id: str = None
 
-
     # DX (spotted) operator info
 
     # Callsign of the operator that has been spotted
@@ -50,12 +51,13 @@ class Spot:
     # lookup
     dx_latitude: float = None
     dx_longitude: float = None
-    # DX Location source. Indicates how accurate the location might be. Values: "SPOT", "QRZ, "DXCC", "NONE"
+    # DX Location source. Indicates how accurate the location might be. Values: "SPOT", "WAB/WAI GRID", "QRZ", "DXCC", "NONE"
     dx_location_source: str = "NONE"
-    # DX Location good. Indicates that the software thinks the location data is good enough to plot on a map.
+    # DX Location good. Indicates that the software thinks the location data is good enough to plot on a map. This is
+    # true if the location source is "SPOT" or "WAB/WAI GRID", or if the location source is "QRZ" and the DX callsign
+    # doesn't have a suffix like /P.
     dx_location_good: bool = False
 
-
     # DE (Spotter) info
 
     # Callsign of the spotter
@@ -76,7 +78,6 @@ class Spot:
     de_latitude: float = None
     de_longitude: float = None
 
-
     # General QSO info
 
     # Reported mode, such as SSB, PHONE, CW, FT8...
@@ -94,7 +95,6 @@ class Spot:
     # QRT state. Some APIs return spots marked as QRT. Otherwise we can check the comments.
     qrt: bool = False
 
-
     # Special Interest Group info
 
     # Special Interest Group (SIG), e.g. outdoor activity programme such as POTA
@@ -108,18 +108,16 @@ class Spot:
     # Activation score. SOTA only
     activation_score: int = None
 
-
     # Display guidance (optional)
 
     # Icon, from the Font Awesome set. This is fairly opinionated but is here to help the Spothole web UI and Field
     # Spotter. Does not include the "fa-" prefix.
-    icon: str = "question"
+    icon: str = None
     # Colour to represent this spot, if a client chooses to colour spots based on their frequency band, using PSK
     # Reporter's default colours. HTML colour e.g. hex. A contrast colour is also provided which will be black or white.
     band_color: str = None
     band_contrast_color: str = None
 
-
     # Timing info
 
     # Time of the spot, UTC seconds since UNIX epoch
@@ -133,7 +131,6 @@ class Spot:
     # Time that this software received the spot, ISO 8601
     received_time_iso: str = None
 
-
     # Source info
 
     # Where we got the spot from, e.g. "POTA", "Cluster"...
@@ -180,6 +177,20 @@ class Spot:
         if self.de_call and "-" in self.de_call:
             self.de_call = self.de_call.split("-")[0]
 
+        # If we have a spotter of "RBNHOLE", we should have the actual spotter callsign in the comment, so extract it.
+        # RBNHole posts come from a number of providers, so it's dealt with here in the generic spot handling code.
+        if self.de_call == "RBNHOLE" and self.comment:
+            rbnhole_call_match = re.search(r"\Wat ([a-z0-9/]+)\W", self.comment, re.IGNORECASE)
+            if rbnhole_call_match:
+                self.de_call = rbnhole_call_match.group(1).upper()
+
+        # If we have a spotter of "SOTAMAT", we might have the actual spotter callsign in the comment, if so extract it.
+        # SOTAMAT can do POTA as well as SOTA, so it's dealt with here in the generic spot handling code.
+        if self.de_call == "SOTAMAT" and self.comment:
+            sotamat_call_match = re.search(r"\Wfrom ([a-z0-9/]+)]", self.comment, re.IGNORECASE)
+            if sotamat_call_match:
+                self.de_call = sotamat_call_match.group(1).upper()
+
         # Spotter country, continent, zones etc. from callsign.
         # DE of "RBNHOLE" and "SOTAMAT" are not things we can look up location for
         if self.de_call != "RBNHOLE" and self.de_call != "SOTAMAT":
@@ -218,6 +229,10 @@ class Spot:
         if self.mode and not self.mode_type:
             self.mode_type = lookup_helper.infer_mode_type_from_mode(self.mode)
 
+        # Icon from SIG
+        if self.sig and not self.icon:
+            self.icon = get_icon_for_sig(self.sig)
+
         # DX Grid to lat/lon and vice versa
         if self.dx_grid and not self.dx_latitude:
             ll = locator_to_latlong(self.dx_grid)
@@ -231,17 +246,23 @@ class Spot:
         if self.dx_latitude:
             self.dx_location_source = "SPOT"
 
+        # WAB/WAI grid to lat/lon
+        if not self.dx_latitude and self.sig and self.sig_refs and len(self.sig_refs) > 0 and (
+                self.sig == "WAB" or self.sig == "WAI"):
+            ll = wab_wai_square_to_lat_lon(self.sig_refs[0])
+            if ll:
+                self.dx_latitude = ll[0]
+                self.dx_longitude = ll[1]
+                try:
+                    self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8)
+                except:
+                    logging.debug("Invalid lat/lon received from WAB/WAI grid")
+                self.dx_location_source = "WAB/WAI GRID"
+
         # QRT comment detection
         if self.comment and not self.qrt:
             self.qrt = "QRT" in self.comment.upper()
 
-        # Clean up comments
-        if self.comment:
-            comment = re.sub(r"\[.*]:", "", self.comment)
-            comment = re.sub(r"\[.*]", "", comment)
-            comment = re.sub(r"\"\"", "", comment)
-            self.comment = comment.strip()
-
         # DX operator details lookup, using QRZ.com. This should be the last resort compared to taking the data from
         # the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of
         # the one from the park reference they're at.
@@ -266,7 +287,7 @@ class Spot:
 
         # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
         # is likely at home.
-        self.dx_location_good = self.dx_location_source == "SPOT" or (
+        self.dx_location_good = self.dx_location_source == "SPOT" or self.dx_location_source == "WAB/WAI GRID" or (
                 self.dx_location_source == "QRZ" and not "/" in self.dx_call)
 
         # DE of "RBNHOLE" and "SOTAMAT" are not things we can look up location for

requirements.txt

@@ -10,3 +10,4 @@ diskcache~=5.6.3
 psutil~=7.1.0
 requests-sse~=0.5.2
 rss-parser~=2.1.1
+pyproj~=3.7.2

server/webserver.py

@@ -31,14 +31,15 @@ class WebServer:
         bottle.BaseTemplate.defaults['software_version'] = SOFTWARE_VERSION
 
         # Routes for API calls
-        bottle.get("/api/v1/spots")(lambda: self.serve_api(self.get_spot_list_with_filters()))
+        bottle.get("/api/v1/spots")(lambda: self.serve_spots_api())
-        bottle.get("/api/v1/alerts")(lambda: self.serve_api(self.get_alert_list_with_filters()))
+        bottle.get("/api/v1/alerts")(lambda: self.serve_alerts_api())
         bottle.get("/api/v1/options")(lambda: self.serve_api(self.get_options()))
         bottle.get("/api/v1/status")(lambda: self.serve_api(self.status_data))
         bottle.post("/api/v1/spot")(lambda: self.accept_spot())
         # Routes for templated pages
         bottle.get("/")(lambda: self.serve_template('webpage_spots'))
         bottle.get("/map")(lambda: self.serve_template('webpage_map'))
+        bottle.get("/bands")(lambda: self.serve_template('webpage_bands'))
         bottle.get("/alerts")(lambda: self.serve_template('webpage_alerts'))
         bottle.get("/status")(lambda: self.serve_template('webpage_status'))
         bottle.get("/about")(lambda: self.serve_template('webpage_about'))
@@ -56,6 +57,38 @@ class WebServer:
         self.status = "Waiting"
         run(host='localhost', port=self.port)
 
+    # Serve the JSON API /spots endpoint
+    def serve_spots_api(self):
+        try:
+            data = self.get_spot_list_with_filters()
+            return self.serve_api(data)
+        except ValueError as e:
+            logging.error(e)
+            response.content_type = 'application/json'
+            response.status = 400
+            return json.dumps("Bad request - " + str(e), default=serialize_everything)
+        except Exception as e:
+            logging.error(e)
+            response.content_type = 'application/json'
+            response.status = 500
+            return json.dumps("Error - " + str(e), default=serialize_everything)
+
+    # Serve the JSON API /alerts endpoint
+    def serve_alerts_api(self):
+        try:
+            data = self.get_alert_list_with_filters()
+            return self.serve_api(data)
+        except ValueError as e:
+            logging.error(e)
+            response.content_type = 'application/json'
+            response.status = 400
+            return json.dumps("Bad request - " + str(e), default=serialize_everything)
+        except Exception as e:
+            logging.error(e)
+            response.content_type = 'application/json'
+            response.status = 500
+            return json.dumps("Error - " + str(e), default=serialize_everything)
+
     # Serve a JSON API endpoint
     def serve_api(self, data):
         self.last_api_access_time = datetime.now(pytz.UTC)
@@ -109,6 +142,7 @@ class WebServer:
 
             response.content_type = 'application/json'
             response.set_header('Cache-Control', 'no-store')
+            response.status = 201
             return json.dumps("OK", default=serialize_everything)
         except Exception as e:
             logging.error(e)
@@ -173,6 +207,11 @@ class WebServer:
                     needs_sig = query.get(k).upper() == "TRUE"
                     if needs_sig:
                         spots = [s for s in spots if s.sig]
+                case "needs_sig_ref":
+                    # If true, at least one sig ref is required, regardless of what it is, it just can't be missing.
+                    needs_sig_ref = query.get(k).upper() == "TRUE"
+                    if needs_sig_ref:
+                        spots = [s for s in spots if s.sig_refs and len(s.sig_refs) > 0]
                 case "band":
                     bands = query.get(k).split(",")
                     spots = [s for s in spots if s.band and s.band in bands]
@@ -191,6 +230,16 @@ class WebServer:
                 case "comment_includes":
                     comment_includes = query.get(k).strip()
                     spots = [s for s in spots if s.comment and comment_includes.upper() in s.comment.upper()]
+                case "allow_qrt":
+                    # If false, spots that are flagged as QRT are not returned.
+                    prevent_qrt = query.get(k).upper() == "FALSE"
+                    if prevent_qrt:
+                        spots = [s for s in spots if not s.qrt or s.qrt == False]
+                case "needs_good_location":
+                    # If true, spots require a "good" location to be returned
+                    needs_good_location = query.get(k).upper() == "TRUE"
+                    if needs_good_location:
+                        spots = [s for s in spots if s.dx_location_good]
                 case "dedupe":
                     # Ensure only the latest spot of each callsign is present in the list. This relies on the list being
                     # in reverse time order, so if any future change allows re-ordering the list, that should be done

spotproviders/dxcluster.py

@@ -7,6 +7,8 @@ from time import sleep
 import pytz
 import telnetlib3
 
+from core.constants import SIGS
+from core.sig_utils import ANY_SIG_REGEX, ANY_XOTA_SIG_REF_REGEX, get_icon_for_sig, get_ref_regex_for_sig
 from data.spot import Spot
 from core.config import SERVER_OWNER_CALLSIGN
 from spotproviders.spot_provider import SpotProvider
@@ -17,9 +19,9 @@ class DXCluster(SpotProvider):
     # Note the callsign pattern deliberately excludes calls ending in "-#", which are from RBN and can be enabled by
     # default on some clusters. If you want RBN spots, there is a separate provider for that.
     CALLSIGN_PATTERN = "([a-z|0-9|/]+)"
-    FREQUENCY_PATTERM = "([0-9|.]+)"
+    FREQUENCY_PATTERN = "([0-9|.]+)"
     LINE_PATTERN = re.compile(
-        "^DX de " + CALLSIGN_PATTERN + ":\\s+" + FREQUENCY_PATTERM + "\\s+" + CALLSIGN_PATTERN + "\\s+(.*)\\s+(\\d{4}Z)",
+        "^DX de " + CALLSIGN_PATTERN + ":\\s+" + FREQUENCY_PATTERN + "\\s+" + CALLSIGN_PATTERN + "\\s+(.*)\\s+(\\d{4}Z)",
         re.IGNORECASE)
 
     # Constructor requires hostname and port
@@ -75,6 +77,21 @@ class DXCluster(SpotProvider):
                                     icon="desktop",
                                     time=spot_datetime.timestamp())
 
+                        # See if the comment looks like it contains a SIG (and optionally SIG reference). Currently,
+                        # only one sig ref is supported. Note that this code is specifically in the DX Cluster class and
+                        # not in the general "spot" infer_missing() method. Because we only support one SIG per spot
+                        # at the moment (see issue #54), we don't want to risk e.g. a POTA spot with comment "WWFF GFF-0001"
+                        # being converted into a WWFF spot.
+                        sig_match = re.search(r"(^|\W)" + ANY_SIG_REGEX + r"($|\W)", spot.comment, re.IGNORECASE)
+                        if sig_match:
+                            spot.sig = sig_match.group(2).upper()
+                            spot.icon = get_icon_for_sig(spot.sig)
+                            ref_regex = get_ref_regex_for_sig(spot.sig)
+                            if ref_regex:
+                                sig_ref_match = re.search(r"(^|\W)" + spot.sig + r"($|\W)(" + ref_regex + r")($|\W)", spot.comment, re.IGNORECASE)
+                                if sig_ref_match:
+                                    spot.sig_refs = [sig_ref_match.group(3).upper()]
+
                         # Add to our list
                         self.submit(spot)
 

spotproviders/gma.py

@@ -5,6 +5,7 @@ import pytz
 from requests_cache import CachedSession
 
 from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -57,27 +58,21 @@ class GMA(HTTPSpotProvider):
                     match ref_info["reftype"]:
                         case "Summit":
                             spot.sig = "GMA"
-                            spot.icon = "mountain"
                         case "IOTA Island":
                             spot.sig = "IOTA"
-                            spot.icon = "umbrella-beach"
                         case "Lighthouse (ILLW)":
                             spot.sig = "ILLW"
-                            spot.icon = "tower-observation"
                         case "Lighthouse (ARLHS)":
                             spot.sig = "ARLHS"
-                            spot.icon = "tower-observation"
                         case "Castle":
-                            spot.sig = "WCA/COTA"
+                            spot.sig = "WCA"
-                            spot.icon = "chess-rook"
                         case "Mill":
                             spot.sig = "MOTA"
-                            spot.icon = "fan"
                         case _:
                             logging.warn("GMA spot found with ref type " + ref_info[
-                                "reftype"] + ", developer needs to figure out an icon for this!")
+                                "reftype"] + ", developer needs to add support for this!")
                             spot.sig = ref_info["reftype"]
-                            spot.icon = "person-hiking"
+                    spot.icon = get_icon_for_sig(spot.sig)
 
                     # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
                     # that for us.

spotproviders/hema.py

@@ -5,6 +5,7 @@ import pytz
 import requests
 
 from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -54,7 +55,7 @@ class HEMA(HTTPSpotProvider):
                                 sig="HEMA",
                                 sig_refs=[spot_items[3].upper()],
                                 sig_refs_names=[spot_items[4]],
-                                icon="mound",
+                                icon=get_icon_for_sig("HEMA"),
                                 time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(tzinfo=pytz.UTC).timestamp(),
                                 dx_latitude=float(spot_items[7]),
                                 dx_longitude=float(spot_items[8]))

spotproviders/parksnpeaks.py

@@ -1,11 +1,13 @@
 import csv
 import logging
+import re
 from datetime import datetime, timedelta
 
 import pytz
 from requests_cache import CachedSession
 
 from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -17,9 +19,6 @@ class ParksNPeaks(HTTPSpotProvider):
     SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
     SIOTA_LIST_CACHE_TIME_DAYS = 30
     SIOTA_LIST_CACHE = CachedSession("cache/siota_data_cache", expire_after=timedelta(days=SIOTA_LIST_CACHE_TIME_DAYS))
-    ZLOTA_LIST_URL = "https://ontheair.nz/assets/assets.json"
-    ZLOTA_LIST_CACHE_TIME_DAYS = 30
-    ZLOTA_LIST_CACHE = CachedSession("cache/zlota_data_cache", expire_after=timedelta(days=ZLOTA_LIST_CACHE_TIME_DAYS))
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -32,7 +31,7 @@ class ParksNPeaks(HTTPSpotProvider):
             spot = Spot(source=self.name,
                         source_id=source_spot["actID"],
                         dx_call=source_spot["actCallsign"].upper(),
-                        de_call=source_spot["actSpoter"].upper(),  # typo exists in API
+                        de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,  # typo exists in API
                         freq=float(source_spot["actFreq"].replace(",", "")) * 1000000 if (
                                     source_spot["actFreq"] != "") else None,
                         # Seen PNP spots with empty frequency, and with comma-separated thousands digits
@@ -40,22 +39,22 @@ class ParksNPeaks(HTTPSpotProvider):
                         comment=source_spot["actComments"],
                         sig=source_spot["actClass"],
                         sig_refs=[source_spot["actSiteID"]],
+                        icon=get_icon_for_sig(source_spot["actClass"]),
                         time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S").replace(
                             tzinfo=pytz.UTC).timestamp())
 
-            # PNP supports a bunch of programs which should have different icons
+            # Free text location is not present in all spots, so only add it if it's set
-            if spot.sig == "SiOTA":
+            if "actLocation" in source_spot and source_spot["actLocation"] != "":
-                spot.icon = "wheat-awn"
+                spot.sig_refs_names = [source_spot["actLocation"]]
-            elif spot.sig == "ZLOTA":
+
-                spot.icon = "kiwi-bird"
+            # Extract a de_call if it's in the comment but not in the "actSpoter" field
-            elif spot.sig in ["POTA", "SOTA", "WWFF"]:
+            m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment)
-                # Don't care about an icon as this will be rejected anyway, we have better data from POTA/SOTA/WWFF direct
+            if not spot.de_call and m:
-                spot.icon = ""
+                spot.de_call = m.group(1)
-            else:
+
-                # Unknown programme we've never seen before
+            # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-                logging.warn(
+            if spot.sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA"]:
-                    "PNP spot found with sig " + spot.sig + ", developer needs to add support for icon and grid/lat/lon lookup!")
+                logging.warn("PNP spot found with sig " + spot.sig + ", developer needs to add support for this!")
-                spot.icon = "question"
 
             # SiOTA lat/lon/grid lookup
             if spot.sig == "SiOTA":
@@ -68,20 +67,10 @@ class ParksNPeaks(HTTPSpotProvider):
                         spot.dx_grid = row["LOCATOR"]
                         break
 
-            # ZLOTA name/lat/lon lookup
+            # Note there is currently no support for KRMNPA location lookup, see issue #61.
-            if spot.sig == "ZLOTA":
-                zlota_data = self.ZLOTA_LIST_CACHE.get(self.ZLOTA_LIST_URL, headers=HTTP_HEADERS).json()
-                for asset in zlota_data:
-                    if asset["code"] == spot.sig_refs[0]:
-                        spot.sig_refs_names = [asset["name"]]
-                        spot.dx_latitude = asset["y"]
-                        spot.dx_longitude = asset["x"]
-                        # Junk the "DE call", PNP always returns "ZLOTA" as the spotter for ZLOTA spots
-                        spot.de_call = None
-                        break
 
-            # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
+            # If this is POTA, SOTA, WWFF or ZLOTA data we already have it through other means, so ignore. Otherwise,
-            # the spot list.
+            # add to the spot list.
-            if spot.sig not in ["POTA", "SOTA", "WWFF"]:
+            if spot.sig not in ["POTA", "SOTA", "WWFF", "ZLOTA"]:
                 new_spots.append(spot)
         return new_spots

spotproviders/pota.py

@@ -1,7 +1,11 @@
-from datetime import datetime
+import re
+from datetime import datetime, timedelta
 
 import pytz
+from requests_cache import CachedSession
 
+from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig, get_ref_regex_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -10,6 +14,11 @@ from spotproviders.http_spot_provider import HTTPSpotProvider
 class POTA(HTTPSpotProvider):
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://api.pota.app/spot/activator"
+    # Might need to look up extra park data
+    PARK_URL_ROOT = "https://api.pota.app/park/"
+    PARK_DATA_CACHE_TIME_DAYS = 30
+    PARK_DATA_CACHE = CachedSession("cache/pota_park_data_cache",
+                                    expire_after=timedelta(days=PARK_DATA_CACHE_TIME_DAYS))
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -30,13 +39,27 @@ class POTA(HTTPSpotProvider):
                         sig_refs=[source_spot["reference"]],
                         sig_refs_names=[source_spot["name"]],
                         sig_refs_urls=["https://pota.app/#/park/" + source_spot["reference"]],
-                        icon="tree",
+                        icon=get_icon_for_sig("POTA"),
-                        time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
+                        time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(
+                            tzinfo=pytz.UTC).timestamp(),
                         dx_grid=source_spot["grid6"],
                         dx_latitude=source_spot["latitude"],
                         dx_longitude=source_spot["longitude"])
 
+            # Sometimes we can get other refs in the comments for n-fer activations, extract them
+            all_comment_refs = re.findall(get_ref_regex_for_sig("POTA"), spot.comment)
+            for r in all_comment_refs:
+                if r not in spot.sig_refs:
+                    spot.sig_refs.append(r.upper())
+                    spot.sig_refs_urls.append("https://pota.app/#/park/" + r.upper())
+
+                    # Now we need to look up the name of that reference from the API, because the comment won't have it
+                    park_response = self.PARK_DATA_CACHE.get(self.PARK_URL_ROOT + r.upper(), headers=HTTP_HEADERS)
+                    park_data = park_response.json()
+                    if park_data and "name" in park_data:
+                        spot.sig_refs_names.append(park_data["name"])
+
             # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
             # that for us.
             new_spots.append(spot)
-        return new_spots
+        return new_spots

spotproviders/sota.py

@@ -5,6 +5,7 @@ import requests
 from requests_cache import CachedSession
 
 from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -51,7 +52,7 @@ class SOTA(HTTPSpotProvider):
                             sig_refs=[source_spot["summitCode"]],
                             sig_refs_names=[source_spot["summitName"]],
                             sig_refs_urls=["https://www.sotadata.org.uk/en/summit/" + source_spot["summitCode"]],
-                            icon="mountain-sun",
+                            icon=get_icon_for_sig("SOTA"),
                             time=datetime.fromisoformat(source_spot["timeStamp"]).timestamp(),
                             activation_score=source_spot["points"])
 

spotproviders/sse_spot_provider.py

@@ -34,11 +34,19 @@ class SSESpotProvider(SpotProvider):
         if self.thread:
             self.thread.join()
 
+    def _on_open(self):
+        self.status = "Waiting for Data"
+
+    def _on_error(self):
+        self.status = "Connecting"
+
     def run(self):
         while not self.stopped:
             try:
                 logging.debug("Connecting to " + self.name + " spot API...")
-                with EventSource(self.url, headers=HTTP_HEADERS, latest_event_id=self.last_event_id, timeout=30) as event_source:
+                self.status = "Connecting"
+                with EventSource(self.url, headers=HTTP_HEADERS, latest_event_id=self.last_event_id, timeout=30,
+                                 on_open=self._on_open, on_error=self._on_error) as event_source:
                     self.event_source = event_source
                     for event in self.event_source:
                         if event.type == 'message':
@@ -58,6 +66,8 @@ class SSESpotProvider(SpotProvider):
             except Exception as e:
                 self.status = "Error"
                 logging.exception("Exception in SSE Spot Provider (" + self.name + ")")
+            else:
+                self.status = "Disconnected"
             sleep(5)  # Wait before trying to reconnect
 
     # Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass

spotproviders/wota.py

@@ -0,0 +1,88 @@
+from datetime import timedelta, datetime
+
+import pytz
+from requests_cache import CachedSession
+from rss_parser import RSSParser
+
+from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
+from data.spot import Spot
+from spotproviders.http_spot_provider import HTTPSpotProvider
+
+
+# Spot provider for Wainwrights on the Air
+class WOTA(HTTPSpotProvider):
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://www.wota.org.uk/spots_rss.php"
+    LIST_URL = "https://www.wota.org.uk/mapping/data/summits.json"
+    LIST_CACHE_TIME_DAYS = 30
+    LIST_CACHE = CachedSession("cache/wota_data_cache", expire_after=timedelta(days=LIST_CACHE_TIME_DAYS))
+    RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
+
+    def __init__(self, provider_config):
+        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def http_response_to_spots(self, http_response):
+        new_spots = []
+        rss = RSSParser.parse(http_response.content.decode())
+        # Iterate through source data
+        for source_spot in rss.channel.items:
+
+            # Reject GUID missing or zero
+            if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0":
+                continue
+
+            # Pick apart the title
+            title_split = source_spot.title.split(" on ")
+            dx_call = title_split[0]
+            ref = None
+            ref_name = None
+            if len(title_split) > 1:
+                ref_split = title_split[1].split(" - ")
+                ref = ref_split[0]
+                if len(ref_split) > 1:
+                    ref_name = ref_split[1]
+
+            # Pick apart the description
+            desc_split = source_spot.description.split(". ")
+            freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
+            freq_mode_split = freq_mode.split("-")
+            freq_hz = float(freq_mode_split[0]) * 1000000
+            mode = freq_mode_split[1]
+
+            comment = None
+            if len(desc_split) > 1:
+                comment = desc_split[1].strip()
+            spotter = None
+            if len(desc_split) > 2:
+                spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").strip()
+
+            time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+
+            # Convert to our spot format
+            spot = Spot(source=self.name,
+                        source_id=source_spot.guid.content,
+                        dx_call=dx_call,
+                        de_call=spotter,
+                        freq=freq_hz,
+                        mode=mode,
+                        comment=comment,
+                        sig="WOTA",
+                        sig_refs=[ref] if ref else [],
+                        sig_refs_names=[ref_name] if ref_name else [],
+                        sig_refs_urls="https://www.wota.org.uk/MM_" + ref if ref else [],
+                        icon=get_icon_for_sig("WOTA"),
+                        time=time.timestamp())
+
+            # WOTA name/lat/lon lookup
+            wota_data = self.LIST_CACHE.get(self.LIST_URL, headers=HTTP_HEADERS).json()
+            for feature in wota_data["features"]:
+                if feature["properties"]["wotaId"] == spot.sig_refs[0]:
+                    spot.sig_refs_names = [feature["properties"]["title"]]
+                    spot.dx_latitude = feature["geometry"]["coordinates"][1]
+                    spot.dx_longitude = feature["geometry"]["coordinates"][0]
+                    spot.dx_grid = feature["properties"]["qthLocator"]
+                    break
+
+            new_spots.append(spot)
+        return new_spots

spotproviders/wwbota.py

@@ -1,6 +1,7 @@
 import json
 from datetime import datetime
 
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.sse_spot_provider import SSESpotProvider
 
@@ -38,7 +39,7 @@ class WWBOTA(SSESpotProvider):
                     sig="WWBOTA",
                     sig_refs=refs,
                     sig_refs_names=ref_names,
-                    icon="radiation",
+                    icon=get_icon_for_sig("WWBOTA"),
                     time=datetime.fromisoformat(source_spot["time"]).timestamp(),
                     # WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
                     # now, we will just pick the first one to use as our grid, latitude and longitude.

spotproviders/wwff.py

@@ -2,6 +2,7 @@ from datetime import datetime
 
 import pytz
 
+from core.sig_utils import get_icon_for_sig
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
@@ -30,7 +31,7 @@ class WWFF(HTTPSpotProvider):
                         sig_refs=[source_spot["reference"]],
                         sig_refs_names=[source_spot["reference_name"]],
                         sig_refs_urls=["https://wwff.co/directory/?showRef=" + source_spot["reference"]],
-                        icon="seedling",
+                        icon=get_icon_for_sig("WWFF"),
                         time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
                         dx_latitude=source_spot["latitude"],
                         dx_longitude=source_spot["longitude"])

spotproviders/zlota.py

@@ -0,0 +1,59 @@
+import csv
+import logging
+import re
+from datetime import datetime, timedelta
+
+import pytz
+from requests_cache import CachedSession
+
+from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
+from data.spot import Spot
+from spotproviders.http_spot_provider import HTTPSpotProvider
+
+
+# Spot provider for ZLOTA
+class ZLOTA(HTTPSpotProvider):
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://ontheair.nz/api/spots?zlota_only=true"
+    LIST_URL = "https://ontheair.nz/assets/assets.json"
+    LIST_CACHE_TIME_DAYS = 30
+    LIST_CACHE = CachedSession("cache/zlota_data_cache", expire_after=timedelta(days=LIST_CACHE_TIME_DAYS))
+
+    def __init__(self, provider_config):
+        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def http_response_to_spots(self, http_response):
+        new_spots = []
+        # Iterate through source data
+        for source_spot in http_response.json():
+            # Frequency is often inconsistent as to whether it's in Hz or kHz. Make a guess.
+            freq_hz = float(source_spot["frequency"])
+            if freq_hz < 1000000:
+                freq_hz = freq_hz * 1000
+
+            # Convert to our spot format
+            spot = Spot(source=self.name,
+                        source_id=source_spot["id"],
+                        dx_call=source_spot["activator"].upper(),
+                        de_call=source_spot["spotter"].upper(),
+                        freq=freq_hz,
+                        mode=source_spot["mode"].upper().strip(),
+                        comment=source_spot["comments"],
+                        sig="ZLOTA",
+                        sig_refs=[source_spot["reference"]],
+                        sig_refs_names=[source_spot["name"]],
+                        icon=get_icon_for_sig("ZLOTA"),
+                        time=datetime.fromisoformat(source_spot["referenced_time"]).astimezone(pytz.UTC).timestamp())
+
+            # ZLOTA name/lat/lon lookup
+            zlota_data = self.LIST_CACHE.get(self.LIST_URL, headers=HTTP_HEADERS).json()
+            for asset in zlota_data:
+                if asset["code"] == spot.sig_refs[0]:
+                    spot.sig_refs_names = [asset["name"]]
+                    spot.dx_latitude = asset["y"]
+                    spot.dx_longitude = asset["x"]
+                    break
+
+            new_spots.append(spot)
+        return new_spots

views/webpage_about.tpl

@@ -24,6 +24,8 @@
     <h4 class="mt-4">Why does this website ask me if I want to install it?</h4>
     <p>Spothole is a Progressive Web App, which means you can install it on an Android or iOS device by opening the site in Chrome or Safari respectively, and clicking "Install" on the pop-up panel. It'll only prompt you once, so if you dismiss the prompt and change your mind, you'll find an Install / Add to Home Screen option on your browser's menu.</p>
     <p>Installing Spothole on your phone is completely optional, the website works exactly the same way as the "app" does.</p>
+    <h4 class="mt-4">Why hasn't my spot/alert shown up yet?</h4>
+    <p>To avoid putting too much load on the various servers that Spothole connects to, the Spothole server only polls them once every two minutes for spots, and once every hour for alerts. (Some sources, such as DX clusters, RBN, APRS-IS and WWBOTA use a non-polling mechanism, and their updates will therefore arrive more quickly.) Then if you are using the web interface, that has its own rate at which it reloads the data from Spothole, which is once a minute for spots or 30 minutes for alerts. So you could be waiting around three minutes to see a newly added spot, or 90 minutes to see a newly added alert.</p>
     <h4 class="mt-4">What licence does Spothole use?</h4>
     <p>Spothole's source code is licenced under the Public Domain. You can write a Spothole client, run your own server, modify it however you like, you can claim you wrote it and charge people £1000 for a copy, I don't really mind. (Please don't do the last one. But if you're using my code for something cool, it would be nice to hear from you!)</p>
     <h2 id="privacy" class="mt-4">Privacy</h2>

views/webpage_bands.tpl

@@ -0,0 +1,117 @@
+% rebase('webpage_base.tpl')
+
+<div class="mt-3">
+    <div class="row">
+        <div class="col-auto me-auto pt-3">
+            <p id="timing-container">Loading...</p>
+        </div>
+        <div class="col-auto">
+            <p class="d-inline-flex gap-1">
+                <button id="filters-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleFiltersPanel();"><i class="fa-solid fa-filter"></i> Filters</button>
+                <button id="display-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleDisplayPanel();"><i class="fa-solid fa-desktop"></i> Display</button>
+            </p>
+        </div>
+    </div>
+
+    <div id="filters-area" class="appearing-panel card mb-3">
+        <div class="card-header text-white bg-primary">
+            <div class="row">
+                <div class="col-auto me-auto">
+                    Filters
+                </div>
+                <div class="col-auto d-inline-flex">
+                    <button id="close-filters-button" type="button" class="btn-close btn-close-white" aria-label="Close" onclick="closeFiltersPanel();"></button>
+                </div>
+            </div>
+
+        </div>
+        <div class="card-body">
+            <div class="row row-cols-1 g-4 mb-4">
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Bands</h5>
+                            <p id="band-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+            <div class="row row-cols-1 row-cols-md-4 g-4">
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">DX Continent</h5>
+                            <p id="dx-continent-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">DE Continent</h5>
+                            <p id="de-continent-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Modes</h5>
+                            <p id="mode-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Sources</h5>
+                            <p id="source-options" class="card-text spothole-card-text"></p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div id="display-area" class="appearing-panel card mb-3">
+        <div class="card-header text-white bg-primary">
+            <div class="row">
+                <div class="col-auto me-auto">
+                    Display
+                </div>
+                <div class="col-auto d-inline-flex">
+                    <button id="close-display-button" type="button" class="btn-close btn-close-white" aria-label="Close" onclick="closeDisplayPanel();"></button>
+                </div>
+            </div>
+
+        </div>
+        <div class="card-body">
+            <div id="display-container" class="row row-cols-1 row-cols-md-4 g-4">
+                <div class="col">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Spot Age</h5>
+                            <p class="card-text spothole-card-text">Last
+                                <select id="max-spot-age" class="storeable-select form-select ms-2 me-2 d-inline-block" oninput="filtersUpdated();" style="width: 5em; display: inline-block;">
+                                    <option value="300">5</option>
+                                    <option value="600">10</option>
+                                    <option value="1800" selected>30</option>
+                                    <option value="3600">60</option>
+                                </select>
+                                minutes
+                            </p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div id="bands-container"></div>
+
+</div>
+
+<script src="/js/common.js"></script>
+<script src="/js/spotandmap.js"></script>
+<script src="/js/bands.js"></script>
+<script>$(document).ready(function() { $("#nav-link-bands").addClass("active"); }); <!-- highlight active page in nav --></script>

views/webpage_base.tpl

@@ -31,6 +31,7 @@
     <link href="/fa/css/solid.min.css" rel="stylesheet" />
 
     <link rel="icon" type="image/png" href="/img/icon-512.png">
+    <link rel="apple-touch-icon" href="img/icon-512-pwa.png">
     <link rel="alternate icon" type="image/png" href="/img/icon-192.png">
     <link rel="alternate icon" type="image/png" href="/img/icon-32.png">
     <link rel="alternate icon" type="image/png" href="/img/icon-16.png">
@@ -59,6 +60,7 @@
                 <ul class="navbar-nav me-auto mb-2 mb-lg-0">
                     <li class="nav-item ms-4"><a href="/" class="nav-link" id="nav-link-spots"><i class="fa-solid fa-tower-cell"></i> Spots</a></li>
                     <li class="nav-item ms-4"><a href="/map" class="nav-link" id="nav-link-map"><i class="fa-solid fa-map"></i> Map</a></li>
+                    <li class="nav-item ms-4"><a href="/bands" class="nav-link" id="nav-link-bands"><i class="fa-solid fa-ruler-vertical"></i> Bands</a></li>
                     <li class="nav-item ms-4"><a href="/alerts" class="nav-link" id="nav-link-alerts"><i class="fa-solid fa-bell"></i> Alerts</a></li>
                     <li class="nav-item ms-4"><a href="/status" class="nav-link" id="nav-link-status"><i class="fa-solid fa-chart-simple"></i> Status</a></li>
                     <li class="nav-item ms-4"><a href="/about" class="nav-link" id="nav-link-about"><i class="fa-solid fa-circle-info"></i> About</a></li>

webassets/apidocs/openapi.yml

@@ -85,7 +85,14 @@ paths:
               - IOTA
         - name: needs_sig
           in: query
-          description: "Limit the spots to only ones from a Special Interest Grous such as POTA. Because supplying all known SIGs as a `sigs` parameter is unwieldy, and leaving `sigs` blank will also return spots with *no* SIG, this parameter can be set true to return only spots with a SIG, regardless of what it is, so long as it's not blank. This is what Field Spotter uses to exclude generic cluster spots and only retrieve xOTA things."
+          description: "Limit the spots to only ones with a Special Interest Group such as POTA. Because supplying all known SIGs as a `sigs` parameter is unwieldy, and leaving `sigs` blank will also return spots with *no* SIG, this parameter can be set true to return only spots with a SIG, regardless of what it is, so long as it's not blank. This is what Field Spotter uses to exclude generic cluster spots and only retrieve xOTA things."
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: needs_sig_ref
+          in: query
+          description: "Limit the spots to only ones which have at least one reference (e.g. a park reference) for Special Interest Groups such as POTA."
           required: false
           schema:
             type: boolean
@@ -195,6 +202,20 @@ paths:
           required: false
           schema:
             type: string
+        - name: needs_good_location
+          in: query
+          description: "Return only spots with a 'good' location. (See the spot `dx_location_good` parameter for details. Useful for map-based clients, to avoid spots with 'bad' locations e.g. loads of cluster spots ending up in the centre of the DXCC entitity.)"
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: allow_qrt
+          in: query
+          description: Allow spots that are known to be QRT to be returned.
+          required: false
+          schema:
+            type: boolean
+            default: true
       responses:
         '200':
           description: Success
@@ -413,8 +434,7 @@ paths:
                     type: array
                     description: An array of all the supported Special Interest Groups.
                     items:
-                      type: string
+                      $ref: '#/components/schemas/SIG'
-                      example: "POTA"
                   sources:
                     type: array
                     description: An array of all the supported data sources.
@@ -552,13 +572,14 @@ components:
           description: Where we got the DX location (grid/latitude/longitude) from. If this was from the spot itself, it's likely quite accurate, but if we had to fall back to QRZ lookup, or even a location based on the DXCC itself, it will be a lot less accurate.
           enum:
             - SPOT
+            - "WAB/WAI GRID"
             - QRZ
             - DXCC
             - NONE
           example: SPOT
         dx_location_good:
           type: boolean
-          description: Does the software think the location is good enough to put a marker on a map? This is true if the source is "SPOT", or alternatively if the source is "QRZ" and the callsign doesn't have a slash in it (i.e. operator likely at home).
+          description: Does the software think the location is good enough to put a marker on a map? This is true if the source is "SPOT" or "WAB/WAI GRID", or alternatively if the source is "QRZ" and the callsign doesn't have a slash in it (i.e. operator likely at home).
           example: true
         de_call:
           type: string
@@ -975,4 +996,24 @@ components:
         contrast_color:
           type: string
           description: Black or white, whichever provides the best contrast against the band colour.
-          example: white
+          example: white
+
+    SIG:
+      type: object
+      properties:
+        name:
+          type: string
+          description: The abbreviated name of the SIG
+          example: POTA
+        description:
+          type: string
+          description: The full name of the SIG
+          example: Parks on the Air
+        icon:
+          type: string
+          description: Icon, from the Font Awesome set. This is fairly opinionated but is here to help the Spothole web UI and Field Spotter. Does not include the "fa-" prefix.
+          example: tree
+        ref_regex:
+          type: string
+          description: Regex that matches this SIG's reference IDs. Generally for Spothole's own internal use, clients probably won't need this.
+          example: "[A-Z]{2}\\-\\d+"

webassets/css/style.css

@@ -150,13 +150,108 @@ div#map {
 }
 
 
+/* BANDS PANEL */
+
+div#bands-container {
+    min-height: 64em;
+    margin: 0;
+    padding: 0;
+    overflow-x: auto;
+    white-space: nowrap;
+    overscroll-behavior-x: none;
+}
+
+#bands-table {
+    min-width: 100%;
+}
+
+#bands-table th {
+    width: 20%;
+    max-height: 40px;
+    min-width: 12em;
+    padding: 0.5em;
+    text-align: center;
+    font-weight: bold;
+}
+
+#bands-table td {
+    width: 20%;
+    min-width: 12em;
+    height: 62em;
+}
+
+div.band-container {
+    height: 62em;
+    width: 20%;
+    min-width: 12em;
+    position: relative;
+}
+
+div.band-markers {
+    width: 100%;
+    height: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    z-index: 13;
+    border-left: 2px dotted black;
+}
+
+div.band-spots {
+    width: 100%;
+    height: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    z-index: 15;
+}
+
+canvas.band-lines-canvas {
+    width: 5em;
+    height: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    z-index: 11;
+}
+
+div.band-spot {
+    position: absolute;
+    left: 5em;
+    padding: 0 0.25em;
+    background-color: white;
+    border-radius: 3px;
+    cursor: default;
+}
+
+div.band-spot:hover {
+    z-index: 999;
+}
+
+div.band-spot span.band-spot-call {
+    display: inline;
+}
+
+div.band-spot:hover span.band-spot-call {
+    display: none;
+}
+
+div.band-spot span.band-spot-info {
+    display: none;
+}
+
+div.band-spot:hover span.band-spot-info {
+    display: inline;
+}
+
+
 /* GENERAL MOBILE SUPPORT */
 
 @media (max-width: 991.99px) {
     .hideonmobile {
         display: none !important;
     }
-    div#map, div#table-container {
+    div#map, div#table-container, div#bands-container {
         margin-left: -1em;
         margin-right: -1em;
     }

webassets/js/alerts.js

@@ -124,16 +124,12 @@ function addAlertRowsToTable(tbody, alerts) {
         var showRef = $("#tableShowRef")[0].checked;
 
         // Get times for the alert, and convert to local time if necessary.
-        var start_time_unix = moment.unix(a["start_time"]);
+        var start_time_utc = moment.unix(a["start_time"]).utc();
-        var start_time = start_time_unix.utc();
+        var start_time_local = start_time_utc.clone().local();
-        if (useLocalTime) {
+        start_time = useLocalTime ? start_time_local : start_time_utc;
-            start_time = start_time.local();
+        var end_time_utc = moment.unix(a["end_time"]).utc();
-        }
+        var end_time_local = end_time_utc.clone().local();
-        var end_time_unix = moment.unix(a["end_time"]);
+        end_time = useLocalTime ? end_time_local : end_time_utc;
-        var end_time = end_time_unix.utc();
-        if (useLocalTime) {
-            end_time = end_time.local();
-        }
 
         // Format the times for display. Start time is displayed as e.g. 7 Oct 12:34 unless the time is in a
         // different year to the current year, in which case the year is inserted between month and hour.
@@ -143,8 +139,8 @@ function addAlertRowsToTable(tbody, alerts) {
         // Overriding all of that, if the start time is 00:00 and the end time is 23:59 when considered in UTC, the
         // hours and minutes are stripped out from the display, as we assume the server is just giving us full days.
         // Finally, if there is no end date set, "---" is displayed.
-        var whole_days = start_time_unix.utc().format("HH:mm") == "00:00" &&
+        var whole_days = start_time_utc.format("HH:mm") == "00:00" &&
-            (end_time_unix != null || end_time_unix > 0 || end_time_unix.utc().format("HH:mm") == "23:59");
+            (end_time_utc != null || end_time_utc > 0 || end_time_utc.format("HH:mm") == "23:59");
         var hours_minutes_format = whole_days ? "" : " HH:mm";
         var start_time_formatted = start_time.format("D MMM" + hours_minutes_format);
         if (start_time.format("YYYY") != moment().format("YYYY")) {
@@ -153,11 +149,13 @@ function addAlertRowsToTable(tbody, alerts) {
             start_time_formatted = start_time.format("[Today]" + hours_minutes_format);
         }
         var end_time_formatted = "---";
-        if (end_time_unix != null && end_time_unix > 0 && end_time != null) {
+        if (end_time_utc != null && end_time_utc > 0 && end_time != null) {
             var end_time_formatted = whole_days ? start_time_formatted : end_time.format("HH:mm");
             if (end_time.format("D MMM") != start_time.format("D MMM")) {
                 if (end_time.format("YYYY") != moment().format("YYYY")) {
                     end_time_formatted = end_time.format("D MMM YYYY" + hours_minutes_format);
+                } else if (useLocalTime && end_time.format("D MMM YYYY") == moment().format("D MMM YYYY")) {
+                    end_time_formatted = end_time.format("[Today]" + hours_minutes_format);
                 } else {
                     end_time_formatted = end_time.format("D MMM" + hours_minutes_format);
                 }

webassets/js/bands.js

@@ -0,0 +1,286 @@
+// A couple of constants that must match what's in CSS. We need to know them before the content actually renders, so we
+// can't just ask the elements themselves for their dimensions.
+BAND_COLUMN_HEIGHT_EM = 62;
+BAND_COLUMN_CANVAS_WIDTH_EM = 4;
+BAND_COLUMN_FONT_SIZE = 16;
+BAND_COLUMN_HEIGHT_PX = BAND_COLUMN_HEIGHT_EM * BAND_COLUMN_FONT_SIZE;
+BAND_COLUMN_CANVAS_WIDTH_PX = BAND_COLUMN_CANVAS_WIDTH_EM * BAND_COLUMN_FONT_SIZE;
+BAND_COLUMN_SPOT_DIV_HEIGHT_PX = BAND_COLUMN_FONT_SIZE * 1.6;
+
+// Load spots and populate the bands display.
+function loadSpots() {
+    $.getJSON('/api/v1/spots' + buildQueryString(), function(jsonData) {
+        // Store last updated time
+        lastUpdateTime = moment.utc();
+        updateRefreshDisplay();
+        // Store data
+        spots = jsonData;
+        // Update bands display
+        updateBands();
+    });
+}
+
+// Build a query string for the API, based on the filters that the user has selected.
+function buildQueryString() {
+    var str =  "?";
+    ["dx_continent", "de_continent", "mode_type", "source", "band"].forEach(fn => {
+        if (!allFilterOptionsSelected(fn)) {
+            str = str + getQueryStringFor(fn) + "&";
+        }
+    });
+    str = str + "max_age=" + $("#max-spot-age option:selected").val();
+    // Additional filters for the bands view: No dupes, no QRT
+    str = str + "&dedupe=true&allow_qrt=false";
+    return str;
+}
+
+// Update the bands display
+function updateBands() {
+    // Stop here if nothing to display
+    var bandsContainer = $("#bands-container");
+    if (spots.length === 0) {
+        bandsContainer.html("<div class='alert alert-danger' role='alert'>No spots match your filters.</div>");
+        return;
+    }
+
+    // Do some harsher de-duping. Because we only display callsign, frequency and mode here, the previous
+    // de-duplication could have let some through that don't look like dupes on the map, but would do here.
+    // Typically that's a person activating two programs at the same time, e.g. POTA & WWFF.
+    spotList = removeDuplicatesForBandPanel(spots);
+
+    // Convert to a map of band names to the spots on that band. Bands with no
+    // spots in view will not be present.
+    const bandToSpots = new Map();
+    options["bands"].forEach(function (band) {
+        const matchingSpots = spotList.filter(function (s) {
+            return s.band === band.name;
+        });
+        if (matchingSpots.length > 0) {
+            bandToSpots.set(band.name, matchingSpots);
+        }
+    });
+
+    // Track if any columns end up taller than expected, so we can resize the container and avoid vertical scroll.
+    var maxHeightBand = 0;
+
+    // Build up table content for each band
+    var table = $('<table id="bands-table">').append('<thead><tr></tr></thead><tbody><tr></tr></tbody>');
+    bandToSpots.forEach(function (spotList, bandName) {
+        // Get the colours for the band from the first spot, and prepare the header
+        table.find('thead tr').append(`<th style='background-color:${spotList[0].band_color}; color:${spotList[0].band_contrast_color}'>${spotList[0].band}</th>`);
+
+        // Get the band data to fetch start and end frequencies
+        let band = options["bands"].filter(function (b) {
+            return b.name === bandName;
+        })[0];
+
+        // Print the frequency band markers. This is 41 steps to divide the band evenly into 40 markers. One in every
+        // four will show the actual frequency, the others will just be dashes.
+        bandMarkersDiv = $('<div class="band-markers">');
+        const freqStep = (band.end_freq - band.start_freq) / 40.0;
+        for (let i = 0; i <= 40; i++) {
+            if (i % 4 === 0) {
+                bandMarkersDiv.append("&mdash;" + ((band.start_freq + i * freqStep)/1000000).toFixed(3) + "<br/>");
+            } else if (i % 4 === 2) {
+                bandMarkersDiv.append("&ndash;<br/>");
+            } else {
+                bandMarkersDiv.append("-<br/>");
+            }
+        }
+
+        // Prepare the spots list
+        var bandSpotsDiv = $("<div class='band-spots'>");
+        var lastSpotPxDownBand = -999;
+        // Sort by frequency so have a consistent order in which to plan where they will appear on the band div.
+        spotList.sort(function(a, b) { return a.freq - b.freq; });
+        // First calculate how we should be displaying the spots. There are three "modes" to try to place them in a
+        // visually appealing way:
+        // 1) Spaced normally, not going over the end of the band, so we populate them forwards.
+        // 2) Would go over the end, but the spots don't fill the band, so we populate them backwards.
+        // 3) Spots totally fill the band (or more), so we space them evenly starting at the top.
+        // In each case, we don't add anything to the DOM yet, we just calculate "pxDownBandLabel" (how far the *top* of
+        // the label is from the top of the div) and add that as a property to the spot for later use.
+        if (spotList.length >= BAND_COLUMN_HEIGHT_PX / BAND_COLUMN_SPOT_DIV_HEIGHT_PX) {
+            // Mode 3.
+            // Just lay out all spots simply, starting at 0px offset and working down with each one touching.
+            lastSpotPxDownBand = 0 - BAND_COLUMN_SPOT_DIV_HEIGHT_PX;
+            spotList.forEach(s => {
+                lastSpotPxDownBand = lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX;
+                s["pxDownBandLabel"] = lastSpotPxDownBand;
+            });
+        } else {
+            // Mode 1 or 2. Run through adding things to the list forwards as a test.
+            spotList.forEach(s => {
+                // Work out how far down the div to draw it
+                var percentDownBand = (s.freq - band.start_freq) / (band.end_freq - band.start_freq) * 0.97; // not 100% due to fudge, the first and last dashes are not exactly at the top and bottom of the div as some space is needed for text
+                var pxDownBand = percentDownBand * BAND_COLUMN_HEIGHT_PX;
+                if (pxDownBand < lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX) {
+                    pxDownBand = lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX; // Prevent overlap
+                }
+                s["pxDownBandLabel"] = pxDownBand;
+                lastSpotPxDownBand = pxDownBand;
+            });
+
+            // Work out if we overflowed the end.
+            if (lastSpotPxDownBand <= BAND_COLUMN_HEIGHT_PX) {
+                // Mode 1. Current positions are fine and there's nothing to do.
+            } else {
+                // Mode 2. Repeat the process but backwards, starting at the end and working upwards.
+                lastSpotPxDownBand = 999999;
+                spotList.reverse().forEach(s => {
+                    // Work out how far down the div to draw it
+                    var percentDownBand = (s.freq - band.start_freq) / (band.end_freq - band.start_freq) * 0.97; // not 100% due to fudge, the first and last dashes are not exactly at the top and bottom of the div as some space is needed for text
+                    var pxDownBand = percentDownBand * BAND_COLUMN_HEIGHT_PX;
+                    if (pxDownBand > lastSpotPxDownBand - BAND_COLUMN_SPOT_DIV_HEIGHT_PX) {
+                        pxDownBand = lastSpotPxDownBand - BAND_COLUMN_SPOT_DIV_HEIGHT_PX; // Prevent overlap
+                    }
+                    s["pxDownBandLabel"] = pxDownBand;
+                    lastSpotPxDownBand = pxDownBand;
+                });
+            }
+        }
+
+        // Now each spot is tagged with how far down the div it should go, add them to the DOM.
+        spotList.forEach(s => {
+            bandSpotsDiv.append(`<div class="band-spot" style="top: ${s['pxDownBandLabel']}px; border-top: 1px solid ${s.band_color}; border-left: 5px solid ${s.band_color}; border-bottom: 1px solid ${s.band_color}; border-right: 1px solid ${s.band_color};"><span class="band-spot-call">${s.dx_call}</span><span class="band-spot-info">${s.dx_call} ${(s.freq/1000000).toFixed(3)} ${s.mode}</span></div>`);
+        });
+
+        // Work out how tall the canvas should be. Normally this is matching the normal band column height, but if some
+        // spots have gone off the end of the band markers and stretched their div, we need to resize the canvas to
+        // match, otherwise we have nowhere to draw their connecting lines.
+        var canvasHeight = Math.max(BAND_COLUMN_HEIGHT_PX, lastSpotPxDownBand + BAND_COLUMN_SPOT_DIV_HEIGHT_PX);
+        maxHeightBand = Math.max(maxHeightBand, canvasHeight);
+
+        // Draw horizontal or diagonal lines to join up the "real" frequency with where the spot div ended up
+        var bandLinesCanvas = $(`<canvas class='band-lines-canvas' width='${BAND_COLUMN_CANVAS_WIDTH_PX}px' height='${canvasHeight}px' style='height:${canvasHeight}px !important;'>`);
+        spotList.forEach(s => {
+            // Work out how far down the div to draw it
+            var percentDownBand = (s.freq - band.start_freq) / (band.end_freq - band.start_freq) * 0.97; // not 100% due to fudge, the first and last dashes are not exactly at the top and bottom of the div as some space is needed for text
+            var pxDownBandFreq = (percentDownBand + 0.015) * BAND_COLUMN_HEIGHT_PX; // same fudge but add half to put the left end of the line in the right place
+            var pxDownBandLabel = s["pxDownBandLabel"] + (BAND_COLUMN_SPOT_DIV_HEIGHT_PX / 1.75); // line should be to the vertical text-centre spot, not to the top corner
+
+            // Draw the line on the canvas
+            var ctx = bandLinesCanvas[0].getContext('2d');
+            ctx.beginPath();
+            ctx.lineWidth = 2;
+            ctx.lineCap = "round";
+            ctx.strokeStyle = s.band_color;
+            ctx.moveTo(0, pxDownBandFreq);
+            ctx.lineTo(BAND_COLUMN_CANVAS_WIDTH_PX, pxDownBandLabel);
+            ctx.stroke();
+        });
+
+        // Assemble the table cell
+        td = $("<td>");
+        container = $("<div class='band-container'>");
+        container.append(bandLinesCanvas);
+        container.append(bandMarkersDiv);
+        container.append(bandSpotsDiv);
+        td.append(container);
+        table.find('tbody tr').append(td);
+    });
+
+    // Update the DOM with the band HTML
+    bandsContainer.html(table);
+
+    // Increase the height of the bands container so we don't have any vertical scroll bars except the browser ones
+    bandsContainer.css("min-height", `${maxHeightBand + 42}px`);
+
+    // Desktop mouse wheel to scroll bands horizontally if used on the headers
+    table.find('thead tr').on("wheel", () => {
+        bandsContainer.scrollLeft(bandsContainer.scrollLeft() + event.deltaY / 10.0);
+        return false;
+    });
+}
+
+// Iterate through a temporary list of spots, merging duplicates in a way suitable for the band panel. If two or more
+// spots with the activator, mode and frequency are found, these will be merged and reduced until only one remains,
+// with the best data. Note that unlike removeDuplicates(), which operates on the main spot map, this operates only
+// on the temporary array of spots provided as an argument, and returns the output, for use when constructing the
+// band panel.
+function removeDuplicatesForBandPanel(spotList) {
+    const spotsToRemove = [];
+    spotList.forEach(function (check) {
+        spotList.forEach(function (s) {
+            if (s !== check) {
+                if (s.dx_call === check.dx_call && s.freq === check.freq && s.mode === check.mode) {
+                    // Find which one to keep and which to delete
+                    const checkSpotNewer = check.time > s.time;
+                    const keepSpot = checkSpotNewer ? check : s;
+                    const deleteSpot = checkSpotNewer ? s : check;
+                    // Aggregate list of spots to remove
+                    spotsToRemove.push(deleteSpot.uid);
+                }
+            }
+        });
+    });
+    // Perform the removal
+    return spotList.filter(s => !spotsToRemove.includes(s.uid));
+}
+
+// Load server options. Once a successful callback is made from this, we then query spots and set up the timer to query
+// spots repeatedly.
+function loadOptions() {
+    $.getJSON('/api/v1/options', function(jsonData) {
+        // Store options
+        options = jsonData;
+
+        // Add CSS for band toggle buttons
+        addBandToggleColourCSS(options["bands"]);
+
+        // Populate the filters panel
+        generateBandsMultiToggleFilterCard(options["bands"]);
+        generateMultiToggleFilterCard("#dx-continent-options", "dx_continent", options["continents"]);
+        generateMultiToggleFilterCard("#de-continent-options", "de_continent", options["continents"]);
+        generateMultiToggleFilterCard("#mode-options", "mode_type", options["mode_types"]);
+        generateMultiToggleFilterCard("#source-options", "source", options["spot_sources"]);
+
+        // Load settings from settings storage now all the controls are available
+        loadSettings();
+
+        // Load spots and set up the timer
+        loadSpots();
+        setInterval(loadSpots, REFRESH_INTERVAL_SEC * 1000);
+    });
+}
+
+// Method called when any display property is changed to reload the map and persist the display settings.
+function displayUpdated() {
+    updateMap();
+    saveSettings();
+}
+
+// React to toggling/closing panels
+function toggleFiltersPanel() {
+    // If we are going to show the filters panel, hide the display panel
+    if (!$("#filters-area").is(":visible") && $("#display-area").is(":visible")) {
+        $("#display-area").hide();
+        $("#display-button").button("toggle");
+    }
+    $("#filters-area").toggle();
+}
+function closeFiltersPanel() {
+    $("#filters-button").button("toggle");
+    $("#filters-area").hide();
+}
+
+function toggleDisplayPanel() {
+    // If we are going to show the display panel, hide the filters panel
+    if (!$("#display-area").is(":visible") && $("#filters-area").is(":visible")) {
+        $("#filters-area").hide();
+        $("#filters-button").button("toggle");
+    }
+    $("#display-area").toggle();
+}
+function closeDisplayPanel() {
+    $("#display-button").button("toggle");
+    $("#display-area").hide();
+}
+
+// Startup
+$(document).ready(function() {
+    // Call loadOptions(), this will then trigger loading spots and setting up timers.
+    loadOptions();
+    // Update the refresh timing display every second
+    setInterval(updateRefreshDisplay, 1000);
+});

webassets/js/map.js

@@ -3,7 +3,7 @@ var markersLayer;
 var geodesicsLayer;
 var terminator;
 
-// Load spots and populate the table.
+// Load spots and populate the map.
 function loadSpots() {
     $.getJSON('/api/v1/spots' + buildQueryString(), function(jsonData) {
         // Store data
@@ -23,6 +23,8 @@ function buildQueryString() {
         }
     });
     str = str + "max_age=" + $("#max-spot-age option:selected").val();
+    // Additional filters for the map view: No dupes, no QRT, only spots with good locations
+    str = str + "&dedupe=true&allow_qrt=false&needs_good_location=true";
     return str;
 }
 
@@ -32,29 +34,20 @@ function updateMap() {
     markersLayer.clearLayers();
     geodesicsLayer.clearLayers();
 
-    // Make new markers for all spots with a good location, not QRT, and not a duplicate spot within the data set.
+    // Make new markers for all spots that match the filter
-    var callsAlreadyDisplayed = [];
     spots.forEach(function (s) {
-        if (s["dx_location_good"] && (s["qrt"] == null || s["qrt"] == false)) {
+        var m = L.marker([s["dx_latitude"], s["dx_longitude"]], {icon: getIcon(s)});
-            if (!callsAlreadyDisplayed.includes(s["dx_call"])) {
+        m.bindPopup(getTooltipText(s));
+        markersLayer.addLayer(m);
 
-                // OK, create the marker
+        // Create geodesics if required
-                var m = L.marker([s["dx_latitude"], s["dx_longitude"]], {icon: getIcon(s)});
+        if ($("#mapShowGeodesics")[0].checked && s["de_latitude"] != null && s["de_longitude"] != null) {
-                m.bindPopup(getTooltipText(s));
+            var geodesic = L.geodesic([[s["de_latitude"], s["de_longitude"]], m.getLatLng()], {
-                markersLayer.addLayer(m);
+                color: s["band_color"],
-
+                wrap: false,
-                // Create geodesics if required
+                steps: 5
-                if ($("#mapShowGeodesics")[0].checked && s["de_latitude"] != null && s["de_longitude"] != null) {
+            });
-                    var geodesic = L.geodesic([[s["de_latitude"], s["de_longitude"]], m.getLatLng()], {
+            geodesicsLayer.addLayer(geodesic);
-                        color: s["band_color"],
-                        wrap: false,
-                        steps: 5
-                    });
-                    geodesicsLayer.addLayer(geodesic);
-                }
-
-            }
-            callsAlreadyDisplayed.push(s["dx_call"]);
         }
     });
 }

webassets/js/spots.js

@@ -88,7 +88,7 @@ function updateTable() {
         // Format a UTC or local time for display
         var time = moment.unix(s["time"]).utc();
         if (useLocalTime) {
-            time = time.local();
+            time.local();
         }
         var time_formatted = time.format("HH:mm");
 
@@ -148,22 +148,16 @@ function updateTable() {
 
         // Format sig_refs
         var sig_refs = "";
-        if (s["sig_refs"] && s["sig_refs_urls"] && s["sig_refs"].length == s["sig_refs_urls"].length) {
+        if (s["sig_refs"] && s["sig_refs_urls"] && s["sig_refs"].length == s["sig_refs_urls"].length && s["sig_refs"].length == s["sig_refs_names"].length) {
             items = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`)
             for (var i = 0; i < items.length; i++) {
-                items[i] = `<a href='${s["sig_refs_urls"][i]}' target='_new' class='sig-ref-link'>${items[i]}</a>`
+                items[i] = `<a href='${s["sig_refs_urls"][i]}' title='${s["sig_refs_names"][i]}' target='_new' class='sig-ref-link'>${items[i]}</a>`
             }
             sig_refs = items.join(", ");
         } else if (s["sig_refs"]) {
             sig_refs = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`).join(", ");
         }
 
-        // Format sig_refs title
-        var sig_refs_title_string = "";
-        if (s["sig_refs_names"]) {
-            sig_refs_title_string = " title=\"" + s["sig_refs_names"].join(", ") + "\"";
-        }
-
         // Format DE flag
         var de_flag = "<i class='fa-solid fa-circle-question'></i>";
         if (s["de_flag"] && s["de_flag"] != null && s["de_flag"] != "") {
@@ -191,7 +185,7 @@ function updateTable() {
             $tr.append(`<td class='nowrap'>${time_formatted}</td>`);
         }
         if (showDX) {
-            $tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${dx_flag}</span><a class='dx-link' href='https://qrz.com/db/${s["dx_call"]}' target='_new'>${s["dx_call"]}</a></td>`);
+            $tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${dx_flag}</span><a class='dx-link' href='https://qrz.com/db/${s["dx_call"]}' target='_new' title='${s["dx_name"] != null ? s["dx_name"] : ""}'>${s["dx_call"]}</a></td>`);
         }
         if (showFreq) {
             $tr.append(`<td class='nowrap'><span class='band-bullet' title='${bandFullName}' style='color: ${s["band_color"]}'>&#9632;</span>${freq_string}</td>`);
@@ -209,7 +203,7 @@ function updateTable() {
             $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${typeText}</td>`);
         }
         if (showRef) {
-            $tr.append(`<td class='hideonmobile'><span ${sig_refs_title_string}>${sig_refs}</span></td>`);
+            $tr.append(`<td class='hideonmobile'>${sig_refs}</td>`);
         }
         if (showDE) {
             $tr.append(`<td class='nowrap hideonmobile'><span class='flag-wrapper' title='${de_country}'>${de_flag}</span>${de_call}</td>`);