Remove duplicate SIG

core/constants.py

@@ -9,7 +9,7 @@ SOFTWARE_VERSION = "0.1"
 HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + " " + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
 
 # Special Interest Groups
-SIGS = ["POTA", "SOTA", "WWFF", "GMA", "WWBOTA", "HEMA", "MOTA", "ARLHS", "SiOTA", "WCA"]
+SIGS = ["POTA", "SOTA", "WWFF", "GMA", "WWBOTA", "HEMA", "MOTA", "ARLHS", "ILLW", "SiOTA", "WCA", "ZLOTA", "IOTA"]
 
 # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
 CW_MODES = ["CW"]

data/spot.py

@@ -2,6 +2,7 @@ import copy
 import hashlib
 import json
 import logging
+import re
 from dataclasses import dataclass
 from datetime import datetime
 
@@ -102,6 +103,8 @@ class Spot:
     sig_refs: list = None
     # SIG reference names
     sig_refs_names: list = None
+    # SIG reference URLs
+    sig_refs_urls: list = None
     # Activation score. SOTA only
     activation_score: int = None
 
@@ -232,6 +235,13 @@ class Spot:
         if self.comment and not self.qrt:
             self.qrt = "QRT" in self.comment.upper()
 
+        # Clean up comments
+        if self.comment:
+            comment = re.sub(r"\[.*]:", "", self.comment)
+            comment = re.sub(r"\[.*]", "", comment)
+            comment = re.sub(r"\"\"", "", comment)
+            self.comment = comment.strip()
+
         # DX operator details lookup, using QRZ.com. This should be the last resort compared to taking the data from
         # the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of
         # the one from the park reference they're at.

server/webserver.py

@@ -137,6 +137,9 @@ class WebServer:
         # in seconds UTC.
         # We can also filter by source, sig, band, mode, dx_continent and de_continent. Each of these accepts a single
         # value or a comma-separated list.
+        # We can filter by comments, accepting a single string, where the API will only return spots where the comment
+        # contains the provided value (case-insensitive).
+        # We can "de-dupe" spots, so only the latest spot will be sent for each callsign.
         # We can provide a "limit" number as well. Spots are always returned newest-first; "limit" limits to only the
         # most recent X spots.
         spot_ids = list(self.spots.iterkeys())
@@ -162,8 +165,14 @@ class WebServer:
                     sources = query.get(k).split(",")
                     spots = [s for s in spots if s.source and s.source in sources]
                 case "sig":
+                    # If a list of sigs is provided, the spot must have a sig and it must match one of them
                     sigs = query.get(k).split(",")
                     spots = [s for s in spots if s.sig and s.sig in sigs]
+                case "needs_sig":
+                    # If true, a sig is required, regardless of what it is, it just can't be missing.
+                    needs_sig = query.get(k).upper() == "TRUE"
+                    if needs_sig:
+                        spots = [s for s in spots if s.sig]
                 case "band":
                     bands = query.get(k).split(",")
                     spots = [s for s in spots if s.band and s.band in bands]
@@ -179,6 +188,22 @@ class WebServer:
                 case "de_continent":
                     deconts = query.get(k).split(",")
                     spots = [s for s in spots if s.de_continent and s.de_continent in deconts]
+                case "comment_includes":
+                    comment_includes = query.get(k).strip()
+                    spots = [s for s in spots if s.comment and comment_includes.upper() in s.comment.upper()]
+                case "dedupe":
+                    # Ensure only the latest spot of each callsign is present in the list. This relies on the list being
+                    # in reverse time order, so if any future change allows re-ordering the list, that should be done
+                    # *after* this.
+                    dedupe = query.get(k).upper() == "TRUE"
+                    if dedupe:
+                        spots_temp = []
+                        already_seen = []
+                        for s in spots:
+                            if s.dx_call not in already_seen:
+                                spots_temp.append(s)
+                                already_seen.append(s.dx_call)
+                        spots = spots_temp
         # If we have a "limit" parameter, we apply that last, regardless of where it appeared in the list of keys.
         if "limit" in query.keys():
             spots = spots[:int(query.get("limit"))]

spotproviders/gma.py

@@ -36,6 +36,7 @@ class GMA(HTTPSpotProvider):
                         comment=source_spot["TEXT"],
                         sig_refs=[source_spot["REF"]],
                         sig_refs_names=[source_spot["NAME"]],
+                        sig_refs_urls=["https://www.cqgma.org/zinfo.php?ref=" + source_spot["REF"]],
                         time=datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
                             tzinfo=pytz.UTC).timestamp(),
                         dx_latitude=float(source_spot["LAT"]) if (source_spot["LAT"] and source_spot["LAT"] != "") else None,

spotproviders/pota.py

@@ -29,6 +29,7 @@ class POTA(HTTPSpotProvider):
                         sig="POTA",
                         sig_refs=[source_spot["reference"]],
                         sig_refs_names=[source_spot["name"]],
+                        sig_refs_urls=["https://pota.app/#/park/" + source_spot["reference"]],
                         icon="tree",
                         time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
                         dx_grid=source_spot["grid6"],

spotproviders/sota.py

@@ -50,6 +50,7 @@ class SOTA(HTTPSpotProvider):
                             sig="SOTA",
                             sig_refs=[source_spot["summitCode"]],
                             sig_refs_names=[source_spot["summitName"]],
+                            sig_refs_urls=["https://www.sotadata.org.uk/en/summit/" + source_spot["summitCode"]],
                             icon="mountain-sun",
                             time=datetime.fromisoformat(source_spot["timeStamp"]).timestamp(),
                             activation_score=source_spot["points"])

spotproviders/wwbota.py

@@ -18,9 +18,16 @@ class WWBOTA(SSESpotProvider):
         # n-fer activations.
         refs = []
         ref_names = []
+        ref_urls = []
         for ref in source_spot["references"]:
             refs.append(ref["reference"])
             ref_names.append(ref["name"])
+            # Bunkerbase URLs only work for UK bunkers, so only add a URL if we have a B/G prefix. In theory this could
+            # lead to array alignment mismatches if there was e.g. a B/F bunker followed by a B/G one, we'd end up with
+            # the B/G URL in index 0. But in practice there are no overlaps between B/G bunkers and any others, so an
+            # activation will either be entirely B/G or not B/G at all.
+            if ref["reference"].startswith("B/G"):
+                ref_urls.append("https://bunkerwiki.org/?s=" + ref["reference"])
 
         spot = Spot(source=self.name,
                     dx_call=source_spot["call"].upper(),

spotproviders/wwff.py

@@ -29,6 +29,7 @@ class WWFF(HTTPSpotProvider):
                         sig="WWFF",
                         sig_refs=[source_spot["reference"]],
                         sig_refs_names=[source_spot["reference_name"]],
+                        sig_refs_urls=["https://wwff.co/directory/?showRef=" + source_spot["reference"]],
                         icon="seedling",
                         time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
                         dx_latitude=source_spot["latitude"],

views/webpage_base.tpl

@@ -57,12 +57,12 @@
             </button>
             <div class="collapse navbar-collapse" id="navbarTogglerDemo02">
                 <ul class="navbar-nav me-auto mb-2 mb-lg-0">
-                    <li class="nav-item ms-4"><a href="/" class="nav-link" id="nav-link-spots">Spots</a></li>
-                    <li class="nav-item ms-4"><a href="/map" class="nav-link" id="nav-link-map">Map</a></li>
-                    <li class="nav-item ms-4"><a href="/alerts" class="nav-link" id="nav-link-alerts">Alerts</a></li>
-                    <li class="nav-item ms-4"><a href="/status" class="nav-link" id="nav-link-status">Status</a></li>
-                    <li class="nav-item ms-4"><a href="/about" class="nav-link" id="nav-link-about">About</a></li>
-                    <li class="nav-item ms-4"><a href="/apidocs" class="nav-link" id="nav-link-api">API</a></li>
+                    <li class="nav-item ms-4"><a href="/" class="nav-link" id="nav-link-spots"><i class="fa-solid fa-tower-cell"></i> Spots</a></li>
+                    <li class="nav-item ms-4"><a href="/map" class="nav-link" id="nav-link-map"><i class="fa-solid fa-map"></i> Map</a></li>
+                    <li class="nav-item ms-4"><a href="/alerts" class="nav-link" id="nav-link-alerts"><i class="fa-solid fa-bell"></i> Alerts</a></li>
+                    <li class="nav-item ms-4"><a href="/status" class="nav-link" id="nav-link-status"><i class="fa-solid fa-chart-simple"></i> Status</a></li>
+                    <li class="nav-item ms-4"><a href="/about" class="nav-link" id="nav-link-about"><i class="fa-solid fa-circle-info"></i> About</a></li>
+                    <li class="nav-item ms-4"><a href="/apidocs" class="nav-link" id="nav-link-api"><i class="fa-solid fa-gear"></i> API</a></li>
                 </ul>
             </div>
         </div>

views/webpage_spots.tpl

@@ -154,8 +154,8 @@
                                   <label class="form-check-label" for="tableShowBearing">Bearing</label>
                                 </div>
                                 <div class="form-check form-check-inline">
-                                  <input class="form-check-input storeable-checkbox" type="checkbox" id="tableShowSource" value="tableShowSource" oninput="columnsUpdated();" checked>
-                                  <label class="form-check-label" for="tableShowSource">Source</label>
+                                  <input class="form-check-input storeable-checkbox" type="checkbox" id="tableShowType" value="tableShowType" oninput="columnsUpdated();" checked>
+                                  <label class="form-check-label" for="tableShowType">Type</label>
                                 </div>
                                 <div class="form-check form-check-inline">
                                   <input class="form-check-input storeable-checkbox" type="checkbox" id="tableShowRef" value="tableShowRef" oninput="columnsUpdated();" checked>

webassets/apidocs/openapi.yml

@@ -65,7 +65,7 @@ paths:
               - APRS-IS
         - name: sig
           in: query
-          description: "Limit the spots to only ones from one or more Special Interest Groups. To select more than one SIG, supply a comma-separated list."
+          description: "Limit the spots to only ones from one or more Special Interest Groups provided as an argument. To select more than one SIG, supply a comma-separated list."
           required: false
           schema:
             type: string
@@ -76,6 +76,20 @@ paths:
               - WWBOTA
               - GMA
               - HEMA
+              - WCA
+              - MOTA
+              - SiOTA
+              - ARLHS
+              - ILLW
+              - ZLOTA
+              - IOTA
+        - name: needs_sig
+          in: query
+          description: "Limit the spots to only ones from a Special Interest Grous such as POTA. Because supplying all known SIGs as a `sigs` parameter is unwieldy, and leaving `sigs` blank will also return spots with *no* SIG, this parameter can be set true to return only spots with a SIG, regardless of what it is, so long as it's not blank. This is what Field Spotter uses to exclude generic cluster spots and only retrieve xOTA things."
+          required: false
+          schema:
+            type: boolean
+            default: false
         - name: band
           in: query
           description: "Limit the spots to only ones from one or more bands. To select more than one band, supply a comma-separated list."
@@ -168,6 +182,19 @@ paths:
               - AF
               - OC
               - AN
+        - name: dedupe
+          in: query
+          description: "\"De-duplicate\" the spots, returning only the latest spot for any given callsign."
+          required: false
+          schema:
+            type: boolean
+            default: false
+        - name: comment_includes
+          in: query
+          description: "Return only spots where the comment includes the provided string (case-insensitive)."
+          required: false
+          schema:
+            type: string
       responses:
         '200':
           description: Success
@@ -241,6 +268,13 @@ paths:
               - WWBOTA
               - GMA
               - HEMA
+              - WCA
+              - MOTA
+              - SiOTA
+              - ARLHS
+              - ILLW
+              - ZLOTA
+              - IOTA
         - name: dx_continent
           in: query
           description: "Limit the alerts to only ones where the DX (the operator being spotted) is on the given continent(s). To select more than one continent, supply a comma-separated list."
@@ -667,6 +701,13 @@ components:
             - WWBOTA
             - GMA
             - HEMA
+            - WCA
+            - MOTA
+            - SiOTA
+            - ARLHS
+            - ILLW
+            - ZLOTA
+            - IOTA
           example: POTA
         sig_refs:
           type: array
@@ -680,6 +721,12 @@ components:
             type: string
           description: SIG reference names
           example: Null Country Park
+        sig_refs_urls:
+          type: array
+          items:
+            type: string
+          description: SIG reference URLs, which the user can look up for more information
+          example: "https://pota.app/#/park/GB-0001"
         activation_score:
           type: integer
           description: Activation score. SOTA only
@@ -688,6 +735,14 @@ components:
           type: string
           descripton: Icon, from the Font Awesome set. This is fairly opinionated but is here to help the Spothole web UI and Field Spotter. Does not include the "fa-" prefix.
           example: tree
+        band_color:
+          type: string
+          descripton: Colour to represent this spot, if a client chooses to colour spots based on their frequency band, using PSK Reporter's default colours. HTML colour e.g. hex.
+          example: #ff0000"
+        band_contrast_color:
+          type: string
+          descripton: Black or white, whichever best contrasts with "band_color".
+          example: "white"
         qrt:
           type: boolean
           description: QRT state. Some APIs return spots marked as QRT. Otherwise we can check the comments.
@@ -806,6 +861,13 @@ components:
             - WWBOTA
             - GMA
             - HEMA
+            - WCA
+            - MOTA
+            - SiOTA
+            - ARLHS
+            - ILLW
+            - ZLOTA
+            - IOTA
           example: POTA
         sig_refs:
           type: array
@@ -827,14 +889,6 @@ components:
           type: string
           descripton: Icon, from the Font Awesome set. This is fairly opinionated but is here to help the Spothole web UI and Field Spotter. Does not include the "fa-" prefix.
           example: tree
-        band_color:
-          type: string
-          descripton: Colour to represent this spot, if a client chooses to colour spots based on their frequency band, using PSK Reporter's default colours. HTML colour e.g. hex.
-          example: #ff0000"
-        band_contrast_color:
-          type: string
-          descripton: Black or white, whichever best contrasts with "band_color".
-          example: "white"
         source:
           type: string
           description: Where we got the alert from.

webassets/css/style.css

@@ -92,10 +92,13 @@ span.icon-wrapper {
 }
 
 span.freq-mhz {
+    font-weight: bold;
+}
+
+span.freq-mhz-pad {
     display: inline-block;
     min-width: 1.7em;
     text-align: right;
-    font-weight: bold;
 }
 
 span.freq-khz {
@@ -117,6 +120,10 @@ a.dx-link {
     text-decoration: none;
     font-weight: bold;
 }
+a.sig-ref-link {
+    color: var(--bs-emphasis-color);
+    text-decoration: none;
+}
 
 /* QRT/faded styles */
 tr.table-faded td {
@@ -142,12 +149,6 @@ div#map {
     font-family: var(--bs-body-font-family) !important;
 }
 
-a.leaflet-popup-callsign-link {
-    color: black;
-    font-weight: bold;
-    text-decoration: none;
-}
-
 
 /* GENERAL MOBILE SUPPORT */
 
@@ -155,6 +156,10 @@ a.leaflet-popup-callsign-link {
     .hideonmobile {
         display: none !important;
     }
+    div#map, div#table-container {
+        margin-left: -1em;
+        margin-right: -1em;
+    }
 }
 
 @media (min-width: 992px) {

webassets/js/map.js

@@ -100,7 +100,13 @@ function getTooltipText(s) {
 
     // Format sig_refs
     var sig_refs = "";
-    if (s["sig_refs"]) {
+    if (s["sig_refs"] && s["sig_refs_urls"] && s["sig_refs"].length == s["sig_refs_urls"].length) {
+        items = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`)
+        for (var i = 0; i < items.length; i++) {
+            items[i] = `<a href='${s["sig_refs_urls"][i]}' target='_new' class='sig-ref-link'>${items[i]}</a>`
+        }
+        sig_refs = items.join(", ");
+    } else if (s["sig_refs"]) {
         sig_refs = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`).join(", ");
     }
 
@@ -108,10 +114,13 @@ function getTooltipText(s) {
     const shortCall = s["dx_call"].split("/").sort(function (a, b) {
         return b.length - a.length;
     })[0];
-    ttt = `<span class='nowrap'>${dx_flag} <a href='https://www.qrz.com/db/${shortCall}' target='_blank' class="leaflet-popup-callsign-link">${s["dx_call"]}</a></span><br/>`;
+    ttt = `<span class='nowrap'><span class='icon-wrapper'>${dx_flag}</span> <a href='https://www.qrz.com/db/${shortCall}' target='_blank' class="dx-link">${s["dx_call"]}</a></span><br/>`;
 
     // Frequency & band
-    ttt += `<i class='fa-solid fa-walkie-talkie markerPopupIcon'></i>&nbsp;${freq_string} (${s["band"]})`;
+    ttt += `<span class='icon-wrapper'><i class='fa-solid fa-radio markerPopupIcon'></i></span>&nbsp;${freq_string}`;
+    if (s["band"] != null) {
+        ttt += ` (${s["band"]})`;
+    }
     // Mode
     if (s["mode"] != null) {
         ttt += ` &nbsp;&nbsp; <i class='fa-solid fa-wave-square markerPopupIcon'></i>&nbsp;${s["mode"]}`;
@@ -119,14 +128,14 @@ function getTooltipText(s) {
     ttt += "<br/>";
 
     // Source / SIG / Ref
-    ttt += `<span class='nowrap'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i>&nbsp;${sigSourceText} ${sig_refs}</span><br/>`;
+    ttt += `<span class='nowrap'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span>&nbsp;${sigSourceText} ${sig_refs}</span><br/>`;
 
     // Time
-    ttt += `<i class='fa-solid fa-clock markerPopupIcon'></i>&nbsp;${moment.unix(s["time"]).fromNow()}`;
+    ttt += `<span class='icon-wrapper'><i class='fa-solid fa-clock markerPopupIcon'></i></span>&nbsp;${moment.unix(s["time"]).fromNow()}`;
 
     // Comment
     if (commentText.length > 0) {
-        ttt += `<br/><i class='fa-solid fa-comment markerPopupIcon'></i> ${commentText}`;
+        ttt += `<br/><span class='icon-wrapper'><i class='fa-solid fa-comment markerPopupIcon'></i></span> ${commentText}`;
     }
 
     return ttt;

webassets/js/spots.js

@@ -38,7 +38,7 @@ function updateTable() {
     var showMode = $("#tableShowMode")[0].checked;
     var showComment = $("#tableShowComment")[0].checked;
     var showBearing = $("#tableShowBearing")[0].checked && userPos != null;
-    var showSource = $("#tableShowSource")[0].checked;
+    var showType = $("#tableShowType")[0].checked;
     var showRef = $("#tableShowRef")[0].checked;
     var showDE = $("#tableShowDE")[0].checked;
 
@@ -62,8 +62,8 @@ function updateTable() {
     if (showBearing) {
         table.find('thead tr').append(`<th class='hideonmobile'>Bearing</th>`);
     }
-    if (showSource) {
-        table.find('thead tr').append(`<th class='hideonmobile'>Source</th>`);
+    if (showType) {
+        table.find('thead tr').append(`<th class='hideonmobile'>Type</th>`);
     }
     if (showRef) {
         table.find('thead tr').append(`<th class='hideonmobile'>Ref.</th>`);
@@ -109,7 +109,7 @@ function updateTable() {
         var khz = Math.floor((s["freq"] - (mhz * 1000000.0)) / 1000.0);
         var hz = Math.floor(s["freq"] - (mhz * 1000000.0) - (khz * 1000.0));
         var hz_string = (hz > 0) ? hz.toFixed(0)[0] : "";
-        var freq_string = `<span class='freq-mhz'>${mhz.toFixed(0)}</span><span class='freq-khz'>${khz.toFixed(0).padStart(3, '0')}</span><span class='freq-hz hideonmobile'>${hz_string}</span>`
+        var freq_string = `<span class='freq-mhz freq-mhz-pad'>${mhz.toFixed(0)}</span><span class='freq-khz'>${khz.toFixed(0).padStart(3, '0')}</span><span class='freq-hz hideonmobile'>${hz_string}</span>`
 
         // Format the mode
         mode_string = s["mode"];
@@ -140,15 +140,21 @@ function updateTable() {
             }
         }
 
-        // Sig or fallback to source
-        var sigSourceText = s["source"];
+        // Format "type" (Sig or fallback to source)
+        var typeText = s["source"];
         if (s["sig"]) {
-            sigSourceText = s["sig"];
+            typeText = s["sig"];
         }
 
         // Format sig_refs
         var sig_refs = "";
-        if (s["sig_refs"]) {
+        if (s["sig_refs"] && s["sig_refs_urls"] && s["sig_refs"].length == s["sig_refs_urls"].length) {
+            items = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`)
+            for (var i = 0; i < items.length; i++) {
+                items[i] = `<a href='${s["sig_refs_urls"][i]}' target='_new' class='sig-ref-link'>${items[i]}</a>`
+            }
+            sig_refs = items.join(", ");
+        } else if (s["sig_refs"]) {
             sig_refs = s["sig_refs"].map(s => `<span class='nowrap'>${s}</span>`).join(", ");
         }
 
@@ -199,8 +205,8 @@ function updateTable() {
         if (showBearing) {
             $tr.append(`<td class='nowrap hideonmobile'>${bearingText}</td>`);
         }
-        if (showSource) {
-            $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${sigSourceText}</td>`);
+        if (showType) {
+            $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${typeText}</td>`);
         }
         if (showRef) {
             $tr.append(`<td class='hideonmobile'><span ${sig_refs_title_string}>${sig_refs}</span></td>`);
@@ -210,14 +216,14 @@ function updateTable() {
         }
         table.find('tbody').append($tr);
 
-        // Second row for mobile view only, containing source, ref & comment
+        // Second row for mobile view only, containing type, ref & comment
         $tr2 = $("<tr class='hidenotonmobile'>");
         if (s["qrt"] == true) {
           $tr2.addClass("table-faded");
         }
         $td2 = $("<td colspan='100'>");
-        if (showSource) {
-            $td2.append(`<span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${sigSourceText} `);
+        if (showType) {
+            $td2.append(`<span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${typeText} `);
         }
         if (showRef) {
             $td2.append(`${sig_refs} `);