Support Packet (PKT) as a digi mode

alertproviders/wota.py

@@ -1,4 +1,3 @@
-import re
 from datetime import datetime
 
 import pytz
@@ -23,6 +22,11 @@ class WOTA(HTTPAlertProvider):
         rss = RSSParser.parse(http_response.content.decode())
         # Iterate through source data
         for source_alert in rss.channel.items:
+
+            # Reject GUID missing or zero
+            if not source_alert.guid or not source_alert.guid.content or source_alert.guid.content == "http://www.wota.org.uk/alerts/0":
+                continue
+
             # Pick apart the title
             title_split = source_alert.title.split(" on ")
             dx_call = title_split[0]

config-example.yml

@@ -41,6 +41,14 @@ spot-providers:
     class: "ParksNPeaks"
     name: "ParksNPeaks"
     enabled: true
+  -
+    class: "ZLOTA"
+    name: "ZLOTA"
+    enabled: true
+  -
+    class: "WOTA"
+    name: "WOTA"
+    enabled: true
   -
     class: "APRSIS"
     name: "APRS-IS"
@@ -88,6 +96,10 @@ alert-providers:
     class: "ParksNPeaks"
     name: "ParksNPeaks"
     enabled: true
+  -
+    class: "WOTA"
+    name: "WOTA"
+    enabled: true
   -
     class: "NG3K"
     name: "NG3K"

core/constants.py

@@ -33,7 +33,7 @@ SIGS = [
 # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
 CW_MODES = ["CW"]
 PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "M17"]
-DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "BPSK", "PSK", "PSK31", "BPSK31", "OLIVIA", "MFSK", "MFSK32"]
+DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "BPSK", "PSK", "PSK31", "BPSK31", "OLIVIA", "MFSK", "MFSK32", "PKT"]
 ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
 MODE_TYPES = ["CW", "PHONE", "DATA"]
 

core/lookup_helper.py

@@ -3,7 +3,7 @@ import logging
 from datetime import timedelta
 
 from diskcache import Cache
-from pyhamtools import LookupLib, Callinfo
+from pyhamtools import LookupLib, Callinfo, callinfo
 from pyhamtools.exceptions import APIKeyMissingError
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
@@ -266,14 +266,19 @@ class LookupHelper:
     # Utility method to get QRZ.com data from cache if possible, if not get it from the API and cache it
     def get_qrz_data_for_callsign(self, call):
         # Fetch from cache if we can, otherwise fetch from the API and cache it
-        qrz_data = self.QRZ_CALLSIGN_DATA_CACHE.get(call)
-        if qrz_data:
-            return qrz_data
+        if call in self.QRZ_CALLSIGN_DATA_CACHE:
+            return self.QRZ_CALLSIGN_DATA_CACHE.get(call)
         elif self.QRZ_AVAILABLE:
             try:
                 data = self.LOOKUP_LIB_QRZ.lookup_callsign(callsign=call)
                 self.QRZ_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
                 return data
+            except KeyError:
+                # QRZ had no info for the call, but maybe it had prefixes or suffixes. Try again with the base call.
+                try:
+                    data = self.LOOKUP_LIB_QRZ.lookup_callsign(callsign=callinfo.Callinfo.get_homecall(call))
+                    self.QRZ_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
+                    return data
                 except KeyError:
                     # QRZ had no info for the call, that's OK. Cache a None so we don't try to look this up again
                     self.QRZ_CALLSIGN_DATA_CACHE.add(call, None, expire=604800)  # 1 week in seconds
@@ -284,14 +289,19 @@ class LookupHelper:
     # Utility method to get Clublog API data from cache if possible, if not get it from the API and cache it
     def get_clublog_api_data_for_callsign(self, call):
         # Fetch from cache if we can, otherwise fetch from the API and cache it
-        clublog_data = self.CLUBLOG_CALLSIGN_DATA_CACHE.get(call)
-        if clublog_data:
-            return clublog_data
+        if call in self.CLUBLOG_CALLSIGN_DATA_CACHE:
+            return self.CLUBLOG_CALLSIGN_DATA_CACHE.get(call)
         elif self.CLUBLOG_API_AVAILABLE:
             try:
                 data = self.LOOKUP_LIB_CLUBLOG_API.lookup_callsign(callsign=call)
                 self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
                 return data
+            except KeyError:
+                # Clublog had no info for the call, but maybe it had prefixes or suffixes. Try again with the base call.
+                try:
+                    data = self.LOOKUP_LIB_CLUBLOG_API.lookup_callsign(callsign=callinfo.Callinfo.get_homecall(call))
+                    self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
+                    return data
                 except KeyError:
                     # Clublog had no info for the call, that's OK. Cache a None so we don't try to look this up again
                     self.CLUBLOG_CALLSIGN_DATA_CACHE.add(call, None, expire=604800)  # 1 week in seconds

data/alert.py

@@ -111,11 +111,6 @@ class Alert:
         if self.dx_calls and not self.dx_names:
             self.dx_names = list(map(lambda c: lookup_helper.infer_name_from_callsign(c), self.dx_calls))
 
-        # Clean up comments
-        if self.comment:
-            comment = re.sub(r"\(de [A-Za-z0-9]*\)", "", self.comment)
-            self.comment = comment.strip()
-
         # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
         # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
         # apart from that they were retrieved from the API at different times. Note that the simple Python hash()

data/spot.py

@@ -177,6 +177,20 @@ class Spot:
         if self.de_call and "-" in self.de_call:
             self.de_call = self.de_call.split("-")[0]
 
+        # If we have a spotter of "RBNHOLE", we should have the actual spotter callsign in the comment, so extract it.
+        # RBNHole posts come from a number of providers, so it's dealt with here in the generic spot handling code.
+        if self.de_call == "RBNHOLE" and self.comment:
+            rbnhole_call_match = re.search(r"\Wat ([a-z0-9/]+)\W", self.comment, re.IGNORECASE)
+            if rbnhole_call_match:
+                self.de_call = rbnhole_call_match.group(1).upper()
+
+        # If we have a spotter of "SOTAMAT", we might have the actual spotter callsign in the comment, if so extract it.
+        # SOTAMAT can do POTA as well as SOTA, so it's dealt with here in the generic spot handling code.
+        if self.de_call == "SOTAMAT" and self.comment:
+            sotamat_call_match = re.search(r"\Wfrom ([a-z0-9/]+)]", self.comment, re.IGNORECASE)
+            if sotamat_call_match:
+                self.de_call = sotamat_call_match.group(1).upper()
+
         # Spotter country, continent, zones etc. from callsign.
         # DE of "RBNHOLE" and "SOTAMAT" are not things we can look up location for
         if self.de_call != "RBNHOLE" and self.de_call != "SOTAMAT":
@@ -249,14 +263,6 @@ class Spot:
         if self.comment and not self.qrt:
             self.qrt = "QRT" in self.comment.upper()
 
-        # Clean up comments
-        if self.comment:
-            comment = re.sub(r"\(de [A-Za-z0-9]*\)", "", self.comment)
-            comment = re.sub(r"\[.*]:", "", comment)
-            comment = re.sub(r"\[.*]", "", comment)
-            comment = re.sub(r"\"\"", "", comment)
-            self.comment = comment.strip()
-
         # DX operator details lookup, using QRZ.com. This should be the last resort compared to taking the data from
         # the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of
         # the one from the park reference they're at.
@@ -284,8 +290,8 @@ class Spot:
         self.dx_location_good = self.dx_location_source == "SPOT" or self.dx_location_source == "WAB/WAI GRID" or (
                 self.dx_location_source == "QRZ" and not "/" in self.dx_call)
 
-        # DE of "RBNHOLE", "SOTAMAT" and "ZLOTA" are not things we can look up location for
-        if self.de_call != "RBNHOLE" and self.de_call != "SOTAMAT" and self.de_call != "ZLOTA":
+        # DE of "RBNHOLE" and "SOTAMAT" are not things we can look up location for
+        if self.de_call != "RBNHOLE" and self.de_call != "SOTAMAT":
             # DE operator position lookup, using QRZ.com.
             if self.de_call and not self.de_latitude:
                 latlon = lookup_helper.infer_latlon_from_callsign_qrz(self.de_call)

spotproviders/dxcluster.py

@@ -19,9 +19,9 @@ class DXCluster(SpotProvider):
     # Note the callsign pattern deliberately excludes calls ending in "-#", which are from RBN and can be enabled by
     # default on some clusters. If you want RBN spots, there is a separate provider for that.
     CALLSIGN_PATTERN = "([a-z|0-9|/]+)"
-    FREQUENCY_PATTERM = "([0-9|.]+)"
+    FREQUENCY_PATTERN = "([0-9|.]+)"
     LINE_PATTERN = re.compile(
-        "^DX de " + CALLSIGN_PATTERN + ":\\s+" + FREQUENCY_PATTERM + "\\s+" + CALLSIGN_PATTERN + "\\s+(.*)\\s+(\\d{4}Z)",
+        "^DX de " + CALLSIGN_PATTERN + ":\\s+" + FREQUENCY_PATTERN + "\\s+" + CALLSIGN_PATTERN + "\\s+(.*)\\s+(\\d{4}Z)",
         re.IGNORECASE)
 
     # Constructor requires hostname and port

spotproviders/wota.py

@@ -1,7 +1,12 @@
-import re
+from datetime import timedelta, datetime
 
+import pytz
+from requests_cache import CachedSession
 from rss_parser import RSSParser
 
+from core.constants import HTTP_HEADERS
+from core.sig_utils import get_icon_for_sig
+from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
 
 
@@ -9,6 +14,10 @@ from spotproviders.http_spot_provider import HTTPSpotProvider
 class WOTA(HTTPSpotProvider):
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://www.wota.org.uk/spots_rss.php"
+    LIST_URL = "https://www.wota.org.uk/mapping/data/summits.json"
+    LIST_CACHE_TIME_DAYS = 30
+    LIST_CACHE = CachedSession("cache/wota_data_cache", expire_after=timedelta(days=LIST_CACHE_TIME_DAYS))
+    RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -17,7 +26,63 @@ class WOTA(HTTPSpotProvider):
         new_spots = []
         rss = RSSParser.parse(http_response.content.decode())
         # Iterate through source data
-        for source_alert in rss.channel.items:
+        for source_spot in rss.channel.items:
+
+            # Reject GUID missing or zero
+            if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0":
+                continue
+
+            # Pick apart the title
+            title_split = source_spot.title.split(" on ")
+            dx_call = title_split[0]
+            ref = None
+            ref_name = None
+            if len(title_split) > 1:
+                ref_split = title_split[1].split(" - ")
+                ref = ref_split[0]
+                if len(ref_split) > 1:
+                    ref_name = ref_split[1]
+
+            # Pick apart the description
+            desc_split = source_spot.description.split(". ")
+            freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
+            freq_mode_split = freq_mode.split("-")
+            freq_hz = float(freq_mode_split[0]) * 1000000
+            mode = freq_mode_split[1]
+
+            comment = None
+            if len(desc_split) > 1:
+                comment = desc_split[1].strip()
+            spotter = None
+            if len(desc_split) > 2:
+                spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").strip()
+
+            time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+
+            # Convert to our spot format
+            spot = Spot(source=self.name,
+                        source_id=source_spot.guid.content,
+                        dx_call=dx_call,
+                        de_call=spotter,
+                        freq=freq_hz,
+                        mode=mode,
+                        comment=comment,
+                        sig="WOTA",
+                        sig_refs=[ref] if ref else [],
+                        sig_refs_names=[ref_name] if ref_name else [],
+                        sig_refs_urls="https://www.wota.org.uk/MM_" + ref if ref else [],
+                        icon=get_icon_for_sig("WOTA"),
+                        time=time.timestamp())
+
+            # WOTA name/lat/lon lookup
+            wota_data = self.LIST_CACHE.get(self.LIST_URL, headers=HTTP_HEADERS).json()
+            for feature in wota_data["features"]:
+                if feature["properties"]["wotaId"] == spot.sig_refs[0]:
+                    spot.sig_refs_names = [feature["properties"]["title"]]
+                    spot.dx_latitude = feature["geometry"]["coordinates"][1]
+                    spot.dx_longitude = feature["geometry"]["coordinates"][0]
+                    spot.dx_grid = feature["properties"]["qthLocator"]
                     break
-            # TODO: Need to see a live spot to know what this feed looks like
+
+            new_spots.append(spot)
         return new_spots

views/webpage_base.tpl

@@ -31,6 +31,7 @@
     <link href="/fa/css/solid.min.css" rel="stylesheet" />
 
     <link rel="icon" type="image/png" href="/img/icon-512.png">
+    <link rel="apple-touch-icon" href="img/icon-512-pwa.png">
     <link rel="alternate icon" type="image/png" href="/img/icon-192.png">
     <link rel="alternate icon" type="image/png" href="/img/icon-32.png">
     <link rel="alternate icon" type="image/png" href="/img/icon-16.png">

webassets/apidocs/openapi.yml

@@ -1016,4 +1016,4 @@ components:
         ref_regex:
           type: string
           description: Regex that matches this SIG's reference IDs. Generally for Spothole's own internal use, clients probably won't need this.
-          example: "[A-Z]{2}\-\d+"
+          example: "[A-Z]{2}\\-\\d+"

webassets/js/alerts.js

@@ -126,7 +126,6 @@ function addAlertRowsToTable(tbody, alerts) {
         // Get times for the alert, and convert to local time if necessary.
         var start_time_utc = moment.unix(a["start_time"]).utc();
         var start_time_local = start_time_utc.clone().local();
-        console.log(a["dx_calls"])
         start_time = useLocalTime ? start_time_local : start_time_utc;
         var end_time_utc = moment.unix(a["end_time"]).utc();
         var end_time_local = end_time_utc.clone().local();

webassets/js/spots.js

@@ -185,7 +185,7 @@ function updateTable() {
             $tr.append(`<td class='nowrap'>${time_formatted}</td>`);
         }
         if (showDX) {
-            $tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${dx_flag}</span><a class='dx-link' href='https://qrz.com/db/${s["dx_call"]}' target='_new'>${s["dx_call"]}</a></td>`);
+            $tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${dx_flag}</span><a class='dx-link' href='https://qrz.com/db/${s["dx_call"]}' target='_new' title='${s["dx_name"] != null ? s["dx_name"] : ""}'>${s["dx_call"]}</a></td>`);
         }
         if (showFreq) {
             $tr.append(`<td class='nowrap'><span class='band-bullet' title='${bandFullName}' style='color: ${s["band_color"]}'>&#9632;</span>${freq_string}</td>`);