Fix WOTA parsing bug

core/constants.py

@@ -4,7 +4,7 @@ from data.sig import SIG
 
 # General software
 SOFTWARE_NAME = "Spothole by M0TRT"
-SOFTWARE_VERSION = "0.1"
+SOFTWARE_VERSION = "1.0.1"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + ", v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}

core/lookup_helper.py

@@ -315,23 +315,25 @@ class LookupHelper:
             return None
 
     # Infer a latitude and longitude from a callsign (requires QRZ.com/HamQTH)
+    # Coordinates that look default are rejected (apologies if your position really is 0,0, enjoy your voyage)
     def infer_latlon_from_callsign_online_lookup(self, call):
         data = self.get_qrz_data_for_callsign(call)
-        if data and "latitude" in data and "longitude" in data:
+        if data and "latitude" in data and "longitude" in data and (data["latitude"] != 0 or data["longitude"] != 0):
             return [data["latitude"], data["longitude"]]
         data = self.get_hamqth_data_for_callsign(call)
-        if data and "latitude" in data and "longitude" in data:
+        if data and "latitude" in data and "longitude" in data and (data["latitude"] != 0 or data["longitude"] != 0):
             return [data["latitude"], data["longitude"]]
         else:
             return None
 
-    # Infer a grid locator from a callsign (requires QRZ.com/HamQTH)
+    # Infer a grid locator from a callsign (requires QRZ.com/HamQTH).
+    # Grids that look default are rejected (apologies if your grid really is AA00aa, enjoy your research)
     def infer_grid_from_callsign_online_lookup(self, call):
         data = self.get_qrz_data_for_callsign(call)
-        if data and "locator" in data:
+        if data and "locator" in data and data["locator"].upper() != "AA00" and data["locator"].upper() != "AA00AA" and data["locator"].upper() != "AA00AA00":
             return data["locator"]
         data = self.get_hamqth_data_for_callsign(call)
-        if data and "grid" in data:
+        if data and "grid" in data and data["grid"].upper() != "AA00" and data["grid"].upper() != "AA00AA" and data["grid"].upper() != "AA00AA00":
             return data["grid"]
         else:
             return None

spotproviders/gma.py

@@ -5,7 +5,6 @@ import pytz
 
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import HTTP_HEADERS
-from core.sig_utils import get_icon_for_sig
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -51,7 +50,7 @@ class GMA(HTTPSpotProvider):
                 # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
                 # and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry
                 # to determine if it's a SOTA summit.
-                if ref_info["reftype"] not in ["POTA", "WWFF"] and (ref_info["reftype"] != "Summit" or ref_info["sota"] == ""):
+                if "reftype" in ref_info and ref_info["reftype"] not in ["POTA", "WWFF"] and (ref_info["reftype"] != "Summit" or ref_info["sota"] == ""):
                     match ref_info["reftype"]:
                         case "Summit":
                             spot.sig_refs[0].sig = "GMA"

spotproviders/wota.py

@@ -1,9 +1,10 @@
+import logging
+import re
 from datetime import datetime
 
 import pytz
 from rss_parser import RSSParser
 
-from core.sig_utils import get_icon_for_sig
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -12,7 +13,7 @@ from spotproviders.http_spot_provider import HTTPSpotProvider
 # Spot provider for Wainwrights on the Air
 class WOTA(HTTPSpotProvider):
     POLL_INTERVAL_SEC = 120
-    SPOTS_URL = "https://www.wota.org.uk/spots_rss.php"
+    SPOTS_URL = "http://127.0.0.1:8000/spots_rss.php"
     LIST_URL = "https://www.wota.org.uk/mapping/data/summits.json"
     RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
 
@@ -25,47 +26,50 @@ class WOTA(HTTPSpotProvider):
         # Iterate through source data
         for source_spot in rss.channel.items:
 
-            # Reject GUID missing or zero
+            try:
-            if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0":
+                # Reject GUID missing or zero
-                continue
+                if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0":
+                    continue
 
-            # Pick apart the title
+                # Pick apart the title
-            title_split = source_spot.title.split(" on ")
+                title_split = source_spot.title.split(" on ")
-            dx_call = title_split[0]
+                dx_call = title_split[0]
-            ref = None
+                ref = None
-            ref_name = None
+                ref_name = None
-            if len(title_split) > 1:
+                if len(title_split) > 1:
-                ref_split = title_split[1].split(" - ")
+                    ref_split = title_split[1].split(" - ")
-                ref = ref_split[0]
+                    ref = ref_split[0]
-                if len(ref_split) > 1:
+                    if len(ref_split) > 1:
-                    ref_name = ref_split[1]
+                        ref_name = ref_split[1]
 
-            # Pick apart the description
+                # Pick apart the description
-            desc_split = source_spot.description.split(". ")
+                desc_split = source_spot.description.split(". ")
-            freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
+                freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
-            freq_mode_split = freq_mode.split("-")
+                freq_mode_split = re.split(r'[\-\s]+', freq_mode)
-            freq_hz = float(freq_mode_split[0]) * 1000000
+                freq_hz = float(freq_mode_split[0]) * 1000000
-            mode = freq_mode_split[1]
+                mode = freq_mode_split[1].upper()
 
-            comment = None
+                comment = None
-            if len(desc_split) > 1:
+                if len(desc_split) > 1:
-                comment = desc_split[1].strip()
+                    comment = desc_split[1].strip()
-            spotter = None
+                spotter = None
-            if len(desc_split) > 2:
+                if len(desc_split) > 2:
-                spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").strip()
+                    spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").upper().strip()
 
-            time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+                time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
 
-            # Convert to our spot format
+                # Convert to our spot format
-            spot = Spot(source=self.name,
+                spot = Spot(source=self.name,
-                        source_id=source_spot.guid.content,
+                            source_id=source_spot.guid.content,
-                        dx_call=dx_call,
+                            dx_call=dx_call,
-                        de_call=spotter,
+                            de_call=spotter,
-                        freq=freq_hz,
+                            freq=freq_hz,
-                        mode=mode,
+                            mode=mode,
-                        comment=comment,
+                            comment=comment,
-                        sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
+                            sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
-                        time=time.timestamp())
+                            time=time.timestamp())
 
-            new_spots.append(spot)
+                new_spots.append(spot)
+            except Exception as e:
+                logging.error("Exception parsing WOTA spot", e)
         return new_spots

views/webpage_about.tpl

@@ -32,6 +32,8 @@
     <p>To avoid putting too much load on the various servers that Spothole connects to, the Spothole server only polls them once every two minutes for spots, and once every hour for alerts. (Some sources, such as DX clusters, RBN, APRS-IS and WWBOTA use a non-polling mechanism, and their updates will therefore arrive more quickly.) Then if you are using the web interface, that has its own rate at which it reloads the data from Spothole, which is once a minute for spots or 30 minutes for alerts. So you could be waiting around three minutes to see a newly added spot, or 90 minutes to see a newly added alert.</p>
     <h4 class="mt-4">What licence does Spothole use?</h4>
     <p>Spothole's source code is licenced under the Public Domain. You can write a Spothole client, run your own server, modify it however you like, you can claim you wrote it and charge people £1000 for a copy, I don't really mind. (Please don't do the last one. But if you're using my code for something cool, it would be nice to hear from you!)</p>
+    <h2 class="mt-4">Data Accuracy</h2>
+    <p>Please note that the data coming out of Spothole is only as good as the data going in. People mis-hear and make typos when spotting callsigns all the time. There are also plenty of cases where Spothole's data, particularly location data, may be inaccurate. For example, there are POTA parks that span multiple US states, countries that span multiple CQ zones, portable operators with no requirement to sign /P, etc. If you are doing something where accuracy is important, such as contesting, you should not rely on Spothole's data to fill in any gaps in your log.</p>
     <h2 id="privacy" class="mt-4">Privacy</h2>
     <p>Spothole collects no data about you, and there is no way to enter personally identifying information into the site apart from by spotting and alerting through Spothole or the various services it connects to. All spots and alerts are "timed out" and deleted from the system after a set interval, which by default is one hour for spots and one week for alerts.</p>
     <p>Settings you select from Spothole's menus are sent to the server, in order to provide the data with the requested filters. They are also stored in your browser's local storage, so that your preferences are remembered between sessions.</p>

views/webpage_spots.tpl

@@ -16,7 +16,7 @@
             <p class="d-inline-flex gap-1">
                 <span style="position: relative;">
                     <i class="fa-solid fa-magnifying-glass" style="position: absolute; left: 0px; top: 2px; padding: 10px; pointer-events: none;"></i>
-                    <input id="filter-dx-call" type="text" class="form-control" oninput="filtersUpdated();" placeholder="Search for call">
+                    <input id="filter-dx-call" type="search" class="form-control" oninput="filtersUpdated();" placeholder="Callsign">
                 </span>
                 <button id="filters-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleFiltersPanel();"><i class="fa-solid fa-filter"></i> Filters</button>
                 <button id="display-button" type="button" class="btn btn-outline-primary" data-bs-toggle="button" onclick="toggleDisplayPanel();"><i class="fa-solid fa-desktop"></i> Display</button>

webassets/apidocs/openapi.yml

@@ -5,6 +5,10 @@ info:
     Spothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open JSON API as well as a website to browse the data.
     
     While there are other web-based interfaces to DX clusters, and sites that aggregate spots from various outdoor activity programmes for amateur radio, Spothole differentiates itself by supporting a large number of data sources, and by being "API first" rather than just providing a web front-end. This allows other software to be built on top of it. Spothole itself is also open source, Public Domain licenced code that anyone can take and modify.
+    
+    The API calls described below allow third-party software to access data from Spothole, and receive data on spots and alerts in a consistent format regardless of the data sources used by Spothole itself. Utility calls are also provided for general data lookups.
+    
+    Please note that the data coming out of Spothole is only as good as the data going in. People mis-hear and make typos when spotting callsigns all the time, and there are plenty of areas where Spothole's location data may be inaccurate. If you are doing something where accuracy is important, such as contesting, you should not rely on Spothole's data to fill in any gaps in your log.
   contact:
     email: ian@ianrenton.com
   license:
@@ -524,11 +528,11 @@ paths:
                     example: Dorset
                   country:
                     type: string
-                    description: Country of the operator
+                    description: Country of the operator. Note that this is named "country" for commonality with other amateur radio tools, but in reality this is more of a "DXCC Name", as it includes many options which are not countries, just territories that DXCC uniquely identifies.
-                    example: United Kingdom
+                    example: England
                   flag:
                     type: string
-                    description: Country flag of the operator
+                    description: Country flag of the operator. This is limited to the range of emoji flags. For some DXCCs there may not be an official emoji flag, e.g. Northern Ireland, so the appearance may vary depending on your browser and operating system. Some small islands may also have no flag. Many DXCCs may also share a flag, e.g. mainland Spain, Balearic Islands, etc.
                     example: ""
                   continent:
                     type: string
@@ -755,11 +759,11 @@ components:
           example: Dorset
         dx_country:
           type: string
-          description: Country of the DX operator
+          description: Country of the operator. Note that this is named "country" for commonality with other amateur radio tools, but in reality this is more of a "DXCC Name", as it includes many options which are not countries, just territories that DXCC uniquely identifies.
-          example: United Kingdom
+          example: England
         dx_flag:
           type: string
-          description: Country flag of the DX operator
+          description: Country flag of the DX operator. This is limited to the range of emoji flags. For some DXCCs there may not be an official emoji flag, e.g. Northern Ireland, so the appearance may vary depending on your browser and operating system. Some small islands may also have no flag. Many DXCCs may also share a flag, e.g. mainland Spain, Balearic Islands, etc.
           example: ""
         dx_continent:
           type: string
@@ -822,11 +826,11 @@ components:
           example: M0TEST
         de_country:
           type: string
-          description: Country of the spotter
+          description: Country of the operator. Note that this is named "country" for commonality with other amateur radio tools, but in reality this is more of a "DXCC Name", as it includes many options which are not countries, just territories that DXCC uniquely identifies.
-          example: United Kingdom
+          example: England
         de_flag:
           type: string
-          description: Country flag of the spotter
+          description: Country flag of the spotter. This is limited to the range of emoji flags. For some DXCCs there may not be an official emoji flag, e.g. Northern Ireland, so the appearance may vary depending on your browser and operating system. Some small islands may also have no flag. Many DXCCs may also share a flag, e.g. mainland Spain, Balearic Islands, etc.
           example: ""
         de_continent:
           type: string
@@ -1046,11 +1050,11 @@ components:
             example: Ian
         dx_country:
           type: string
-          description: Country of the DX operator. This, and the subsequent fields, assume that all activators will be in the same country!
+          description: Country of the DX operator. Country of the operator. Note that this is named "country" for commonality with other amateur radio tools, but in reality this is more of a "DXCC Name", as it includes many options which are not countries, just territories that DXCC uniquely identifies. This, and the subsequent fields, assume that all activators will be in the same country!
-          example: United Kingdom
+          example: England
         dx_flag:
           type: string
-          description: Country flag of the DX operator
+          description: Country flag of the DX operator. This is limited to the range of emoji flags. For some DXCCs there may not be an official emoji flag, e.g. Northern Ireland, so the appearance may vary depending on your browser and operating system. Some small islands may also have no flag. Many DXCCs may also share a flag, e.g. mainland Spain, Balearic Islands, etc.
           example: ""
         dx_continent:
           type: string

webassets/css/style.css

@@ -44,7 +44,7 @@ div.container {
 /* SPOTS/ALERTS PAGES, SETTINGS/STATUS AREAS */
 
 input#filter-dx-call {
-    max-width: 10em;
+    max-width: 12em;
     margin-right: 1rem;
     padding-left: 2em;
 }
@@ -71,11 +71,16 @@ td.nowrap, span.nowrap {
 
 span.flag-wrapper {
     display: inline-block;
-    width: 1.7em;
+    width: 1.8em;
     text-align: center;
     cursor: default;
 }
 
+img.flag {
+    position: relative;
+    top: -2px;
+}
+
 span.band-bullet {
     display: inline-block;
     cursor: default;
@@ -265,7 +270,7 @@ div.band-spot:hover span.band-spot-info {
     }
     /* Filter/search DX Call field should be smaller on mobile */
     input#filter-dx-call {
-        max-width: 6em;
+        max-width: 9em;
         margin-right: 0;
     }
 }