v1.0 release

core/constants.py

@@ -4,7 +4,7 @@ from data.sig import SIG
 
 # General software
 SOFTWARE_NAME = "Spothole by M0TRT"
-SOFTWARE_VERSION = "0.1"
+SOFTWARE_VERSION = "1.0"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + ", v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}

core/lookup_helper.py

@@ -301,7 +301,7 @@ class LookupHelper:
         return ituz
 
     # Infer an operator name from a callsign (requires QRZ.com/HamQTH)
-    def infer_name_from_callsign(self, call):
+    def infer_name_from_callsign_online_lookup(self, call):
         data = self.get_qrz_data_for_callsign(call)
         if data and "fname" in data:
             name = data["fname"]
@@ -315,27 +315,40 @@ class LookupHelper:
             return None
 
     # Infer a latitude and longitude from a callsign (requires QRZ.com/HamQTH)
-    def infer_latlon_from_callsign_qrz(self, call):
+    # Coordinates that look default are rejected (apologies if your position really is 0,0, enjoy your voyage)
+    def infer_latlon_from_callsign_online_lookup(self, call):
         data = self.get_qrz_data_for_callsign(call)
-        if data and "latitude" in data and "longitude" in data:
+        if data and "latitude" in data and "longitude" in data and (data["latitude"] != 0 or data["longitude"] != 0):
             return [data["latitude"], data["longitude"]]
         data = self.get_hamqth_data_for_callsign(call)
-        if data and "latitude" in data and "longitude" in data:
+        if data and "latitude" in data and "longitude" in data and (data["latitude"] != 0 or data["longitude"] != 0):
             return [data["latitude"], data["longitude"]]
         else:
             return None
 
-    # Infer a grid locator from a callsign (requires QRZ.com/HamQTH)
-    def infer_grid_from_callsign_qrz(self, call):
+    # Infer a grid locator from a callsign (requires QRZ.com/HamQTH).
+    # Grids that look default are rejected (apologies if your grid really is AA00aa, enjoy your research)
+    def infer_grid_from_callsign_online_lookup(self, call):
         data = self.get_qrz_data_for_callsign(call)
-        if data and "locator" in data:
+        if data and "locator" in data and data["locator"].upper() != "AA00" and data["locator"].upper() != "AA00AA" and data["locator"].upper() != "AA00AA00":
             return data["locator"]
         data = self.get_hamqth_data_for_callsign(call)
-        if data and "grid" in data:
+        if data and "grid" in data and data["grid"].upper() != "AA00" and data["grid"].upper() != "AA00AA" and data["grid"].upper() != "AA00AA00":
             return data["grid"]
         else:
             return None
 
+    # Infer a textual QTH from a callsign (requires QRZ.com/HamQTH)
+    def infer_qth_from_callsign_online_lookup(self, call):
+        data = self.get_qrz_data_for_callsign(call)
+        if data and "addr2" in data:
+            return data["addr2"]
+        data = self.get_hamqth_data_for_callsign(call)
+        if data and "qth" in data:
+            return data["qth"]
+        else:
+            return None
+
     # Infer a latitude and longitude from a callsign (using DXCC, probably very inaccurate)
     def infer_latlon_from_callsign_dxcc(self, call):
         try:

core/sig_utils.py

@@ -1,4 +1,5 @@
 import csv
+import logging
 
 from pyhamtools.locator import latlong_to_locator
 
@@ -28,89 +29,104 @@ def get_ref_regex_for_sig(sig):
 # Note there is currently no support for KRMNPA location lookup, see issue #61.
 def get_sig_ref_info(sig, sig_ref_id):
     sig_ref = SIGRef(id=sig_ref_id, sig=sig)
-    if sig.upper() == "POTA":
-        data = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + sig_ref_id, headers=HTTP_HEADERS).json()
-        if data:
-            fullname = data["name"] if "name" in data else None
-            if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
-                fullname = fullname + " " + data["parktypeDesc"]
-            sig_ref.name = fullname
-            sig_ref.url = "https://pota.app/#/park/" + sig_ref_id
-            sig_ref.grid = data["grid6"] if "grid6" in data else None
-            sig_ref.latitude = data["latitude"] if "latitude" in data else None
-            sig_ref.longitude = data["longitude"] if "longitude" in data else None
-    elif sig.upper() == "SOTA":
-        data = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + sig_ref_id,
-                                              headers=HTTP_HEADERS).json()
-        if data:
-            sig_ref.name = data["name"] if "name" in data else None
-            sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + sig_ref_id
-            sig_ref.grid = data["locator"] if "locator" in data else None
-            sig_ref.latitude = data["latitude"] if "latitude" in data else None
-            sig_ref.longitude = data["longitude"] if "longitude" in data else None
-    elif sig.upper() == "WWBOTA":
-        data = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + sig_ref_id,
-                                              headers=HTTP_HEADERS).json()
-        if data:
-            sig_ref.name = data["name"] if "name" in data else None
-            sig_ref.url = "https://bunkerwiki.org/?s=" + sig_ref_id if sig_ref_id.startswith("B/G") else None
-            sig_ref.grid = data["locator"] if "locator" in data else None
-            sig_ref.latitude = data["lat"] if "lat" in data else None
-            sig_ref.longitude = data["long"] if "long" in data else None
-    elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
-        data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + sig_ref_id,
-                                              headers=HTTP_HEADERS).json()
-        if data:
-            sig_ref.name = data["name"] if "name" in data else None
-            sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + sig_ref_id
-            sig_ref.grid = data["locator"] if "locator" in data else None
-            sig_ref.latitude = data["latitude"] if "latitude" in data else None
-            sig_ref.longitude = data["longitude"] if "longitude" in data else None
-    elif sig.upper() == "WWFF":
-        sig_ref.url = "https://wwff.co/directory/?showRef=" + sig_ref_id
-    elif sig.upper() == "SIOTA":
-        siota_csv_data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
-                                                        headers=HTTP_HEADERS)
-        siota_dr = csv.DictReader(siota_csv_data.content.decode().splitlines())
-        for row in siota_dr:
-            if row["SILO_CODE"] == sig_ref_id:
-                sig_ref.name = row["NAME"] if "NAME" in row else None
-                sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
-                sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
-                sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
-    elif sig.upper() == "WOTA":
-        data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
-                                              headers=HTTP_HEADERS).json()
-        if data:
-            for feature in data["features"]:
-                if feature["properties"]["wotaId"] == sig_ref_id:
-                    sig_ref.name = feature["properties"]["title"]
-                    sig_ref.url = "https://www.wota.org.uk/MM_" + sig_ref_id
-                    sig_ref.grid = feature["properties"]["qthLocator"]
-                    sig_ref.latitude = feature["geometry"]["coordinates"][1]
-                    sig_ref.longitude = feature["geometry"]["coordinates"][0]
-    elif sig.upper() == "ZLOTA":
-        data = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS).json()
-        if data:
-            for asset in data:
-                if asset["code"] == sig_ref_id:
-                    sig_ref.name = asset["name"]
-                    sig_ref.url = "https://ontheair.nz/assets/ZLI_OT-030" + sig_ref_id.replace("/", "_")
-                    sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
-                    sig_ref.latitude = asset["y"]
-                    sig_ref.longitude = asset["x"]
-    elif sig.upper() == "BOTA":
-        if not sig_ref.name:
-            sig_ref.name = sig_ref.id
-        sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
-    elif sig.upper() == "WAB" or sig.upper() == "WAI":
-        ll = wab_wai_square_to_lat_lon(sig_ref_id)
-        if ll:
-            sig_ref.name = sig_ref_id
-            sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
-            sig_ref.latitude = ll[0]
-            sig_ref.longitude = ll[1]
-
+    try:
+        if sig.upper() == "POTA":
+            data = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + sig_ref_id, headers=HTTP_HEADERS).json()
+            if data:
+                fullname = data["name"] if "name" in data else None
+                if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
+                    fullname = fullname + " " + data["parktypeDesc"]
+                sig_ref.name = fullname
+                sig_ref.url = "https://pota.app/#/park/" + sig_ref_id
+                sig_ref.grid = data["grid6"] if "grid6" in data else None
+                sig_ref.latitude = data["latitude"] if "latitude" in data else None
+                sig_ref.longitude = data["longitude"] if "longitude" in data else None
+        elif sig.upper() == "SOTA":
+            data = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + sig_ref_id,
+                                                  headers=HTTP_HEADERS).json()
+            if data:
+                sig_ref.name = data["name"] if "name" in data else None
+                sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + sig_ref_id
+                sig_ref.grid = data["locator"] if "locator" in data else None
+                sig_ref.latitude = data["latitude"] if "latitude" in data else None
+                sig_ref.longitude = data["longitude"] if "longitude" in data else None
+        elif sig.upper() == "WWBOTA":
+            data = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + sig_ref_id,
+                                                  headers=HTTP_HEADERS).json()
+            if data:
+                sig_ref.name = data["name"] if "name" in data else None
+                sig_ref.url = "https://bunkerwiki.org/?s=" + sig_ref_id if sig_ref_id.startswith("B/G") else None
+                sig_ref.grid = data["locator"] if "locator" in data else None
+                sig_ref.latitude = data["lat"] if "lat" in data else None
+                sig_ref.longitude = data["long"] if "long" in data else None
+        elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
+            data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + sig_ref_id,
+                                                  headers=HTTP_HEADERS).json()
+            if data:
+                sig_ref.name = data["name"] if "name" in data else None
+                sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + sig_ref_id
+                sig_ref.grid = data["locator"] if "locator" in data else None
+                sig_ref.latitude = data["latitude"] if "latitude" in data else None
+                sig_ref.longitude = data["longitude"] if "longitude" in data else None
+        elif sig.upper() == "WWFF":
+            wwff_csv_data = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
+                                                            headers=HTTP_HEADERS)
+            wwff_dr = csv.DictReader(wwff_csv_data.content.decode().splitlines())
+            for row in wwff_dr:
+                if row["reference"] == sig_ref_id:
+                    sig_ref.name = row["name"] if "name" in row else None
+                    sig_ref.url = "https://wwff.co/directory/?showRef=" + sig_ref_id
+                    sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row else None
+                    sig_ref.latitude = float(row["latitude"]) if "latitude" in row else None
+                    sig_ref.longitude = float(row["longitude"]) if "longitude" in row else None
+                    break
+        elif sig.upper() == "SIOTA":
+            siota_csv_data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
+                                                            headers=HTTP_HEADERS)
+            siota_dr = csv.DictReader(siota_csv_data.content.decode().splitlines())
+            for row in siota_dr:
+                if row["SILO_CODE"] == sig_ref_id:
+                    sig_ref.name = row["NAME"] if "NAME" in row else None
+                    sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
+                    sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
+                    sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
+                    break
+        elif sig.upper() == "WOTA":
+            data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
+                                                  headers=HTTP_HEADERS).json()
+            if data:
+                for feature in data["features"]:
+                    if feature["properties"]["wotaId"] == sig_ref_id:
+                        sig_ref.name = feature["properties"]["title"]
+                        sig_ref.url = "https://www.wota.org.uk/MM_" + sig_ref_id
+                        sig_ref.grid = feature["properties"]["qthLocator"]
+                        sig_ref.latitude = feature["geometry"]["coordinates"][1]
+                        sig_ref.longitude = feature["geometry"]["coordinates"][0]
+                        break
+        elif sig.upper() == "ZLOTA":
+            data = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS).json()
+            if data:
+                for asset in data:
+                    if asset["code"] == sig_ref_id:
+                        sig_ref.name = asset["name"]
+                        sig_ref.url = "https://ontheair.nz/assets/ZLI_OT-030" + sig_ref_id.replace("/", "_")
+                        sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
+                        sig_ref.latitude = asset["y"]
+                        sig_ref.longitude = asset["x"]
+                        break
+        elif sig.upper() == "BOTA":
+            if not sig_ref.name:
+                sig_ref.name = sig_ref.id
+            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
+        elif sig.upper() == "WAB" or sig.upper() == "WAI":
+            ll = wab_wai_square_to_lat_lon(sig_ref_id)
+            if ll:
+                sig_ref.name = sig_ref_id
+                sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
+                sig_ref.latitude = ll[0]
+                sig_ref.longitude = ll[1]
+    except:
+        logging.warn("Failed to look up sig_ref info for " + sig + " ref " + sig_ref_id + ".")
     return sig_ref
 
 

data/alert.py

@@ -121,7 +121,7 @@ class Alert:
         # the actual alertting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of
         # the one from the park reference they're at.
         if self.dx_calls and not self.dx_names:
-            self.dx_names = list(map(lambda c: lookup_helper.infer_name_from_callsign(c), self.dx_calls))
+            self.dx_names = list(map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c), self.dx_calls))
 
         # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
         # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical

data/spot.py

@@ -27,6 +27,9 @@ class Spot:
     dx_call: str = None
     # Name of the operator that has been spotted
     dx_name: str = None
+    # QTH of the operator that has been spotted. This could be from any SIG refs or could be from online lookup of their
+    # home QTH.
+    dx_qth: str = None
     # Country of the DX operator
     dx_country: str = None
     # Country flag of the DX operator
@@ -313,15 +316,24 @@ class Spot:
         # the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of
         # the one from the park reference they're at.
         if self.dx_call and not self.dx_name:
-            self.dx_name = lookup_helper.infer_name_from_callsign(self.dx_call)
+            self.dx_name = lookup_helper.infer_name_from_callsign_online_lookup(self.dx_call)
         if self.dx_call and not self.dx_latitude:
-            latlon = lookup_helper.infer_latlon_from_callsign_qrz(self.dx_call)
+            latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.dx_call)
             if latlon:
                 self.dx_latitude = latlon[0]
                 self.dx_longitude = latlon[1]
-            self.dx_grid = lookup_helper.infer_grid_from_callsign_qrz(self.dx_call)
+            self.dx_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.dx_call)
             self.dx_location_source = "HOME QTH"
 
+        # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable stirng,
+        # otherwise see what they have set on an online lookup service.
+        if self.sig_refs and len(self.sig_refs) > 0:
+            self.dx_qth = self.sig_refs[0].id
+            if self.sig_refs[0].name:
+                self.dx_qth = self.dx_qth + " " + self.sig_refs[0].name
+        else:
+            self.dx_qth = lookup_helper.infer_qth_from_callsign_online_lookup(self.dx_call)
+
         # Last resort for getting a DX position, use the DXCC entity.
         if self.dx_call and not self.dx_latitude:
             latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.dx_call)
@@ -341,11 +353,11 @@ class Spot:
         if self.de_call and any(char.isdigit() for char in self.de_call) and not (self.de_call.startswith("T2") and self.source == "APRS-IS"):
             # DE operator position lookup, using QRZ.com.
             if not self.de_latitude:
-                latlon = lookup_helper.infer_latlon_from_callsign_qrz(self.de_call)
+                latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.de_call)
                 if latlon:
                     self.de_latitude = latlon[0]
                     self.de_longitude = latlon[1]
-                self.de_grid = lookup_helper.infer_grid_from_callsign_qrz(self.de_call)
+                self.de_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.de_call)
 
             # Last resort for getting a DE position, use the DXCC entity.
             if not self.de_latitude:

server/webserver.py

@@ -127,6 +127,7 @@ class WebServer:
             return self.serve_api({
                 "call": call,
                 "name": fake_spot.dx_name,
+                "qth": fake_spot.dx_qth,
                 "country": fake_spot.dx_country,
                 "flag": fake_spot.dx_flag,
                 "continent": fake_spot.dx_continent,

views/webpage_about.tpl

@@ -32,6 +32,8 @@
     <p>To avoid putting too much load on the various servers that Spothole connects to, the Spothole server only polls them once every two minutes for spots, and once every hour for alerts. (Some sources, such as DX clusters, RBN, APRS-IS and WWBOTA use a non-polling mechanism, and their updates will therefore arrive more quickly.) Then if you are using the web interface, that has its own rate at which it reloads the data from Spothole, which is once a minute for spots or 30 minutes for alerts. So you could be waiting around three minutes to see a newly added spot, or 90 minutes to see a newly added alert.</p>
     <h4 class="mt-4">What licence does Spothole use?</h4>
     <p>Spothole's source code is licenced under the Public Domain. You can write a Spothole client, run your own server, modify it however you like, you can claim you wrote it and charge people £1000 for a copy, I don't really mind. (Please don't do the last one. But if you're using my code for something cool, it would be nice to hear from you!)</p>
+    <h2 class="mt-4">Data Accuracy</h2>
+    <p>Please note that the data coming out of Spothole is only as good as the data going in. People mis-hear and make typos when spotting callsigns all the time. There are also plenty of cases where Spothole's data, particularly location data, may be inaccurate. For example, there are POTA parks that span multiple US states, countries that span multiple CQ zones, portable operators with no requirement to sign /P, etc. If you are doing something where accuracy is important, such as contesting, you should not rely on Spothole's data to fill in any gaps in your log.</p>
     <h2 id="privacy" class="mt-4">Privacy</h2>
     <p>Spothole collects no data about you, and there is no way to enter personally identifying information into the site apart from by spotting and alerting through Spothole or the various services it connects to. All spots and alerts are "timed out" and deleted from the system after a set interval, which by default is one hour for spots and one week for alerts.</p>
     <p>Settings you select from Spothole's menus are sent to the server, in order to provide the data with the requested filters. They are also stored in your browser's local storage, so that your preferences are remembered between sessions.</p>

webassets/apidocs/openapi.yml

@@ -5,6 +5,10 @@ info:
     Spothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open JSON API as well as a website to browse the data.
     
     While there are other web-based interfaces to DX clusters, and sites that aggregate spots from various outdoor activity programmes for amateur radio, Spothole differentiates itself by supporting a large number of data sources, and by being "API first" rather than just providing a web front-end. This allows other software to be built on top of it. Spothole itself is also open source, Public Domain licenced code that anyone can take and modify.
+    
+    The API calls described below allow third-party software to access data from Spothole, and receive data on spots and alerts in a consistent format regardless of the data sources used by Spothole itself. Utility calls are also provided for general data lookups.
+    
+    Please note that the data coming out of Spothole is only as good as the data going in. People mis-hear and make typos when spotting callsigns all the time, and there are plenty of areas where Spothole's location data may be inaccurate. If you are doing something where accuracy is important, such as contesting, you should not rely on Spothole's data to fill in any gaps in your log.
   contact:
     email: ian@ianrenton.com
   license:
@@ -518,6 +522,10 @@ paths:
                     type: string
                     description: Name of the operator
                     example: Ian
+                  qth:
+                    type: string
+                    description: QTH of the operator. This could be from any SIG refs or could be from online lookup of their home QTH.
+                    example: Dorset
                   country:
                     type: string
                     description: Country of the operator
@@ -745,6 +753,10 @@ components:
           type: string
           description: Name of the operator that has been spotted
           example: Ian
+        dx_qth:
+          type: string
+          description: QTH of the operator that has been spotted. This could be from any SIG refs or could be from online lookup of their home QTH.
+          example: Dorset
         dx_country:
           type: string
           description: Country of the DX operator