Merge branch 'main' into 95-send-spots-to-xota

alertproviders/parksnpeaks.py

@@ -33,18 +33,24 @@ class ParksNPeaks(HTTPAlertProvider):
             start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(
                 tzinfo=pytz.UTC).timestamp()
 
+            sigrefs = []
+            # PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list,
+            # so mask this out if we got it.
+            if sig != "QRP":
+                sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)]
+
             # Convert to our alert format
             alert = Alert(source=self.name,
                           source_id=source_alert["alID"],
                           dx_calls=[source_alert["CallSign"].upper()],
                           freqs_modes=source_alert["Freq"] + " " + source_alert["MODE"],
                           comment=source_alert["Comments"],
-                          sig_refs=[SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)],
+                          sig_refs=sigrefs,
                           start_time=start_time,
                           is_dxpedition=False)
 
             # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-            if sig and sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA"]:
+            if sig and sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
                 logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!")
 
             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to

core/cache_utils.py

@@ -1,3 +1,4 @@
+import threading
 from datetime import timedelta
 
 from requests_cache import CachedSession
@@ -5,6 +6,19 @@ from requests_cache import CachedSession
 # Cache for "semi-static" data such as the locations of parks, CSVs of reference lists, etc.
 # This has an expiry time of 30 days, so will re-request from the source after that amount
 # of time has passed. This is used throughout Spothole to cache data that does not change
-# rapidly.
+# rapidly. The ThreadSafeSession construct here protects it against some multithreading
-SEMI_STATIC_URL_DATA_CACHE = CachedSession("cache/semi_static_url_data_cache",
+# contention weirdness we sometimes used to see on startup where the cache was hammered
-                                           expire_after=timedelta(days=30))
+# pretty hard.
+_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30))
+_lock = threading.Lock()
+
+
+class _ThreadSafeSession:
+    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads."""
+
+    def get(self, *args, **kwargs):
+        with _lock:
+            return _session.get(*args, **kwargs)
+
+
+SEMI_STATIC_URL_DATA_CACHE = _ThreadSafeSession()

core/sig_utils.py

@@ -29,7 +29,10 @@ def populate_sig_ref_info(sig_ref):
     ref_id = sig_ref.id
     try:
         if sig.upper() == "POTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS).json()
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
             if data:
                 fullname = str(data["name"]) if "name" in data else None
                 if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
@@ -40,8 +43,11 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.latitude = data["latitude"] if "latitude" in data else None
                 sig_ref.longitude = data["longitude"] if "longitude" in data else None
         elif sig.upper() == "SOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
-                                                  headers=HTTP_HEADERS).json()
+                                                      headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
             if data:
                 sig_ref.name = data["name"] if "name" in data else None
                 sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
@@ -50,8 +56,11 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.longitude = data["longitude"] if "longitude" in data else None
                 sig_ref.activation_score = data["points"] if "points" in data else None
         elif sig.upper() == "WWBOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
-                                                  headers=HTTP_HEADERS).json()
+                                                      headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
             if data:
                 sig_ref.name = data["name"] if "name" in data else None
                 sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
@@ -59,8 +68,11 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.latitude = data["lat"] if "lat" in data else None
                 sig_ref.longitude = data["long"] if "long" in data else None
         elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
-                                                  headers=HTTP_HEADERS).json()
+                                                      headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
             if data:
                 sig_ref.name = data["name"] if "name" in data else None
                 sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
@@ -68,9 +80,12 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.latitude = data["latitude"] if "latitude" in data else None
                 sig_ref.longitude = data["longitude"] if "longitude" in data else None
         elif sig.upper() == "WWFF":
-            wwff_csv_data = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
+            wwff_response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
                                                            headers=HTTP_HEADERS)
-            wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_csv_data.content.decode().splitlines())}
+            if not wwff_response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", wwff_response.status_code, sig, ref_id)
+                return sig_ref
+            wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_response.content.decode().splitlines())}
             row = wwff_index.get(ref_id)
             if row:
                 sig_ref.name = row["name"] if "name" in row else None
@@ -79,10 +94,13 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
                 sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row["longitude"] != "-" else None
         elif sig.upper() == "SIOTA":
-            siota_csv_data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
+            siota_response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
                                                             headers=HTTP_HEADERS)
+            if not siota_response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", siota_response.status_code, sig, ref_id)
+                return sig_ref
             siota_index = {row["SILO_CODE"]: row for row in
-                           csv.DictReader(siota_csv_data.content.decode().splitlines())}
+                           csv.DictReader(siota_response.content.decode().splitlines())}
             row = siota_index.get(ref_id)
             if row:
                 sig_ref.name = row["NAME"] if "NAME" in row else None
@@ -90,10 +108,13 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
                 sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
         elif sig.upper() == "WOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
-                                                  headers=HTTP_HEADERS).json()
+                                                      headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
             if data:
-                for feature in data["features"]:
+                for feature in data.get("features", []):
                     if feature["properties"]["wotaId"] == ref_id:
                         sig_ref.name = feature["properties"]["title"]
                         # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -107,8 +128,11 @@ def populate_sig_ref_info(sig_ref):
                         sig_ref.longitude = feature["geometry"]["coordinates"][0]
                         break
         elif sig.upper() == "ZLOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS).json()
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
-            if data:
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
+            if isinstance(data, list):
                 for asset in data:
                     if asset["code"] == ref_id:
                         sig_ref.name = asset["name"]
@@ -125,9 +149,12 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.name = sig_ref.id
             sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
         elif sig.upper() == "LLOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
-                                                  headers=HTTP_HEADERS).json()
+                                                      headers=HTTP_HEADERS)
-            if data:
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
+            if isinstance(data, list):
                 for ref in data:
                     if ref["reference_code"] == ref_id:
                         sig_ref.name = str(ref["name"])
@@ -161,8 +188,8 @@ def populate_sig_ref_info(sig_ref):
                     sig_ref.longitude = ll[1]
                 except:
                     logging.debug("Invalid lat/lon received for reference")
-    except:
+    except Exception:
-        logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id + ".")
+        logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
     return sig_ref
 
 

server/handlers/api/options.py

@@ -61,7 +61,7 @@ class APIOptionsHandler(tornado.web.RequestHandler):
                    "spot_sources": spot_sources,
                    "alert_sources": alert_sources,
                    "continents": CONTINENTS,
-                   "propagation_modes": PROPAGATION_MODES.values(),
+                   "propagation_modes": list(PROPAGATION_MODES.values()),
                    "max_spot_age": MAX_SPOT_AGE,
                    "spot_allowed": ALLOW_SPOTTING,
                    "spot_submit_providers": spot_submit_providers}

spotproviders/parksnpeaks.py

@@ -61,7 +61,7 @@ class ParksNPeaks(HTTPSpotProvider):
                         sig_refs[0].name = source_spot["actLocation"]
 
                     # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-                    if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA"]:
+                    if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA"]:
                         logging.warning("PNP spot found with sig " + sig + ", developer needs to add support for this!")
 
                 # Add new spot to the list

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/js/add-spot.js?v=1782036437"></script>
+<script src="/js/add-spot.js?v=1782076050"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -75,7 +75,7 @@
 
 </div>
 
-<script src="/js/alerts.js?v=1782036437"></script>
+<script src="/js/alerts.js?v=1782076050"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -77,8 +77,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/js/spotsbandsandmap.js?v=1782036437"></script>
+<script src="/js/spotsbandsandmap.js?v=1782076050"></script>
-<script src="/js/bands.js?v=1782036437"></script>
+<script src="/js/bands.js?v=1782076050"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/css/style.css?v=1782036437" type="text/css">
+<link rel="stylesheet" href="/css/style.css?v=1782076050" type="text/css">
 <link href="/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
 <script src="/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
 <script src="/vendor/js/tinycolor2-1.6.0.min.js"></script>
 
-<script src="/js/utils.js?v=1782036437"></script>
+<script src="/js/utils.js?v=1782076050"></script>
-<script src="/js/ui-ham.js?v=1782036437"></script>
+<script src="/js/ui-ham.js?v=1782076050"></script>
-<script src="/js/geo.js?v=1782036437"></script>
+<script src="/js/geo.js?v=1782076050"></script>
-<script src="/js/common.js?v=1782036437"></script>
+<script src="/js/common.js?v=1782076050"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/js/conditions.js?v=1782036437"></script>
+<script src="/js/conditions.js?v=1782076050"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -95,8 +95,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/js/spotsbandsandmap.js?v=1782036437"></script>
+<script src="/js/spotsbandsandmap.js?v=1782076049"></script>
-<script src="/js/map.js?v=1782036437"></script>
+<script src="/js/map.js?v=1782076049"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -116,8 +116,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/js/spotsbandsandmap.js?v=1782036437"></script>
+<script src="/js/spotsbandsandmap.js?v=1782076049"></script>
-<script src="/js/spots.js?v=1782036437"></script>
+<script src="/js/spots.js?v=1782076049"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -59,7 +59,7 @@
     </div>
 </div>
 
-<script src="/js/status.js?v=1782036437"></script>
+<script src="/js/status.js?v=1782076050"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");

webassets/apidocs/openapi.yml

@@ -11,7 +11,7 @@ info:
 
     Please note that the data coming out of Spothole is only as good as the data going in. People mis-hear and make typos when spotting callsigns all the time, and there are plenty of areas where Spothole's location data may be inaccurate. If you are doing something where accuracy is important, such as contesting, you should not rely on Spothole's data to fill in any gaps in your log.
 
-    Spothole's source code is located at https://git.ianrenton.com/ian/spothole and the README there provides setup instructions if you would like to run your own copy. A demonstration server of Spothole is located at https://spothole.app.
+    Spothole's source code is located at https://git.ianrenton.com/ian/spothole and the README there provides setup instructions if you would like to run your own copy. A demonstration server of Spothole is located at https://spothole.app. The README also contains some examples of how you could query the API of the demonstration server to integrate the data into your own apps.
 
     ## Changelog