Add support for DME. Closes #114.

alertproviders/parksnpeaks.py

@@ -33,18 +33,24 @@ class ParksNPeaks(HTTPAlertProvider):
             start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(
                 tzinfo=pytz.UTC).timestamp()
 
+            sigrefs = []
+            # PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list,
+            # so mask this out if we got it.
+            if sig != "QRP":
+                sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)]
+
             # Convert to our alert format
             alert = Alert(source=self.name,
                           source_id=source_alert["alID"],
                           dx_calls=[source_alert["CallSign"].upper()],
                           freqs_modes=source_alert["Freq"] + " " + source_alert["MODE"],
                           comment=source_alert["Comments"],
-                          sig_refs=[SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)],
+                          sig_refs=sigrefs,
                           start_time=start_time,
                           is_dxpedition=False)
 
             # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-            if sig and sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA"]:
+            if sig and sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
                 logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!")
 
             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to

core/cache_utils.py

@@ -1,3 +1,4 @@
+import threading
 from datetime import timedelta
 
 from requests_cache import CachedSession
@@ -5,6 +6,19 @@ from requests_cache import CachedSession
 # Cache for "semi-static" data such as the locations of parks, CSVs of reference lists, etc.
 # This has an expiry time of 30 days, so will re-request from the source after that amount
 # of time has passed. This is used throughout Spothole to cache data that does not change
-# rapidly.
-SEMI_STATIC_URL_DATA_CACHE = CachedSession("cache/semi_static_url_data_cache",
-                                           expire_after=timedelta(days=30))
+# rapidly. The ThreadSafeSession construct here protects it against some multithreading
+# contention weirdness we sometimes used to see on startup where the cache was hammered
+# pretty hard.
+_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30))
+_lock = threading.Lock()
+
+
+class _ThreadSafeSession:
+    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads."""
+
+    def get(self, *args, **kwargs):
+        with _lock:
+            return _session.get(*args, **kwargs)
+
+
+SEMI_STATIC_URL_DATA_CACHE = _ThreadSafeSession()

core/constants.py

@@ -32,6 +32,7 @@ SIGS = [
     SIG(name="Tiles", description="Tiles on the Air", ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}"),
     SIG(name="WAB", description="Worked All Britain", ref_regex=r"[A-Z]{1,2}[0-9]{2}"),
     SIG(name="WAI", description="Worked All Ireland", ref_regex=r"[A-Z][0-9]{2}"),
+    SIG(name="DME", description="Diplomas de Municipios Españoles", ref_regex=r"\d{4,5}"),
     SIG(name="TOTA", description="Toilets on the Air", ref_regex=r"T\-[0-9]{2}")
 ]
 

core/sig_utils.py

@@ -7,6 +7,9 @@ from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import SIGS, HTTP_HEADERS
 from core.geo_utils import wab_wai_square_to_lat_lon
 
+with open("datafiles/dme-geodata.csv", encoding="utf-8") as _f:
+    _DME_INDEX = {row["dme"]: row for row in csv.DictReader(_f)}
+
 
 def get_ref_regex_for_sig(sig):
     """Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
@@ -29,7 +32,10 @@ def populate_sig_ref_info(sig_ref):
     ref_id = sig_ref.id
     try:
         if sig.upper() == "POTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS).json()
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
             if data:
                 fullname = str(data["name"]) if "name" in data else None
                 if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
@@ -40,8 +46,11 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.latitude = data["latitude"] if "latitude" in data else None
                 sig_ref.longitude = data["longitude"] if "longitude" in data else None
         elif sig.upper() == "SOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
-                                                  headers=HTTP_HEADERS).json()
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
+                                                      headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
             if data:
                 sig_ref.name = data["name"] if "name" in data else None
                 sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
@@ -50,8 +59,11 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.longitude = data["longitude"] if "longitude" in data else None
                 sig_ref.activation_score = data["points"] if "points" in data else None
         elif sig.upper() == "WWBOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
-                                                  headers=HTTP_HEADERS).json()
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
+                                                      headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
             if data:
                 sig_ref.name = data["name"] if "name" in data else None
                 sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
@@ -59,8 +71,11 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.latitude = data["lat"] if "lat" in data else None
                 sig_ref.longitude = data["long"] if "long" in data else None
         elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
-                                                  headers=HTTP_HEADERS).json()
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
+                                                      headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
             if data:
                 sig_ref.name = data["name"] if "name" in data else None
                 sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
@@ -68,9 +83,12 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.latitude = data["latitude"] if "latitude" in data else None
                 sig_ref.longitude = data["longitude"] if "longitude" in data else None
         elif sig.upper() == "WWFF":
-            wwff_csv_data = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
+            wwff_response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
                                                            headers=HTTP_HEADERS)
-            wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_csv_data.content.decode().splitlines())}
+            if not wwff_response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", wwff_response.status_code, sig, ref_id)
+                return sig_ref
+            wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_response.content.decode().splitlines())}
             row = wwff_index.get(ref_id)
             if row:
                 sig_ref.name = row["name"] if "name" in row else None
@@ -79,10 +97,13 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
                 sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row["longitude"] != "-" else None
         elif sig.upper() == "SIOTA":
-            siota_csv_data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
+            siota_response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
                                                             headers=HTTP_HEADERS)
+            if not siota_response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", siota_response.status_code, sig, ref_id)
+                return sig_ref
             siota_index = {row["SILO_CODE"]: row for row in
-                           csv.DictReader(siota_csv_data.content.decode().splitlines())}
+                           csv.DictReader(siota_response.content.decode().splitlines())}
             row = siota_index.get(ref_id)
             if row:
                 sig_ref.name = row["NAME"] if "NAME" in row else None
@@ -90,10 +111,13 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
                 sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
         elif sig.upper() == "WOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
-                                                  headers=HTTP_HEADERS).json()
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
+                                                      headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
             if data:
-                for feature in data["features"]:
+                for feature in data.get("features", []):
                     if feature["properties"]["wotaId"] == ref_id:
                         sig_ref.name = feature["properties"]["title"]
                         # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -107,8 +131,11 @@ def populate_sig_ref_info(sig_ref):
                         sig_ref.longitude = feature["geometry"]["coordinates"][0]
                         break
         elif sig.upper() == "ZLOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS).json()
-            if data:
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
+            if isinstance(data, list):
                 for asset in data:
                     if asset["code"] == ref_id:
                         sig_ref.name = asset["name"]
@@ -125,9 +152,12 @@ def populate_sig_ref_info(sig_ref):
                 sig_ref.name = sig_ref.id
             sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
         elif sig.upper() == "LLOTA":
-            data = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
-                                                  headers=HTTP_HEADERS).json()
-            if data:
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
+                                                      headers=HTTP_HEADERS)
+            if not response.ok:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+            data = response.json() if response.ok else None
+            if isinstance(data, list):
                 for ref in data:
                     if ref["reference_code"] == ref_id:
                         sig_ref.name = str(ref["name"])
@@ -161,8 +191,19 @@ def populate_sig_ref_info(sig_ref):
                     sig_ref.longitude = ll[1]
                 except:
                     logging.debug("Invalid lat/lon received for reference")
-    except:
-        logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id + ".")
+        elif sig.upper() == "DME":
+            row = _DME_INDEX.get(ref_id)
+            if row:
+                sig_ref.name = row["municipio"] + ", " + row["provincia"]
+                sig_ref.latitude = float(row["lat"]) if row.get("lat") else None
+                sig_ref.longitude = float(row["lon"]) if row.get("lon") else None
+                if sig_ref.latitude and sig_ref.longitude:
+                    try:
+                        sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6)
+                    except Exception:
+                        logging.debug("Invalid lat/lon received for reference")
+    except Exception:
+        logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
     return sig_ref
 
 

data/spot.py

@@ -263,7 +263,7 @@ class Spot:
                 # If so, add that to the sig_refs list for this spot.
                 ref_regex = get_ref_regex_for_sig(found_sig)
                 if ref_regex:
-                    ref_matches = re.finditer(r"(^|\W)" + found_sig + r"($|\W)(" + ref_regex + r")($|\W)", self.comment,
+                    ref_matches = re.finditer(r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)", self.comment,
                                               re.IGNORECASE)
                     for ref_match in ref_matches:
                         self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig))
@@ -290,23 +290,23 @@ class Spot:
         if self.sig_refs and len(self.sig_refs) > 0 and not self.sig:
             self.sig = self.sig_refs[0].sig
 
-        # Parse "de_grid<prop_mode>dx_grid" structures from the comment, e.g. "JN61ES<ES>JM56XT" or "JO02GQ<>KN17LG".
+        # Parse "de_grid<prop_mode>dx_grid" structures from the comment, e.g. "JN61ES(ES)JM56XT" or "JO02GQ<>KN17LG".
         # These are common on cluster spots and can provide grid references in preference to e.g. QRZ lookup, as well as
-        # being the only source we have for propagation mode.
+        # being the only source we have for propagation mode. Brace for nightmare regex from hell.
         if self.comment:
             grid_mode_grid_match = re.search(
-                r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)<([^>]*)>([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
+                r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
                 self.comment)
             if grid_mode_grid_match:
                 # regex matches, so extract grids:
                 if not self.de_grid:
                     self.de_grid = grid_mode_grid_match.group(1).upper()
                 if not self.dx_grid:
-                    self.dx_grid = grid_mode_grid_match.group(3).upper()
+                    self.dx_grid = grid_mode_grid_match.group(4).upper()
                     self.dx_location_source = "SPOT"
 
-                # And extract propagation mode:
-                mode_tag = grid_mode_grid_match.group(2).upper()
+                # And extract propagation mode (group 2 for <...>, group 3 for (...)):
+                mode_tag = (grid_mode_grid_match.group(2) or grid_mode_grid_match.group(3) or "").upper()
                 if mode_tag and not self.propagation_mode:
                     if mode_tag in PROPAGATION_MODES:
                         self.propagation_mode = PROPAGATION_MODES[mode_tag]

server/handlers/api/options.py

@@ -42,7 +42,7 @@ class APIOptionsHandler(tornado.web.RequestHandler):
                    "alert_sources": list(
                        map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"]))),
                    "continents": CONTINENTS,
-                   "propagation_modes": PROPAGATION_MODES.values(),
+                   "propagation_modes": list(PROPAGATION_MODES.values()),
                    "max_spot_age": MAX_SPOT_AGE,
                    "spot_allowed": ALLOW_SPOTTING}
         # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from

spotproviders/parksnpeaks.py

@@ -57,7 +57,7 @@ class ParksNPeaks(HTTPSpotProvider):
                         sig_refs[0].name = source_spot["actLocation"]
 
                     # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-                    if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA"]:
+                    if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA"]:
                         logging.warning("PNP spot found with sig " + sig + ", developer needs to add support for this!")
 
                 # Add new spot to the list

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/js/add-spot.js?v=1782036437"></script>
+<script src="/js/add-spot.js?v=1782076701"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -75,7 +75,7 @@
 
 </div>
 
-<script src="/js/alerts.js?v=1782036437"></script>
+<script src="/js/alerts.js?v=1782076701"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -77,8 +77,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/js/spotsbandsandmap.js?v=1782036437"></script>
-<script src="/js/bands.js?v=1782036437"></script>
+<script src="/js/spotsbandsandmap.js?v=1782076701"></script>
+<script src="/js/bands.js?v=1782076701"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/css/style.css?v=1782036437" type="text/css">
+<link rel="stylesheet" href="/css/style.css?v=1782076701" type="text/css">
 <link href="/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
 <script src="/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
 <script src="/vendor/js/tinycolor2-1.6.0.min.js"></script>
 
-<script src="/js/utils.js?v=1782036437"></script>
-<script src="/js/ui-ham.js?v=1782036437"></script>
-<script src="/js/geo.js?v=1782036437"></script>
-<script src="/js/common.js?v=1782036437"></script>
+<script src="/js/utils.js?v=1782076701"></script>
+<script src="/js/ui-ham.js?v=1782076701"></script>
+<script src="/js/geo.js?v=1782076701"></script>
+<script src="/js/common.js?v=1782076701"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/js/conditions.js?v=1782036437"></script>
+<script src="/js/conditions.js?v=1782076701"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -95,8 +95,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/js/spotsbandsandmap.js?v=1782036437"></script>
-<script src="/js/map.js?v=1782036437"></script>
+<script src="/js/spotsbandsandmap.js?v=1782076701"></script>
+<script src="/js/map.js?v=1782076701"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -116,8 +116,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/js/spotsbandsandmap.js?v=1782036437"></script>
-<script src="/js/spots.js?v=1782036437"></script>
+<script src="/js/spotsbandsandmap.js?v=1782076701"></script>
+<script src="/js/spots.js?v=1782076701"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -59,7 +59,7 @@
     </div>
 </div>
 
-<script src="/js/status.js?v=1782036437"></script>
+<script src="/js/status.js?v=1782076701"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");

webassets/apidocs/openapi.yml

@@ -11,7 +11,7 @@ info:
 
     Please note that the data coming out of Spothole is only as good as the data going in. People mis-hear and make typos when spotting callsigns all the time, and there are plenty of areas where Spothole's location data may be inaccurate. If you are doing something where accuracy is important, such as contesting, you should not rely on Spothole's data to fill in any gaps in your log.
 
-    Spothole's source code is located at https://git.ianrenton.com/ian/spothole and the README there provides setup instructions if you would like to run your own copy. A demonstration server of Spothole is located at https://spothole.app.
+    Spothole's source code is located at https://git.ianrenton.com/ian/spothole and the README there provides setup instructions if you would like to run your own copy. A demonstration server of Spothole is located at https://spothole.app. The README also contains some examples of how you could query the API of the demonstration server to integrate the data into your own apps.
 
     ## Changelog
     

webassets/js/ui-ham.js

@@ -335,6 +335,7 @@ const SIG_ICONS = {
     "WWTOTA": "fa-tower-observation",
     "WAB": "fa-table-cells-large",
     "WAI": "fa-table-cells-large",
+    "DME": "fa-building",
     "Tiles": "fa-square",
     "TOTA": "fa-toilet"
 }