w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-06-24 05:35:10 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Merge branch 'main' into 95-send-spots-to-xota

Browse Source
This commit is contained in:
Ian Renton
2026-06-21 22:08:25 +01:00
parent 7a34526a91 615e1183a8
commit 59fa6500eb
12 changed files with 89 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
alertproviders/parksnpeaks.py
View File

@@ -33,18 +33,24 @@ class ParksNPeaks(HTTPAlertProvider):
start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace( start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp() tzinfo=pytz.UTC).timestamp()
sigrefs = []
# PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list,
# so mask this out if we got it.
if sig != "QRP":
sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)]
# Convert to our alert format # Convert to our alert format
alert = Alert(source=self.name, alert = Alert(source=self.name,
source_id=source_alert["alID"], source_id=source_alert["alID"],
dx_calls=[source_alert["CallSign"].upper()], dx_calls=[source_alert["CallSign"].upper()],
freqs_modes=source_alert["Freq"] + " " + source_alert["MODE"], freqs_modes=source_alert["Freq"] + " " + source_alert["MODE"],
comment=source_alert["Comments"], comment=source_alert["Comments"],
sig_refs=[SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)], sig_refs=sigrefs,
start_time=start_time, start_time=start_time,
is_dxpedition=False) is_dxpedition=False)
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig and sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA"]: if sig and sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!") logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!")
# If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to

20
core/cache_utils.py
View File

@@ -1,3 +1,4 @@
import threading
from datetime import timedelta from datetime import timedelta
from requests_cache import CachedSession from requests_cache import CachedSession
@@ -5,6 +6,19 @@ from requests_cache import CachedSession
# Cache for "semi-static" data such as the locations of parks, CSVs of reference lists, etc. # Cache for "semi-static" data such as the locations of parks, CSVs of reference lists, etc.
# This has an expiry time of 30 days, so will re-request from the source after that amount # This has an expiry time of 30 days, so will re-request from the source after that amount
# of time has passed. This is used throughout Spothole to cache data that does not change # of time has passed. This is used throughout Spothole to cache data that does not change
# rapidly. # rapidly. The ThreadSafeSession construct here protects it against some multithreading
SEMI_STATIC_URL_DATA_CACHE = CachedSession("cache/semi_static_url_data_cache", # contention weirdness we sometimes used to see on startup where the cache was hammered
expire_after=timedelta(days=30)) # pretty hard.
_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30))
_lock = threading.Lock()
class _ThreadSafeSession:
"""Wraps CachedSession with a lock to prevent concurrent SQLite access across threads."""
def get(self, *args, **kwargs):
with _lock:
return _session.get(*args, **kwargs)
SEMI_STATIC_URL_DATA_CACHE = _ThreadSafeSession()

69
core/sig_utils.py
View File

@@ -29,7 +29,10 @@ def populate_sig_ref_info(sig_ref):
ref_id = sig_ref.id ref_id = sig_ref.id
try: try:
if sig.upper() == "POTA": if sig.upper() == "POTA":
data = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS).json() response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
if not response.ok:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if data: if data:
fullname = str(data["name"]) if "name" in data else None fullname = str(data["name"]) if "name" in data else None
if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "": if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
@@ -40,8 +43,11 @@ def populate_sig_ref_info(sig_ref):
sig_ref.latitude = data["latitude"] if "latitude" in data else None sig_ref.latitude = data["latitude"] if "latitude" in data else None
sig_ref.longitude = data["longitude"] if "longitude" in data else None sig_ref.longitude = data["longitude"] if "longitude" in data else None
elif sig.upper() == "SOTA": elif sig.upper() == "SOTA":
data = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id, response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
headers=HTTP_HEADERS).json() headers=HTTP_HEADERS)
if not response.ok:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if data: if data:
sig_ref.name = data["name"] if "name" in data else None sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
@@ -50,8 +56,11 @@ def populate_sig_ref_info(sig_ref):
sig_ref.longitude = data["longitude"] if "longitude" in data else None sig_ref.longitude = data["longitude"] if "longitude" in data else None
sig_ref.activation_score = data["points"] if "points" in data else None sig_ref.activation_score = data["points"] if "points" in data else None
elif sig.upper() == "WWBOTA": elif sig.upper() == "WWBOTA":
data = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id, response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
headers=HTTP_HEADERS).json() headers=HTTP_HEADERS)
if not response.ok:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if data: if data:
sig_ref.name = data["name"] if "name" in data else None sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
@@ -59,8 +68,11 @@ def populate_sig_ref_info(sig_ref):
sig_ref.latitude = data["lat"] if "lat" in data else None sig_ref.latitude = data["lat"] if "lat" in data else None
sig_ref.longitude = data["long"] if "long" in data else None sig_ref.longitude = data["long"] if "long" in data else None
elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA": elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id, response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
headers=HTTP_HEADERS).json() headers=HTTP_HEADERS)
if not response.ok:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if data: if data:
sig_ref.name = data["name"] if "name" in data else None sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
@@ -68,9 +80,12 @@ def populate_sig_ref_info(sig_ref):
sig_ref.latitude = data["latitude"] if "latitude" in data else None sig_ref.latitude = data["latitude"] if "latitude" in data else None
sig_ref.longitude = data["longitude"] if "longitude" in data else None sig_ref.longitude = data["longitude"] if "longitude" in data else None
elif sig.upper() == "WWFF": elif sig.upper() == "WWFF":
wwff_csv_data = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv", wwff_response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
headers=HTTP_HEADERS) headers=HTTP_HEADERS)
wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_csv_data.content.decode().splitlines())} if not wwff_response.ok:
logging.warning("HTTP %d looking up %s ref %s", wwff_response.status_code, sig, ref_id)
return sig_ref
wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_response.content.decode().splitlines())}
row = wwff_index.get(ref_id) row = wwff_index.get(ref_id)
if row: if row:
sig_ref.name = row["name"] if "name" in row else None sig_ref.name = row["name"] if "name" in row else None
@@ -79,10 +94,13 @@ def populate_sig_ref_info(sig_ref):
sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row["longitude"] != "-" else None sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row["longitude"] != "-" else None
elif sig.upper() == "SIOTA": elif sig.upper() == "SIOTA":
siota_csv_data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv", siota_response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
headers=HTTP_HEADERS) headers=HTTP_HEADERS)
if not siota_response.ok:
logging.warning("HTTP %d looking up %s ref %s", siota_response.status_code, sig, ref_id)
return sig_ref
siota_index = {row["SILO_CODE"]: row for row in siota_index = {row["SILO_CODE"]: row for row in
csv.DictReader(siota_csv_data.content.decode().splitlines())} csv.DictReader(siota_response.content.decode().splitlines())}
row = siota_index.get(ref_id) row = siota_index.get(ref_id)
if row: if row:
sig_ref.name = row["NAME"] if "NAME" in row else None sig_ref.name = row["NAME"] if "NAME" in row else None
@@ -90,10 +108,13 @@ def populate_sig_ref_info(sig_ref):
sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
elif sig.upper() == "WOTA": elif sig.upper() == "WOTA":
data = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json", response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
headers=HTTP_HEADERS).json() headers=HTTP_HEADERS)
if not response.ok:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if data: if data:
for feature in data["features"]: for feature in data.get("features", []):
if feature["properties"]["wotaId"] == ref_id: if feature["properties"]["wotaId"] == ref_id:
sig_ref.name = feature["properties"]["title"] sig_ref.name = feature["properties"]["title"]
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -107,8 +128,11 @@ def populate_sig_ref_info(sig_ref):
sig_ref.longitude = feature["geometry"]["coordinates"][0] sig_ref.longitude = feature["geometry"]["coordinates"][0]
break break
elif sig.upper() == "ZLOTA": elif sig.upper() == "ZLOTA":
data = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS).json() response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
if data: if not response.ok:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if isinstance(data, list):
for asset in data: for asset in data:
if asset["code"] == ref_id: if asset["code"] == ref_id:
sig_ref.name = asset["name"] sig_ref.name = asset["name"]
@@ -125,9 +149,12 @@ def populate_sig_ref_info(sig_ref):
sig_ref.name = sig_ref.id sig_ref.name = sig_ref.id
sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-") sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
elif sig.upper() == "LLOTA": elif sig.upper() == "LLOTA":
data = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references", response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
headers=HTTP_HEADERS).json() headers=HTTP_HEADERS)
if data: if not response.ok:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if isinstance(data, list):
for ref in data: for ref in data:
if ref["reference_code"] == ref_id: if ref["reference_code"] == ref_id:
sig_ref.name = str(ref["name"]) sig_ref.name = str(ref["name"])
@@ -161,8 +188,8 @@ def populate_sig_ref_info(sig_ref):
sig_ref.longitude = ll[1] sig_ref.longitude = ll[1]
except: except:
logging.debug("Invalid lat/lon received for reference") logging.debug("Invalid lat/lon received for reference")
except: except Exception:
logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id + ".") logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
return sig_ref return sig_ref

2
spotproviders/parksnpeaks.py
View File

@@ -61,7 +61,7 @@ class ParksNPeaks(HTTPSpotProvider):
sig_refs[0].name = source_spot["actLocation"] sig_refs[0].name = source_spot["actLocation"]
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA"]: if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA"]:
logging.warning("PNP spot found with sig " + sig + ", developer needs to add support for this!") logging.warning("PNP spot found with sig " + sig + ", developer needs to add support for this!")
# Add new spot to the list # Add new spot to the list

2
templates/add_spot.html
View File

@@ -76,7 +76,7 @@
</div> </div>
<script src="/js/add-spot.js?v=1782073685"></script> <script src="/js/add-spot.js?v=1782076050"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-add-spot").addClass("active"); $("#nav-link-add-spot").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

2
templates/alerts.html
View File

@@ -75,7 +75,7 @@
</div> </div>
<script src="/js/alerts.js?v=1782073685"></script> <script src="/js/alerts.js?v=1782076050"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-alerts").addClass("active"); $("#nav-link-alerts").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

4
templates/bands.html
View File

@@ -77,8 +77,8 @@
<script> <script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %}; let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script> </script>
<script src="/js/spotsbandsandmap.js?v=1782073685"></script> <script src="/js/spotsbandsandmap.js?v=1782076050"></script>
<script src="/js/bands.js?v=1782073685"></script> <script src="/js/bands.js?v=1782076050"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-bands").addClass("active"); $("#nav-link-bands").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

10
templates/base.html
View File

@@ -1,6 +1,6 @@
{% extends "skeleton.html" %} {% extends "skeleton.html" %}
{% block head_extra %} {% block head_extra %}
<link rel="stylesheet" href="/css/style.css?v=1782073685" type="text/css"> <link rel="stylesheet" href="/css/style.css?v=1782076050" type="text/css">
<link href="/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet"> <link href="/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
<link href="/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet"> <link href="/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
<link href="/vendor/css/solid-6.7.2.min.css" rel="stylesheet"> <link href="/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
<script src="/vendor/js/bootstrap-5.3.8.bundle.min.js"></script> <script src="/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
<script src="/vendor/js/tinycolor2-1.6.0.min.js"></script> <script src="/vendor/js/tinycolor2-1.6.0.min.js"></script>
<script src="/js/utils.js?v=1782073685"></script> <script src="/js/utils.js?v=1782076050"></script>
<script src="/js/ui-ham.js?v=1782073685"></script> <script src="/js/ui-ham.js?v=1782076050"></script>
<script src="/js/geo.js?v=1782073685"></script> <script src="/js/geo.js?v=1782076050"></script>
<script src="/js/common.js?v=1782073685"></script> <script src="/js/common.js?v=1782076050"></script>
{% end %} {% end %}
{% block body %} {% block body %}
<div class="container"> <div class="container">

2
templates/conditions.html
View File

@@ -284,7 +284,7 @@
</div> </div>
<script src="/vendor/js/chart-4.4.9.umd.min.js"></script> <script src="/vendor/js/chart-4.4.9.umd.min.js"></script>
<script src="/js/conditions.js?v=1782073685"></script> <script src="/js/conditions.js?v=1782076050"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-conditions").addClass("active"); $("#nav-link-conditions").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

4
templates/map.html
View File

@@ -95,8 +95,8 @@
<script> <script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %}; let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script> </script>
<script src="/js/spotsbandsandmap.js?v=1782073685"></script> <script src="/js/spotsbandsandmap.js?v=1782076049"></script>
<script src="/js/map.js?v=1782073685"></script> <script src="/js/map.js?v=1782076049"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-map").addClass("active"); $("#nav-link-map").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

4
templates/spots.html
View File

@@ -116,8 +116,8 @@
<script> <script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %}; let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script> </script>
<script src="/js/spotsbandsandmap.js?v=1782073685"></script> <script src="/js/spotsbandsandmap.js?v=1782076049"></script>
<script src="/js/spots.js?v=1782073685"></script> <script src="/js/spots.js?v=1782076049"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-spots").addClass("active"); $("#nav-link-spots").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

2
templates/status.html
View File

@@ -59,7 +59,7 @@
</div> </div>
</div> </div>
<script src="/js/status.js?v=1782073685"></script> <script src="/js/status.js?v=1782076050"></script>
<script> <script>
$(document).ready(function () { $(document).ready(function () {
$("#nav-link-status").addClass("active"); $("#nav-link-status").addClass("active");