w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 10:31:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Merge branch 'main' into 95-send-spots-to-xota

Browse Source 
# Conflicts:
#	README.md
#	config-example.yml
#	core/config.py
#	server/handlers/api/addspot.py
#	server/handlers/api/options.py
#	static/js/add-spot.js
#	static/js/bands.js
#	static/js/map.js
This commit is contained in:
Ian Renton
2026-08-02 07:47:40 +01:00
parent 3b5a4efb0b fffd230633
commit fa6c3c0a55
509 changed files with 1072 additions and 609 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/cache_utils.py
+5 -2
View File
@@ -8,8 +8,11 @@ from requests_cache import CachedSession
# of time has passed. This is used throughout Spothole to cache data that does not change
# rapidly. The ThreadSafeSession construct here protects it against some multithreading
# contention weirdness we sometimes used to see on startup where the cache was hammered
# pretty hard.
_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30))
# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type
# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote
# servers for data they've told us they can't provide.
_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30),
allowable_codes=(200, 400, 401, 403, 404))
_lock = threading.Lock()
core/config.py
+2
View File
@@ -25,6 +25,8 @@ WEB_UI_OPTIONS = config.get("web-ui-options", {})
API_ONLY_MODE = config.get("api-only-mode", False)
RECAPTCHA_SECRET_KEY = config.get("recaptcha-secret-key", "")
RECAPTCHA_SITE_KEY = config.get("recaptcha-site-key", "")
LOG_LEVEL = config.get("log-level", "INFO")
LOG_WEB_REQUESTS = config.get("log-web-requests", False)
# For ease of config, each spot provider owns its own config about whether it should be enabled by default in the web UI
# but for consistency we provide this to the front-end in web-ui-options because it has no impact outside of the web UI.
core/constants.py
+4 -3
View File
@@ -27,8 +27,8 @@ SIGS = [
SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"),
SIG(name="BOTA", comment_names=[], description="Beaches on the Air"),
SIG(name="KRMNPA", comment_names=["KRMNPA"], description="Keith Roget Memorial National Parks Award"),
SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"[A-Z]{2}\-\d{4}"),
SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2}R\-\d{4}"),
SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"LL[A-Z]{2}\-\d{4}"),
SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2,3}R\-\d{4}"),
SIG(name="Tiles", comment_names=[], description="Tiles on the Air", ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}"),
SIG(name="WAB", comment_names=["WAB"], description="Worked All Britain", ref_regex=r"[A-Z]{1,2}[0-9]{2}"),
SIG(name="WAI", comment_names=["WAI"], description="Worked All Ireland", ref_regex=r"[A-Z][0-9]{2}"),
@@ -104,5 +104,6 @@ PROPAGATION_MODES = {
"AU": "Aurora",
"MS": "Meteor scatter",
"RS": "Rain scatter",
"AS": "Aircraft scatter"
"AS": "Aircraft scatter",
"SAT": "Satellite"
}
core/lookup_helper.py
+84 -33
View File
@@ -5,13 +5,13 @@ import re
import urllib.parse
from datetime import timedelta
import requests
import xmltodict
from diskcache import Cache
from pyhamtools import LookupLib, Callinfo, callinfo
from pyhamtools.exceptions import APIKeyMissingError
from pyhamtools.frequency import freq_to_band
from pyhamtools.locator import latlong_to_locator
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests_cache import CachedSession
from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
@@ -143,13 +143,21 @@ class LookupHelper:
try:
logging.info("Downloading Country-files.com cty.plist...")
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
headers=HTTP_HEADERS).text
headers=HTTP_HEADERS)
with open(self._country_files_cty_plist_download_location, "w") as f:
f.write(response)
f.flush()
return True
if response.ok:
with open(self._country_files_cty_plist_download_location, "w") as f:
f.write(response.text)
f.flush()
return True
else:
logging.warning(f"HTTP {response.status_code} when downloading Country-files.com cty.plist.")
return False
except ConnectionError:
logging.warning(f"Connection error when downloading Clublog cty.xml.")
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when downloading Clublog cty.xml.")
except Exception as e:
logging.error("Exception when downloading Clublog cty.xml", e)
return False
@@ -161,13 +169,21 @@ class LookupHelper:
logging.info("Downloading dxcc.json...")
response = SEMI_STATIC_URL_DATA_CACHE.get(
"https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
headers=HTTP_HEADERS).text
headers=HTTP_HEADERS)
with open(self._dxcc_json_download_location, "w") as f:
f.write(response)
f.flush()
return True
if response.ok:
with open(self._dxcc_json_download_location, "w") as f:
f.write(response.text)
f.flush()
return True
else:
logging.warning(f"HTTP {response.status_code} when downloading dxcc.json.")
return False
except ConnectionError:
logging.warning(f"Connection error when downloading dxcc.json.")
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when downloading dxcc.json.")
except Exception as e:
logging.error("Exception when downloading dxcc.json", e)
return False
@@ -490,25 +506,47 @@ class LookupHelper:
# Try the call as given, then fall back to the base call (strips /P, /M etc.)
calls_to_try = [call]
home_call = callinfo.Callinfo.get_homecall(call)
if home_call != call:
calls_to_try.append(home_call)
try:
home_call = callinfo.Callinfo.get_homecall(call)
if home_call != call:
calls_to_try.append(home_call)
except ValueError:
logging.debug("Could not look up home call for callsign %s", call)
for lookup_call in calls_to_try:
try:
lookup_response = requests.get(
response = SEMI_STATIC_URL_DATA_CACHE.get(
self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
headers=HTTP_HEADERS, timeout=10).content
raw = xmltodict.parse(lookup_response).get("QRZDatabase", {}).get("Callsign")
if raw:
data = _normalize_qrz_data(raw)
self._qrz_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds
return data
headers=HTTP_HEADERS, timeout=10)
if response.ok:
qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
if qrz_response:
if "Callsign" in qrz_response:
data = _normalize_qrz_data(qrz_response.get("Callsign"))
self._qrz_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds
return data
elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
# Errors here are normally just "callsign not in database", no need to log that ourselves
# above debug level.
logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call,
qrz_response.get("Session").get("Error"))
elif not response.from_cache:
logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call)
elif not response.from_cache:
logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call)
except (KeyError, ValueError):
continue
except ConnectionError:
logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call)
continue
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call)
continue
except Exception:
logging.error("Exception when looking up QRZ data")
return None
logging.error("Exception when looking up callsign %s using QRZ", lookup_call, exc_info=True)
continue
# Not found in QRZ; cache None so we don't keep retrying
self._qrz_callsign_data_cache.add(call, None, expire=604800) # 1 week in seconds
@@ -546,23 +584,36 @@ class LookupHelper:
# Try the call as given, then fall back to the base call (strips /P, /M etc.)
calls_to_try = [call]
home_call = callinfo.Callinfo.get_homecall(call)
if home_call != call:
calls_to_try.append(home_call)
try:
home_call = callinfo.Callinfo.get_homecall(call)
if home_call != call:
calls_to_try.append(home_call)
except ValueError:
logging.debug("Could not look up home call for callsign %s", call)
for lookup_call in calls_to_try:
try:
lookup_data = SEMI_STATIC_URL_DATA_CACHE.get(
response = SEMI_STATIC_URL_DATA_CACHE.get(
self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS).content
data = xmltodict.parse(lookup_data)["HamQTH"]["search"]
self._hamqth_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds
return data
lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
if response.ok:
data = xmltodict.parse(response.content)["HamQTH"]["search"]
self._hamqth_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds
return data
elif not response.from_cache:
logging.warning("HTTP %d looking up callsign %s using HamQTH", response.status_code, lookup_call)
except (KeyError, ValueError):
continue
except ConnectionError:
logging.warning(f"Connection error when looking up callsign %s using HamQTH", lookup_call)
continue
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when looking up callsign %s using HamQTH", lookup_call)
continue
except Exception:
logging.error("Exception when looking up HamQTH data")
return None
logging.error("Exception when looking up callsign %s using HamQTH", lookup_call, exc_info=True)
continue
# Not found in HamQTH; cache None so we don't keep retrying
self._hamqth_callsign_data_cache.add(call, None, expire=604800) # 1 week in seconds
core/sig_utils.py
+173 -89
View File
@@ -2,6 +2,7 @@ import csv
import logging
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
from core.constants import SIGS, HTTP_HEADERS
@@ -11,6 +12,14 @@ from core.geo_utils import wab_wai_square_to_lat_lon
# file in Spothole and load it on startup.
with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
_DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
_WWFF_INDEX_CACHE = {}
_SIOTA_INDEX_CACHE = {}
_WOTA_INDEX_CACHE = {}
_ZLOTA_INDEX_CACHE = {}
_LLOTA_INDEX_CACHE = {}
def get_ref_regex_for_sig(sig):
@@ -45,92 +54,136 @@ def populate_sig_ref_info(sig_ref):
try:
if sig.upper() == "POTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
if not response.ok:
if response.ok:
data = response.json()
if data:
fullname = str(data["name"]) if "name" in data else None
if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
fullname = fullname + " " + data["parktypeDesc"]
sig_ref.name = fullname
sig_ref.url = "https://pota.app/#/park/" + ref_id
sig_ref.grid = data["grid6"] if "grid6" in data else None
sig_ref.latitude = data["latitude"] if "latitude" in data else None
sig_ref.longitude = data["longitude"] if "longitude" in data else None
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if data:
fullname = str(data["name"]) if "name" in data else None
if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
fullname = fullname + " " + data["parktypeDesc"]
sig_ref.name = fullname
sig_ref.url = "https://pota.app/#/park/" + ref_id
sig_ref.grid = data["grid6"] if "grid6" in data else None
sig_ref.latitude = data["latitude"] if "latitude" in data else None
sig_ref.longitude = data["longitude"] if "longitude" in data else None
elif sig.upper() == "SOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
headers=HTTP_HEADERS)
if not response.ok:
if response.ok:
data = response.json()
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
sig_ref.grid = data["locator"] if "locator" in data else None
sig_ref.latitude = data["latitude"] if "latitude" in data else None
sig_ref.longitude = data["longitude"] if "longitude" in data else None
sig_ref.activation_score = data["points"] if "points" in data else None
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
sig_ref.grid = data["locator"] if "locator" in data else None
sig_ref.latitude = data["latitude"] if "latitude" in data else None
sig_ref.longitude = data["longitude"] if "longitude" in data else None
sig_ref.activation_score = data["points"] if "points" in data else None
elif sig.upper() == "WWBOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
headers=HTTP_HEADERS)
if not response.ok:
if response.ok:
data = response.json()
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
sig_ref.grid = data["locator"] if "locator" in data else None
sig_ref.latitude = data["lat"] if "lat" in data else None
sig_ref.longitude = data["long"] if "long" in data else None
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
sig_ref.grid = data["locator"] if "locator" in data else None
sig_ref.latitude = data["lat"] if "lat" in data else None
sig_ref.longitude = data["long"] if "long" in data else None
elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
headers=HTTP_HEADERS)
if not response.ok:
if response.ok:
data = response.json()
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
sig_ref.grid = data["locator"] if "locator" in data else None
# For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
# the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
# and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
# just use the single point we have.
if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
"lat2") is not None and data.get("lng2") is not None:
sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
else:
sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
sig_ref.grid = data["locator"] if "locator" in data else None
sig_ref.latitude = data["latitude"] if "latitude" in data else None
sig_ref.longitude = data["longitude"] if "longitude" in data else None
elif sig.upper() == "WWFF":
wwff_response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
headers=HTTP_HEADERS)
if not wwff_response.ok:
logging.warning("HTTP %d looking up %s ref %s", wwff_response.status_code, sig, ref_id)
return sig_ref
wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_response.content.decode().splitlines())}
row = wwff_index.get(ref_id)
if row:
sig_ref.name = row["name"] if "name" in row else None
sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row["longitude"] != "-" else None
response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
headers=HTTP_HEADERS)
if response.ok:
global _WWFF_INDEX_CACHE
if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
# New data from WWFF, update our internal map
_WWFF_INDEX_CACHE = {row["reference"]: row for row in
csv.DictReader(response.content.decode().splitlines())}
row = _WWFF_INDEX_CACHE.get(ref_id)
if row:
sig_ref.name = row["name"] if "name" in row else None
sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[
"longitude"] != "-" else None
elif not response.from_cache:
logging.warning("WWFF database did not contain data for ref %s", ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "SIOTA":
siota_response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
headers=HTTP_HEADERS)
if not siota_response.ok:
logging.warning("HTTP %d looking up %s ref %s", siota_response.status_code, sig, ref_id)
return sig_ref
siota_index = {row["SILO_CODE"]: row for row in
csv.DictReader(siota_response.content.decode().splitlines())}
row = siota_index.get(ref_id)
if row:
sig_ref.name = row["NAME"] if "NAME" in row else None
sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
headers=HTTP_HEADERS)
if response.ok:
global _SIOTA_INDEX_CACHE
if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
# New data from SIOTA, update our internal map
_SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
csv.DictReader(response.content.decode().splitlines())}
row = _SIOTA_INDEX_CACHE.get(ref_id)
if row:
sig_ref.name = row["NAME"] if "NAME" in row else None
sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
elif not response.from_cache:
logging.warning("SIOTA database did not contain data for ref %s", ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
headers=HTTP_HEADERS)
if not response.ok:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if data:
for feature in data.get("features", []):
if feature["properties"]["wotaId"] == ref_id:
if response.ok:
data = response.json()
if data:
global _WOTA_INDEX_CACHE
if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
# New data from WOTA, update our internal map
_WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
data.get("features", [])}
feature = _WOTA_INDEX_CACHE.get(ref_id)
if feature:
sig_ref.name = feature["properties"]["title"]
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
# added to them
@@ -141,48 +194,68 @@ def populate_sig_ref_info(sig_ref):
sig_ref.grid = feature["properties"]["qthLocator"]
sig_ref.latitude = feature["geometry"]["coordinates"][1]
sig_ref.longitude = feature["geometry"]["coordinates"][0]
break
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "ZLOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
if not response.ok:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if isinstance(data, list):
for asset in data:
if asset["code"] == ref_id:
if response.ok:
data = response.json()
if isinstance(data, list):
global _ZLOTA_INDEX_CACHE
if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
# New data from ZLOTA, update our internal map
_ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
asset = _ZLOTA_INDEX_CACHE.get(ref_id)
if asset:
sig_ref.name = asset["name"]
sig_ref.url = "https://ontheair.nz/assets/ZLI_OT-030" + ref_id.replace("/", "_")
sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
try:
sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
except:
logging.debug("Invalid lat/lon received for reference")
sig_ref.latitude = asset["y"]
sig_ref.longitude = asset["x"]
break
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "BOTA":
if not sig_ref.name:
sig_ref.name = sig_ref.id
sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
elif sig.upper() == "LLOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
headers=HTTP_HEADERS)
if not response.ok:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
data = response.json() if response.ok else None
if isinstance(data, list):
for ref in data:
if ref["reference_code"] == ref_id:
if response.ok:
data = response.json()
if isinstance(data, list):
global _LLOTA_INDEX_CACHE
if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
# New data from LLOTA, update our internal map
_LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
ref = _LLOTA_INDEX_CACHE.get(ref_id)
if ref:
sig_ref.name = str(ref["name"])
sig_ref.url = "https://llota.app/list/ref/" + ref_id
sig_ref.grid = str(ref["grid_locator"])
ll = locator_to_latlong(sig_ref.grid)
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
break
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WWTOTA":
if not sig_ref.name:
sig_ref.name = sig_ref.id
sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + str(sig_ref.name)
elif sig.upper() == "TILES":
# Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
if not sig_ref.name:
@@ -193,6 +266,7 @@ def populate_sig_ref_info(sig_ref):
ll = locator_to_latlong(str(sig_ref.grid))
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
elif sig.upper() == "WAB" or sig.upper() == "WAI":
ll = wab_wai_square_to_lat_lon(ref_id)
if ll:
@@ -202,21 +276,31 @@ def populate_sig_ref_info(sig_ref):
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
except:
logging.debug("Invalid lat/lon received for reference")
logging.warning("Invalid lat/lon received for WAB/WAI reference")
elif sig.upper() == "DME":
# Zero-pad to 5 digits to match our source data
row = _DME_INDEX.get(ref_id.zfill(5))
if row:
sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get("LATITUD_ETRS89_REGCAN95") else None
sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get("LONGITUD_ETRS89_REGCAN95") else None
sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
"LATITUD_ETRS89_REGCAN95") else None
sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
"LONGITUD_ETRS89_REGCAN95") else None
if sig_ref.latitude and sig_ref.longitude:
try:
sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6)
except Exception:
logging.debug("Invalid lat/lon received for reference")
logging.warning("Invalid lat/lon received for DME reference")
else:
logging.warning("DME database did not contain data for ref %s", ref_id)
except ConnectionError:
logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when looking up sig_ref info for " + sig + " ref " + ref_id)
except Exception:
logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
return sig_ref
core/utils.py
+8 -5
View File
@@ -1,8 +1,11 @@
def serialize_everything(obj):
"""Convert objects to serialisable things. Used by JSON serialiser as a default when it encounters unserializable things.
Just converts objects to dict. Try to avoid doing anything clever here when serialising spots, because we also need
to receive spots without complex handling."""
return obj.__dict__
import simplejson
def safe_json_dumps(obj):
"""Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
which are invalid in JSON."""
return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
def empty_queue(q):