w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 02:21:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 1

Browse Source
This commit is contained in:
Ian Renton
2026-07-31 14:12:55 +01:00
parent 266468f938
commit d26ddff7d1
17 changed files with 277 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/sig_utils.py
+39 -49
View File
@@ -4,23 +4,16 @@ import logging
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
from core.constants import SIGS, HTTP_HEADERS
from core.data_store import DATA_STORE
from core.geo_utils import wab_wai_square_to_lat_lon
from core.url_data_cache import URL_DATA_CACHE
# Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data
# file in Spothole and load it on startup.
with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
_DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
_WWFF_INDEX_CACHE = {}
_SIOTA_INDEX_CACHE = {}
_WOTA_INDEX_CACHE = {}
_ZLOTA_INDEX_CACHE = {}
_LLOTA_INDEX_CACHE = {}
for row in csv.DictReader(_f, delimiter=";"):
DATA_STORE.sigrefs_dme.add(row["COD_INE"][:5], row)
def get_ref_regex_for_sig(sig):
"""Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
@@ -53,7 +46,7 @@ def populate_sig_ref_info(sig_ref):
ref_id = sig_ref.id
try:
if sig.upper() == "POTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
response = URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
@@ -71,7 +64,7 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "SOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
response = URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
@@ -88,7 +81,7 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WWBOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
response = URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
@@ -104,7 +97,7 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
@@ -130,15 +123,14 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WWFF":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
response = URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
headers=HTTP_HEADERS)
if response.ok:
global _WWFF_INDEX_CACHE
if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
if not bool(DATA_STORE.sigrefs_wwff) or not response.from_cache:
# New data from WWFF, update our internal map
_WWFF_INDEX_CACHE = {row["reference"]: row for row in
csv.DictReader(response.content.decode().splitlines())}
row = _WWFF_INDEX_CACHE.get(ref_id)
for row in csv.DictReader(response.content.decode().splitlines()):
DATA_STORE.sigrefs_wwff.add(row["reference"], row)
row = DATA_STORE.sigrefs_wwff.get(ref_id)
if row:
sig_ref.name = row["name"] if "name" in row else None
sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
@@ -152,15 +144,14 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "SIOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
response = URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
headers=HTTP_HEADERS)
if response.ok:
global _SIOTA_INDEX_CACHE
if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
if not bool(DATA_STORE.sigrefs_siota) or not response.from_cache:
# New data from SIOTA, update our internal map
_SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
csv.DictReader(response.content.decode().splitlines())}
row = _SIOTA_INDEX_CACHE.get(ref_id)
for row in csv.DictReader(response.content.decode().splitlines()):
DATA_STORE.sigrefs_siota.add(row["SILO_CODE"], row)
row = DATA_STORE.sigrefs_siota.get(ref_id)
if row:
sig_ref.name = row["NAME"] if "NAME" in row else None
sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
@@ -172,17 +163,16 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
response = URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
global _WOTA_INDEX_CACHE
if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
if not bool(DATA_STORE.sigrefs_wota) or not response.from_cache:
# New data from WOTA, update our internal map
_WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
data.get("features", [])}
feature = _WOTA_INDEX_CACHE.get(ref_id)
for feature in data.get("features", []):
DATA_STORE.sigrefs_wota.add(feature["properties"]["wotaId"], feature)
feature = DATA_STORE.sigrefs_wota.get(ref_id)
if feature:
sig_ref.name = feature["properties"]["title"]
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -200,24 +190,24 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "ZLOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
response = URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if isinstance(data, list):
global _ZLOTA_INDEX_CACHE
if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
if not bool(DATA_STORE.sigrefs_zlota) or not response.from_cache:
# New data from ZLOTA, update our internal map
_ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
asset = _ZLOTA_INDEX_CACHE.get(ref_id)
if asset:
sig_ref.name = asset["name"]
for ref in data:
DATA_STORE.sigrefs_zlota.add(ref["code"], ref)
ref = DATA_STORE.sigrefs_zlota.get(ref_id)
if ref:
sig_ref.name = ref["name"]
sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
try:
sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
sig_ref.grid = latlong_to_locator(ref["y"], ref["x"], 6)
except:
logging.debug("Invalid lat/lon received for reference")
sig_ref.latitude = asset["y"]
sig_ref.longitude = asset["x"]
sig_ref.latitude = ref["y"]
sig_ref.longitude = ref["x"]
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
@@ -229,16 +219,16 @@ def populate_sig_ref_info(sig_ref):
sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
elif sig.upper() == "LLOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
response = URL_DATA_CACHE.get("https://llota.app/api/public/references",
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if isinstance(data, list):
global _LLOTA_INDEX_CACHE
if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
if not bool(DATA_STORE.sigrefs_llota) or not response.from_cache:
# New data from LLOTA, update our internal map
_LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
ref = _LLOTA_INDEX_CACHE.get(ref_id)
for ref in data:
DATA_STORE.sigrefs_llota.add(ref["reference_code"], ref)
ref = DATA_STORE.sigrefs_llota.get(ref_id)
if ref:
sig_ref.name = str(ref["name"])
sig_ref.url = "https://llota.app/list/ref/" + ref_id
@@ -280,7 +270,7 @@ def populate_sig_ref_info(sig_ref):
elif sig.upper() == "DME":
# Zero-pad to 5 digits to match our source data
row = _DME_INDEX.get(ref_id.zfill(5))
row = DATA_STORE.sigrefs_dme.get(ref_id.zfill(5))
if row:
sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(