From fffd230633f4eba21b629caaedf38e34bede50af Mon Sep 17 00:00:00 2001 From: Ian Renton Date: Sun, 26 Jul 2026 17:10:21 +0100 Subject: [PATCH] Cache data for the programmes for which we must fetch the whole set of references, and avoid transforming CSV/JSON to dict every time. --- core/sig_utils.py | 101 ++++++++++++++++++++++++-------------- templates/add_spot.html | 2 +- templates/alerts.html | 2 +- templates/bands.html | 4 +- templates/base.html | 10 ++-- templates/conditions.html | 2 +- templates/map.html | 4 +- templates/spots.html | 4 +- templates/status.html | 2 +- 9 files changed, 78 insertions(+), 53 deletions(-) diff --git a/core/sig_utils.py b/core/sig_utils.py index 38c3b23..258d5ac 100644 --- a/core/sig_utils.py +++ b/core/sig_utils.py @@ -12,6 +12,14 @@ from core.geo_utils import wab_wai_square_to_lat_lon # file in Spothole and load it on startup. with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f: _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")} +# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single +# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came +# live from the internet, to avoid repopulating them every time we pull the same data from the cache. +_WWFF_INDEX_CACHE = {} +_SIOTA_INDEX_CACHE = {} +_WOTA_INDEX_CACHE = {} +_ZLOTA_INDEX_CACHE = {} +_LLOTA_INDEX_CACHE = {} def get_ref_regex_for_sig(sig): @@ -125,8 +133,12 @@ def populate_sig_ref_info(sig_ref): response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv", headers=HTTP_HEADERS) if response.ok: - wwff_index = {row["reference"]: row for row in csv.DictReader(response.content.decode().splitlines())} - row = wwff_index.get(ref_id) + global _WWFF_INDEX_CACHE + if not bool(_WWFF_INDEX_CACHE) or not response.from_cache: + # New data from WWFF, update our internal map + _WWFF_INDEX_CACHE = {row["reference"]: row for row in + csv.DictReader(response.content.decode().splitlines())} + row = _WWFF_INDEX_CACHE.get(ref_id) if row: sig_ref.name = row["name"] if "name" in row else None sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id @@ -143,9 +155,12 @@ def populate_sig_ref_info(sig_ref): response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv", headers=HTTP_HEADERS) if response.ok: - siota_index = {row["SILO_CODE"]: row for row in - csv.DictReader(response.content.decode().splitlines())} - row = siota_index.get(ref_id) + global _SIOTA_INDEX_CACHE + if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache: + # New data from SIOTA, update our internal map + _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in + csv.DictReader(response.content.decode().splitlines())} + row = _SIOTA_INDEX_CACHE.get(ref_id) if row: sig_ref.name = row["NAME"] if "NAME" in row else None sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None @@ -162,19 +177,23 @@ def populate_sig_ref_info(sig_ref): if response.ok: data = response.json() if data: - for feature in data.get("features", []): - if feature["properties"]["wotaId"] == ref_id: - sig_ref.name = feature["properties"]["title"] - # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be - # added to them - sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id - if ref_id.upper().startswith("LDO-"): - number = int(ref_id.upper().replace("LDO-", "")) - sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214) - sig_ref.grid = feature["properties"]["qthLocator"] - sig_ref.latitude = feature["geometry"]["coordinates"][1] - sig_ref.longitude = feature["geometry"]["coordinates"][0] - break + global _WOTA_INDEX_CACHE + if not bool(_WOTA_INDEX_CACHE) or not response.from_cache: + # New data from WOTA, update our internal map + _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in + data.get("features", [])} + feature = _WOTA_INDEX_CACHE.get(ref_id) + if feature: + sig_ref.name = feature["properties"]["title"] + # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be + # added to them + sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id + if ref_id.upper().startswith("LDO-"): + number = int(ref_id.upper().replace("LDO-", "")) + sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214) + sig_ref.grid = feature["properties"]["qthLocator"] + sig_ref.latitude = feature["geometry"]["coordinates"][1] + sig_ref.longitude = feature["geometry"]["coordinates"][0] elif not response.from_cache: logging.warning("Malformed response looking up %s ref %s", sig, ref_id) elif not response.from_cache: @@ -185,17 +204,20 @@ def populate_sig_ref_info(sig_ref): if response.ok: data = response.json() if isinstance(data, list): - for asset in data: - if asset["code"] == ref_id: - sig_ref.name = asset["name"] - sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_") - try: - sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6) - except: - logging.debug("Invalid lat/lon received for reference") - sig_ref.latitude = asset["y"] - sig_ref.longitude = asset["x"] - break + global _ZLOTA_INDEX_CACHE + if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache: + # New data from ZLOTA, update our internal map + _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data} + asset = _ZLOTA_INDEX_CACHE.get(ref_id) + if asset: + sig_ref.name = asset["name"] + sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_") + try: + sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6) + except: + logging.debug("Invalid lat/lon received for reference") + sig_ref.latitude = asset["y"] + sig_ref.longitude = asset["x"] elif not response.from_cache: logging.warning("Malformed response looking up %s ref %s", sig, ref_id) elif not response.from_cache: @@ -212,15 +234,18 @@ def populate_sig_ref_info(sig_ref): if response.ok: data = response.json() if isinstance(data, list): - for ref in data: - if ref["reference_code"] == ref_id: - sig_ref.name = str(ref["name"]) - sig_ref.url = "https://llota.app/list/ref/" + ref_id - sig_ref.grid = str(ref["grid_locator"]) - ll = locator_to_latlong(sig_ref.grid) - sig_ref.latitude = ll[0] - sig_ref.longitude = ll[1] - break + global _LLOTA_INDEX_CACHE + if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache: + # New data from LLOTA, update our internal map + _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data} + ref = _LLOTA_INDEX_CACHE.get(ref_id) + if ref: + sig_ref.name = str(ref["name"]) + sig_ref.url = "https://llota.app/list/ref/" + ref_id + sig_ref.grid = str(ref["grid_locator"]) + ll = locator_to_latlong(sig_ref.grid) + sig_ref.latitude = ll[0] + sig_ref.longitude = ll[1] elif not response.from_cache: logging.warning("Malformed response looking up %s ref %s", sig, ref_id) elif not response.from_cache: diff --git a/templates/add_spot.html b/templates/add_spot.html index 11867dc..23a9cc8 100644 --- a/templates/add_spot.html +++ b/templates/add_spot.html @@ -76,7 +76,7 @@ - + diff --git a/templates/alerts.html b/templates/alerts.html index d8db687..75f3540 100644 --- a/templates/alerts.html +++ b/templates/alerts.html @@ -75,7 +75,7 @@ - + diff --git a/templates/bands.html b/templates/bands.html index f5d24b5..5c181bd 100644 --- a/templates/bands.html +++ b/templates/bands.html @@ -75,8 +75,8 @@ - - + + diff --git a/templates/base.html b/templates/base.html index c828934..9dd03cb 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,6 +1,6 @@ {% extends "skeleton.html" %} {% block head_extra %} - + @@ -10,10 +10,10 @@ - - - - + + + + {% end %} {% block body %}
diff --git a/templates/conditions.html b/templates/conditions.html index 4fe2ef7..9eea747 100644 --- a/templates/conditions.html +++ b/templates/conditions.html @@ -284,7 +284,7 @@
- + diff --git a/templates/map.html b/templates/map.html index 84d0d6f..ad5e684 100644 --- a/templates/map.html +++ b/templates/map.html @@ -108,8 +108,8 @@ - - + + diff --git a/templates/spots.html b/templates/spots.html index 3387b41..b659a50 100644 --- a/templates/spots.html +++ b/templates/spots.html @@ -116,8 +116,8 @@ - - + + diff --git a/templates/status.html b/templates/status.html index 813e227..8b0a5ea 100644 --- a/templates/status.html +++ b/templates/status.html @@ -59,7 +59,7 @@ - +