w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-05 18:11:41 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Cache data for the programmes for which we must fetch the whole set of references, and avoid transforming CSV/JSON to dict every time.

Browse Source
This commit is contained in:
Ian Renton
2026-07-26 17:10:21 +01:00
parent 3c1b1821f5
commit fffd230633
9 changed files with 78 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/sig_utils.py
+63 -38
View File
@@ -12,6 +12,14 @@ from core.geo_utils import wab_wai_square_to_lat_lon
# file in Spothole and load it on startup.
with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
_DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
_WWFF_INDEX_CACHE = {}
_SIOTA_INDEX_CACHE = {}
_WOTA_INDEX_CACHE = {}
_ZLOTA_INDEX_CACHE = {}
_LLOTA_INDEX_CACHE = {}
def get_ref_regex_for_sig(sig):
@@ -125,8 +133,12 @@ def populate_sig_ref_info(sig_ref):
response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
headers=HTTP_HEADERS)
if response.ok:
wwff_index = {row["reference"]: row for row in csv.DictReader(response.content.decode().splitlines())}
row = wwff_index.get(ref_id)
global _WWFF_INDEX_CACHE
if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
# New data from WWFF, update our internal map
_WWFF_INDEX_CACHE = {row["reference"]: row for row in
csv.DictReader(response.content.decode().splitlines())}
row = _WWFF_INDEX_CACHE.get(ref_id)
if row:
sig_ref.name = row["name"] if "name" in row else None
sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
@@ -143,9 +155,12 @@ def populate_sig_ref_info(sig_ref):
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
headers=HTTP_HEADERS)
if response.ok:
siota_index = {row["SILO_CODE"]: row for row in
csv.DictReader(response.content.decode().splitlines())}
row = siota_index.get(ref_id)
global _SIOTA_INDEX_CACHE
if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
# New data from SIOTA, update our internal map
_SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
csv.DictReader(response.content.decode().splitlines())}
row = _SIOTA_INDEX_CACHE.get(ref_id)
if row:
sig_ref.name = row["NAME"] if "NAME" in row else None
sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
@@ -162,19 +177,23 @@ def populate_sig_ref_info(sig_ref):
if response.ok:
data = response.json()
if data:
for feature in data.get("features", []):
if feature["properties"]["wotaId"] == ref_id:
sig_ref.name = feature["properties"]["title"]
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
# added to them
sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id
if ref_id.upper().startswith("LDO-"):
number = int(ref_id.upper().replace("LDO-", ""))
sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
sig_ref.grid = feature["properties"]["qthLocator"]
sig_ref.latitude = feature["geometry"]["coordinates"][1]
sig_ref.longitude = feature["geometry"]["coordinates"][0]
break
global _WOTA_INDEX_CACHE
if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
# New data from WOTA, update our internal map
_WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
data.get("features", [])}
feature = _WOTA_INDEX_CACHE.get(ref_id)
if feature:
sig_ref.name = feature["properties"]["title"]
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
# added to them
sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id
if ref_id.upper().startswith("LDO-"):
number = int(ref_id.upper().replace("LDO-", ""))
sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
sig_ref.grid = feature["properties"]["qthLocator"]
sig_ref.latitude = feature["geometry"]["coordinates"][1]
sig_ref.longitude = feature["geometry"]["coordinates"][0]
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
@@ -185,17 +204,20 @@ def populate_sig_ref_info(sig_ref):
if response.ok:
data = response.json()
if isinstance(data, list):
for asset in data:
if asset["code"] == ref_id:
sig_ref.name = asset["name"]
sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
try:
sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
except:
logging.debug("Invalid lat/lon received for reference")
sig_ref.latitude = asset["y"]
sig_ref.longitude = asset["x"]
break
global _ZLOTA_INDEX_CACHE
if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
# New data from ZLOTA, update our internal map
_ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
asset = _ZLOTA_INDEX_CACHE.get(ref_id)
if asset:
sig_ref.name = asset["name"]
sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
try:
sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
except:
logging.debug("Invalid lat/lon received for reference")
sig_ref.latitude = asset["y"]
sig_ref.longitude = asset["x"]
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
@@ -212,15 +234,18 @@ def populate_sig_ref_info(sig_ref):
if response.ok:
data = response.json()
if isinstance(data, list):
for ref in data:
if ref["reference_code"] == ref_id:
sig_ref.name = str(ref["name"])
sig_ref.url = "https://llota.app/list/ref/" + ref_id
sig_ref.grid = str(ref["grid_locator"])
ll = locator_to_latlong(sig_ref.grid)
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
break
global _LLOTA_INDEX_CACHE
if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
# New data from LLOTA, update our internal map
_LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
ref = _LLOTA_INDEX_CACHE.get(ref_id)
if ref:
sig_ref.name = str(ref["name"])
sig_ref.url = "https://llota.app/list/ref/" + ref_id
sig_ref.grid = str(ref["grid_locator"])
ll = locator_to_latlong(sig_ref.grid)
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
templates/add_spot.html
+1 -1
View File
@@ -76,7 +76,7 @@
</div>
<script src="/static/js/add-spot.js?v=1785049068"></script>
<script src="/static/js/add-spot.js?v=1785082221"></script>
<script>$(document).ready(function () {
$("#nav-link-add-spot").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/alerts.html
+1 -1
View File
@@ -75,7 +75,7 @@
</div>
<script src="/static/js/alerts.js?v=1785049068"></script>
<script src="/static/js/alerts.js?v=1785082221"></script>
<script>$(document).ready(function () {
$("#nav-link-alerts").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/bands.html
+2 -2
View File
@@ -75,8 +75,8 @@
<script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script>
<script src="/static/js/spotsbandsandmap.js?v=1785049068"></script>
<script src="/static/js/bands.js?v=1785049068"></script>
<script src="/static/js/spotsbandsandmap.js?v=1785082221"></script>
<script src="/static/js/bands.js?v=1785082221"></script>
<script>$(document).ready(function () {
$("#nav-link-bands").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/base.html
+5 -5
View File
@@ -1,6 +1,6 @@
{% extends "skeleton.html" %}
{% block head_extra %}
<link rel="stylesheet" href="/static/css/style.css?v=1785049068" type="text/css">
<link rel="stylesheet" href="/static/css/style.css?v=1785082221" type="text/css">
<link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
<link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
<link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
<script src="/static/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
<script src="/static/vendor/js/tinycolor2-1.6.0.min.js"></script>
<script src="/static/js/utils.js?v=1785049068"></script>
<script src="/static/js/ui-ham.js?v=1785049068"></script>
<script src="/static/js/geo.js?v=1785049068"></script>
<script src="/static/js/common.js?v=1785049068"></script>
<script src="/static/js/utils.js?v=1785082221"></script>
<script src="/static/js/ui-ham.js?v=1785082221"></script>
<script src="/static/js/geo.js?v=1785082221"></script>
<script src="/static/js/common.js?v=1785082221"></script>
{% end %}
{% block body %}
<div class="container">
templates/conditions.html
+1 -1
View File
@@ -284,7 +284,7 @@
</div>
<script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
<script src="/static/js/conditions.js?v=1785049068"></script>
<script src="/static/js/conditions.js?v=1785082221"></script>
<script>$(document).ready(function () {
$("#nav-link-conditions").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/map.html
+2 -2
View File
@@ -108,8 +108,8 @@
<script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script>
<script src="/static/js/spotsbandsandmap.js?v=1785049068"></script>
<script src="/static/js/map.js?v=1785049068"></script>
<script src="/static/js/spotsbandsandmap.js?v=1785082221"></script>
<script src="/static/js/map.js?v=1785082221"></script>
<script>$(document).ready(function () {
$("#nav-link-map").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/spots.html
+2 -2
View File
@@ -116,8 +116,8 @@
<script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script>
<script src="/static/js/spotsbandsandmap.js?v=1785049068"></script>
<script src="/static/js/spots.js?v=1785049068"></script>
<script src="/static/js/spotsbandsandmap.js?v=1785082221"></script>
<script src="/static/js/spots.js?v=1785082221"></script>
<script>$(document).ready(function () {
$("#nav-link-spots").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/status.html
+1 -1
View File
@@ -59,7 +59,7 @@
</div>
</div>
<script src="/static/js/status.js?v=1785049068"></script>
<script src="/static/js/status.js?v=1785082221"></script>
<script>
$(document).ready(function () {
$("#nav-link-status").addClass("active");