Cache data for the programmes for which we must fetch the whole set of references, and avoid transforming CSV/JSON to dict every time.

core/sig_utils.py

@@ -12,6 +12,14 @@ from core.geo_utils import wab_wai_square_to_lat_lon
 # file in Spothole and load it on startup.
 with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
     _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
+# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
+# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
+# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
+_WWFF_INDEX_CACHE = {}
+_SIOTA_INDEX_CACHE = {}
+_WOTA_INDEX_CACHE = {}
+_ZLOTA_INDEX_CACHE = {}
+_LLOTA_INDEX_CACHE = {}
 
 
 def get_ref_regex_for_sig(sig):
@@ -125,8 +133,12 @@ def populate_sig_ref_info(sig_ref):
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
                                                       headers=HTTP_HEADERS)
             if response.ok:
-                wwff_index = {row["reference"]: row for row in csv.DictReader(response.content.decode().splitlines())}
+                global _WWFF_INDEX_CACHE
-                row = wwff_index.get(ref_id)
+                if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
+                    # New data from WWFF, update our internal map
+                    _WWFF_INDEX_CACHE = {row["reference"]: row for row in
+                                         csv.DictReader(response.content.decode().splitlines())}
+                row = _WWFF_INDEX_CACHE.get(ref_id)
                 if row:
                     sig_ref.name = row["name"] if "name" in row else None
                     sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
@@ -143,9 +155,12 @@ def populate_sig_ref_info(sig_ref):
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
                                                       headers=HTTP_HEADERS)
             if response.ok:
-                siota_index = {row["SILO_CODE"]: row for row in
+                global _SIOTA_INDEX_CACHE
-                               csv.DictReader(response.content.decode().splitlines())}
+                if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
-                row = siota_index.get(ref_id)
+                    # New data from SIOTA, update our internal map
+                    _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
+                                          csv.DictReader(response.content.decode().splitlines())}
+                row = _SIOTA_INDEX_CACHE.get(ref_id)
                 if row:
                     sig_ref.name = row["NAME"] if "NAME" in row else None
                     sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
@@ -162,19 +177,23 @@ def populate_sig_ref_info(sig_ref):
             if response.ok:
                 data = response.json()
                 if data:
-                    for feature in data.get("features", []):
+                    global _WOTA_INDEX_CACHE
-                        if feature["properties"]["wotaId"] == ref_id:
+                    if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
-                            sig_ref.name = feature["properties"]["title"]
+                        # New data from WOTA, update our internal map
-                            # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
+                        _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
-                            # added to them
+                                             data.get("features", [])}
-                            sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id
+                    feature = _WOTA_INDEX_CACHE.get(ref_id)
-                            if ref_id.upper().startswith("LDO-"):
+                    if feature:
-                                number = int(ref_id.upper().replace("LDO-", ""))
+                        sig_ref.name = feature["properties"]["title"]
-                                sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
+                        # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
-                            sig_ref.grid = feature["properties"]["qthLocator"]
+                        # added to them
-                            sig_ref.latitude = feature["geometry"]["coordinates"][1]
+                        sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id
-                            sig_ref.longitude = feature["geometry"]["coordinates"][0]
+                        if ref_id.upper().startswith("LDO-"):
-                            break
+                            number = int(ref_id.upper().replace("LDO-", ""))
+                            sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
+                        sig_ref.grid = feature["properties"]["qthLocator"]
+                        sig_ref.latitude = feature["geometry"]["coordinates"][1]
+                        sig_ref.longitude = feature["geometry"]["coordinates"][0]
                 elif not response.from_cache:
                     logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
             elif not response.from_cache:
@@ -185,17 +204,20 @@ def populate_sig_ref_info(sig_ref):
             if response.ok:
                 data = response.json()
                 if isinstance(data, list):
-                    for asset in data:
+                    global _ZLOTA_INDEX_CACHE
-                        if asset["code"] == ref_id:
+                    if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
-                            sig_ref.name = asset["name"]
+                        # New data from ZLOTA, update our internal map
-                            sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
+                        _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
-                            try:
+                    asset = _ZLOTA_INDEX_CACHE.get(ref_id)
-                                sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
+                    if asset:
-                            except:
+                        sig_ref.name = asset["name"]
-                                logging.debug("Invalid lat/lon received for reference")
+                        sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
-                                sig_ref.latitude = asset["y"]
+                        try:
-                                sig_ref.longitude = asset["x"]
+                            sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
-                            break
+                        except:
+                            logging.debug("Invalid lat/lon received for reference")
+                            sig_ref.latitude = asset["y"]
+                            sig_ref.longitude = asset["x"]
                 elif not response.from_cache:
                     logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
             elif not response.from_cache:
@@ -212,15 +234,18 @@ def populate_sig_ref_info(sig_ref):
             if response.ok:
                 data = response.json()
                 if isinstance(data, list):
-                    for ref in data:
+                    global _LLOTA_INDEX_CACHE
-                        if ref["reference_code"] == ref_id:
+                    if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
-                            sig_ref.name = str(ref["name"])
+                        # New data from LLOTA, update our internal map
-                            sig_ref.url = "https://llota.app/list/ref/" + ref_id
+                        _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
-                            sig_ref.grid = str(ref["grid_locator"])
+                    ref = _LLOTA_INDEX_CACHE.get(ref_id)
-                            ll = locator_to_latlong(sig_ref.grid)
+                    if ref:
-                            sig_ref.latitude = ll[0]
+                        sig_ref.name = str(ref["name"])
-                            sig_ref.longitude = ll[1]
+                        sig_ref.url = "https://llota.app/list/ref/" + ref_id
-                            break
+                        sig_ref.grid = str(ref["grid_locator"])
+                        ll = locator_to_latlong(sig_ref.grid)
+                        sig_ref.latitude = ll[0]
+                        sig_ref.longitude = ll[1]
                 elif not response.from_cache:
                     logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
             elif not response.from_cache:

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1785049068"></script>
+<script src="/static/js/add-spot.js?v=1785082221"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -75,7 +75,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1785049068"></script>
+<script src="/static/js/alerts.js?v=1785082221"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -75,8 +75,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/static/js/spotsbandsandmap.js?v=1785049068"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1785082221"></script>
-<script src="/static/js/bands.js?v=1785049068"></script>
+<script src="/static/js/bands.js?v=1785082221"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1785049068" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1785082221" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
 <script src="/static/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
 <script src="/static/vendor/js/tinycolor2-1.6.0.min.js"></script>
 
-<script src="/static/js/utils.js?v=1785049068"></script>
+<script src="/static/js/utils.js?v=1785082221"></script>
-<script src="/static/js/ui-ham.js?v=1785049068"></script>
+<script src="/static/js/ui-ham.js?v=1785082221"></script>
-<script src="/static/js/geo.js?v=1785049068"></script>
+<script src="/static/js/geo.js?v=1785082221"></script>
-<script src="/static/js/common.js?v=1785049068"></script>
+<script src="/static/js/common.js?v=1785082221"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1785049068"></script>
+<script src="/static/js/conditions.js?v=1785082221"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -108,8 +108,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/static/js/spotsbandsandmap.js?v=1785049068"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1785082221"></script>
-<script src="/static/js/map.js?v=1785049068"></script>
+<script src="/static/js/map.js?v=1785082221"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -116,8 +116,8 @@
 <script>
     let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
 </script>
-<script src="/static/js/spotsbandsandmap.js?v=1785049068"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1785082221"></script>
-<script src="/static/js/spots.js?v=1785049068"></script>
+<script src="/static/js/spots.js?v=1785082221"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -59,7 +59,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1785049068"></script>
+<script src="/static/js/status.js?v=1785082221"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");