w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 02:21:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 3 #118

Browse Source
This commit is contained in:
Ian Renton
2026-07-31 17:45:09 +01:00
parent 818fd2d504
commit 0f59af6f9e
24 changed files with 592 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/config.py
+33
View File
@@ -1,3 +1,4 @@
import importlib
import logging
import os
@@ -33,3 +34,35 @@ WEB_UI_OPTIONS["spot-providers-enabled-by-default"] = [p["name"] for p in config
# one of our proviers. We set that to also be enabled by default.
if ALLOW_SPOTTING:
WEB_UI_OPTIONS["spot-providers-enabled-by-default"].append("API")
def get_spot_provider_from_config(config_providers_entry):
"""Utility method to get a spot provider based on the class specified in its config entry."""
module = importlib.import_module('spotproviders.' + config_providers_entry["class"].lower())
provider_class = getattr(module, config_providers_entry["class"])
return provider_class(config_providers_entry)
def get_alert_provider_from_config(config_providers_entry):
"""Utility method to get an alert provider based on the class specified in its config entry."""
module = importlib.import_module('alertproviders.' + config_providers_entry["class"].lower())
provider_class = getattr(module, config_providers_entry["class"])
return provider_class(config_providers_entry)
def get_solar_conditions_provider_from_config(config_providers_entry):
"""Utility method to get a solar conditions provider based on the class specified in its config entry."""
module = importlib.import_module('solarconditionsproviders.' + config_providers_entry["class"].lower())
provider_class = getattr(module, config_providers_entry["class"])
return provider_class(config_providers_entry)
def get_sig_ref_data_provider_from_config(config_providers_entry):
"""Utility method to get a SIG reference data provider based on the class specified in its config entry."""
module = importlib.import_module('sigrefdataproviders.' + config_providers_entry["class"].lower())
provider_class = getattr(module, config_providers_entry["class"])
return provider_class(config_providers_entry)
core/data_store.py
+21 -26
View File
@@ -12,57 +12,52 @@ class DataStore:
lookup data using different caching strategies for each."""
def __init__(self):
cache_dir = "./cache"
self.CACHE_DIR = "./cache"
self.MAX_SPOT_COUNT = 10000
self.MAX_ALERT_COUNT = 10000
self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
Path(cache_dir).mkdir(parents=True, exist_ok=True)
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval
self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
snapshot_dir=cache_dir + "/spots",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
snapshot_dir=cache_dir + "/alerts",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
def setup(self):
Path(self.CACHE_DIR).mkdir(parents=True, exist_ok=True)
# Standard disk cache for solar data and status data, but each cache contains only a single object which we
# expose to the wider application
self.solar = diskcache.Cache(cache_dir + "/solar")
self.solar = diskcache.Cache(self.CACHE_DIR + "/solar")
if "solar_conditions" not in self.solar:
self.solar.add("solar_conditions", SolarConditions())
self.solar_conditions = self.solar.get("solar_conditions")
self.status = diskcache.Cache(cache_dir + "/status")
self.status = diskcache.Cache(self.CACHE_DIR + "/status")
if "status_data" not in self.status:
self.status.add("status_data", {})
self.status_data = self.status.get("status_data")
# Standard disk caches for SIG ref data. Separate provider threads will repopulate these on a regular basis
# but there's no need for a TTL since old data is better than no data.
self.sigrefs_wwff = diskcache.Cache(cache_dir + "/sigrefs/wwff")
self.sigrefs_siota = diskcache.Cache(cache_dir + "/sigrefs/siota")
self.sigrefs_wota = diskcache.Cache(cache_dir + "/sigrefs/wota")
self.sigrefs_zlota = diskcache.Cache(cache_dir + "/sigrefs/zlota")
self.sigrefs_llota = diskcache.Cache(cache_dir + "/sigrefs/llota")
self.sigrefs_dme = diskcache.Cache(cache_dir + "/sigrefs/dme")
# Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
# but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG
# name and then reference ID, with the final value being a SIGRef object.
self.sigrefs = diskcache.Cache(self.CACHE_DIR + "/sigrefs")
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
# we've seen, so we rely on them timing out and this triggering another lookup.
self.callsigns = diskcache.Cache(cache_dir + "/callsigns")
self.callsigns = diskcache.Cache(self.CACHE_DIR + "/callsigns")
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
# specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
# before the spots and alerts are live in the system.
self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
snapshot_dir=self.CACHE_DIR + "/spots",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
snapshot_dir=self.CACHE_DIR + "/alerts",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
def close(self):
self.spots.close()
self.alerts.close()
self.solar.close()
self.status.close()
self.sigrefs_wwff.close()
self.sigrefs_siota.close()
self.sigrefs_wota.close()
self.sigrefs_zlota.close()
self.sigrefs_llota.close()
self.sigrefs.close()
self.callsigns.close()
# Global object
core/sig_utils.py
+54 -131
View File
@@ -1,4 +1,3 @@
import csv
import logging
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
@@ -9,11 +8,6 @@ from core.data_store import DATA_STORE
from core.geo_utils import wab_wai_square_to_lat_lon
from core.url_data_cache import URL_DATA_CACHE
# Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data
# file in Spothole and load it on startup.
with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
for row in csv.DictReader(_f, delimiter=";"):
DATA_STORE.sigrefs_dme.add(row["COD_INE"][:5], row)
def get_ref_regex_for_sig(sig):
"""Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
@@ -35,12 +29,15 @@ def get_sig_name_from_comment_name(sig):
def populate_sig_ref_info(sig_ref):
"""Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object which
must at minimum have a "sig" and an "id". The rest of the object will be populated and returned.
"""Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
SIG we are getting data for.
Note there is currently no support for KRMNPA location lookup, see issue #61."""
if sig_ref.sig is None or sig_ref.id is None:
logging.warning("Failed to look up sig_ref info, sig or id were not set.")
if sig_ref.sig is None or sig_ref.id is None or sig_ref.id == "":
logging.debug("Failed to look up sig_ref info, sig or id were not set.")
return sig_ref
sig = sig_ref.sig or ""
ref_id = sig_ref.id
@@ -123,95 +120,36 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WWFF":
response = URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
headers=HTTP_HEADERS)
if response.ok:
if not bool(DATA_STORE.sigrefs_wwff) or not response.from_cache:
# New data from WWFF, update our internal map
for row in csv.DictReader(response.content.decode().splitlines()):
DATA_STORE.sigrefs_wwff.add(row["reference"], row)
row = DATA_STORE.sigrefs_wwff.get(ref_id)
if row:
sig_ref.name = row["name"] if "name" in row else None
sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[
"longitude"] != "-" else None
elif not response.from_cache:
logging.warning("WWFF database did not contain data for ref %s", ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
lookup_data = DATA_STORE.sigrefs["WWFF"][ref_id] if ref_id in DATA_STORE.sigrefs["WWFF"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("WWFF database did not contain data for ref %s", ref_id)
elif sig.upper() == "SIOTA":
response = URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
headers=HTTP_HEADERS)
if response.ok:
if not bool(DATA_STORE.sigrefs_siota) or not response.from_cache:
# New data from SIOTA, update our internal map
for row in csv.DictReader(response.content.decode().splitlines()):
DATA_STORE.sigrefs_siota.add(row["SILO_CODE"], row)
row = DATA_STORE.sigrefs_siota.get(ref_id)
if row:
sig_ref.name = row["NAME"] if "NAME" in row else None
sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
elif not response.from_cache:
logging.warning("SIOTA database did not contain data for ref %s", ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
lookup_data = DATA_STORE.sigrefs["SIOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["SIOTA"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("SIOTA database did not contain data for ref %s", ref_id)
elif sig.upper() == "WOTA":
response = URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
if not bool(DATA_STORE.sigrefs_wota) or not response.from_cache:
# New data from WOTA, update our internal map
for feature in data.get("features", []):
DATA_STORE.sigrefs_wota.add(feature["properties"]["wotaId"], feature)
feature = DATA_STORE.sigrefs_wota.get(ref_id)
if feature:
sig_ref.name = feature["properties"]["title"]
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
# added to them
sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id
if ref_id.upper().startswith("LDO-"):
number = int(ref_id.upper().replace("LDO-", ""))
sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
sig_ref.grid = feature["properties"]["qthLocator"]
sig_ref.latitude = feature["geometry"]["coordinates"][1]
sig_ref.longitude = feature["geometry"]["coordinates"][0]
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
lookup_data = DATA_STORE.sigrefs["WOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["WOTA"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("WOTA database did not contain data for ref %s", ref_id)
elif sig.upper() == "ZLOTA":
response = URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if isinstance(data, list):
if not bool(DATA_STORE.sigrefs_zlota) or not response.from_cache:
# New data from ZLOTA, update our internal map
for ref in data:
DATA_STORE.sigrefs_zlota.add(ref["code"], ref)
ref = DATA_STORE.sigrefs_zlota.get(ref_id)
if ref:
sig_ref.name = ref["name"]
sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
try:
sig_ref.grid = latlong_to_locator(ref["y"], ref["x"], 6)
except:
logging.debug("Invalid lat/lon received for reference")
sig_ref.latitude = ref["y"]
sig_ref.longitude = ref["x"]
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
lookup_data = DATA_STORE.sigrefs["ZLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["ZLOTA"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("ZLOTA database did not contain data for ref %s", ref_id)
elif sig.upper() == "BOTA":
if not sig_ref.name:
@@ -219,27 +157,29 @@ def populate_sig_ref_info(sig_ref):
sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
elif sig.upper() == "LLOTA":
response = URL_DATA_CACHE.get("https://llota.app/api/public/references",
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if isinstance(data, list):
if not bool(DATA_STORE.sigrefs_llota) or not response.from_cache:
# New data from LLOTA, update our internal map
for ref in data:
DATA_STORE.sigrefs_llota.add(ref["reference_code"], ref)
ref = DATA_STORE.sigrefs_llota.get(ref_id)
if ref:
sig_ref.name = str(ref["name"])
sig_ref.url = "https://llota.app/list/ref/" + ref_id
sig_ref.grid = str(ref["grid_locator"])
ll = locator_to_latlong(sig_ref.grid)
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
lookup_data = DATA_STORE.sigrefs["LLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["LLOTA"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("LLOTA database did not contain data for ref %s", ref_id)
elif sig.upper() == "DME":
# Zero-pad to 5 digits to match our source data
lookup_data = DATA_STORE.sigrefs["DME"][ref_id.zfill(5)] if ref_id.zfill(5) in DATA_STORE.sigrefs["DME"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("DME database did not contain data for ref %s", ref_id)
elif sig.upper() == "TOTA":
lookup_data = DATA_STORE.sigrefs["TOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["TOTA"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("TOTA database did not contain data for ref %s", ref_id)
elif sig.upper() == "WWTOTA":
if not sig_ref.name:
@@ -268,23 +208,6 @@ def populate_sig_ref_info(sig_ref):
except:
logging.warning("Invalid lat/lon received for WAB/WAI reference")
elif sig.upper() == "DME":
# Zero-pad to 5 digits to match our source data
row = DATA_STORE.sigrefs_dme.get(ref_id.zfill(5))
if row:
sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
"LATITUD_ETRS89_REGCAN95") else None
sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
"LONGITUD_ETRS89_REGCAN95") else None
if sig_ref.latitude and sig_ref.longitude:
try:
sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6)
except Exception:
logging.warning("Invalid lat/lon received for DME reference")
else:
logging.warning("DME database did not contain data for ref %s", ref_id)
except ConnectionError:
logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
except (ConnectTimeout, ReadTimeout):
core/status_reporter.py
+8 -1
View File
@@ -14,7 +14,8 @@ from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alert
class StatusReporter:
"""Provides a timed update of the application's status data."""
def __init__(self, run_interval, web_server,spot_providers, alert_providers, solar_condition_providers):
def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers,
sig_ref_data_providers):
"""Constructor"""
self._run_interval = run_interval
@@ -22,6 +23,7 @@ class StatusReporter:
self._spot_providers = spot_providers
self._alert_providers = alert_providers
self._solar_condition_providers = solar_condition_providers
self._sig_ref_data_providers = sig_ref_data_providers
self._thread = None
self._stop_event = Event()
self._startup_time = datetime.now(pytz.UTC)
@@ -72,6 +74,11 @@ class StatusReporter:
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
self._solar_condition_providers))
DATA_STORE.status_data["sig_ref_data_providers"] = list(
map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
self._sig_ref_data_providers))
DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
"last_api_access": self._web_server.web_server_metrics[
"last_api_access_time"].replace(