w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 10:31:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 4 #118

Browse Source
This commit is contained in:
Ian Renton
2026-07-31 21:02:37 +01:00
parent 0f59af6f9e
commit 23edd4e02e
37 changed files with 822 additions and 528 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/constants.py
+2 -2
View File
@@ -19,9 +19,9 @@ SIGS = [
SIG(name="HEMA", comment_names=["HEMA"], description="HuMPs Excluding Marilyns Award", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}"),
SIG(name="IOTA", comment_names=["IOTA"], description="Islands on the Air", ref_regex=r"[A-Z]{2}\-\d{3}"),
SIG(name="MOTA", comment_names=["MOTA"], description="Mills on the Air", ref_regex=r"X\d{4,6}"),
SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}\-\d{3,4}"),
SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}[\- ]\d{3,4}"),
SIG(name="ILLW", comment_names=["ILLW"], description="International Lighthouse & Lightship Weekend", ref_regex=r"[A-Z]{2}\d{4}"),
SIG(name="SiOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
SIG(name="SIOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
SIG(name="WCA", comment_names=["WCA"], description="World Castles Award", ref_regex=r"[A-Z0-9]{1,3}\-\d{5}"),
SIG(name="ZLOTA", comment_names=["ZLOTA"], description="New Zealand on the Air", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}"),
SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"),
core/data_store.py
+40 -24
View File
@@ -1,8 +1,10 @@
import logging
from pathlib import Path
import diskcache
from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
from core.constants import SIGS
from core.live_data_cache import LiveDataCache
from data.solar_conditions import SolarConditions
@@ -12,51 +14,65 @@ class DataStore:
lookup data using different caching strategies for each."""
def __init__(self):
self.CACHE_DIR = "./cache"
self.MAX_SPOT_COUNT = 10000
self.MAX_ALERT_COUNT = 10000
self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
self._CACHE_DIR = "./cache"
self._MAX_SPOT_COUNT = 100000
self._MAX_ALERT_COUNT = 100000
self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
self.alerts = None
self.spots = None
self.callsigns = None
self.sigrefs = None
self.status_data = None
self._status = None
self.solar_conditions = None
self._solar = None
def setup(self):
Path(self.CACHE_DIR).mkdir(parents=True, exist_ok=True)
Path(self._CACHE_DIR).mkdir(parents=True, exist_ok=True)
# Standard disk cache for solar data and status data, but each cache contains only a single object which we
# expose to the wider application
self.solar = diskcache.Cache(self.CACHE_DIR + "/solar")
if "solar_conditions" not in self.solar:
self.solar.add("solar_conditions", SolarConditions())
self.solar_conditions = self.solar.get("solar_conditions")
self.status = diskcache.Cache(self.CACHE_DIR + "/status")
if "status_data" not in self.status:
self.status.add("status_data", {})
self.status_data = self.status.get("status_data")
self._solar = diskcache.Cache(self._CACHE_DIR + "/solar")
if "solar_conditions" not in self._solar:
self._solar.add("solar_conditions", SolarConditions())
self.solar_conditions = self._solar.get("solar_conditions")
self._status = diskcache.Cache(self._CACHE_DIR + "/status")
if "status_data" not in self._status:
self._status.add("status_data", {})
self.status_data = self._status.get("status_data")
# Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
# but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG
# name and then reference ID, with the final value being a SIGRef object.
self.sigrefs = diskcache.Cache(self.CACHE_DIR + "/sigrefs")
self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs")
for k in list(self.sigrefs.iterkeys()):
logging.info(f"Loaded data for %d references in %s SIG.", len(self.sigrefs[k]), k)
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
# we've seen, so we rely on them timing out and this triggering another lookup.
self.callsigns = diskcache.Cache(self.CACHE_DIR + "/callsigns")
self.callsigns = diskcache.Cache(self._CACHE_DIR + "/callsigns")
logging.info(f"Loaded data for %d callsigns.", len(self.callsigns))
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
# specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
# before the spots and alerts are live in the system.
self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
snapshot_dir=self.CACHE_DIR + "/spots",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
snapshot_dir=self.CACHE_DIR + "/alerts",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
snapshot_dir=self._CACHE_DIR + "/spots",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys()))
self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
snapshot_dir=self._CACHE_DIR + "/alerts",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
def close(self):
self.spots.close()
self.alerts.close()
self.solar.close()
self.status.close()
self._solar.close()
self._status.close()
self.sigrefs.close()
self.callsigns.close()
core/sig_utils.py
+61 -149
View File
@@ -35,157 +35,25 @@ def populate_sig_ref_info(sig_ref):
SIG we are getting data for.
Note there is currently no support for KRMNPA location lookup, see issue #61."""
if sig_ref.sig is None or sig_ref.id is None or sig_ref.id == "":
if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "":
logging.debug("Failed to look up sig_ref info, sig or id were not set.")
return sig_ref
sig = sig_ref.sig or ""
sig = sig_ref.sig
ref_id = sig_ref.id
# DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all activators
# add leading zeros.
if sig.upper() == "DME":
ref_id = ref_id.zfill(5)
try:
if sig.upper() == "POTA":
response = URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
fullname = str(data["name"]) if "name" in data else None
if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
fullname = fullname + " " + data["parktypeDesc"]
sig_ref.name = fullname
sig_ref.url = "https://pota.app/#/park/" + ref_id
sig_ref.grid = data["grid6"] if "grid6" in data else None
sig_ref.latitude = data["latitude"] if "latitude" in data else None
sig_ref.longitude = data["longitude"] if "longitude" in data else None
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "SOTA":
response = URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
sig_ref.grid = data["locator"] if "locator" in data else None
sig_ref.latitude = data["latitude"] if "latitude" in data else None
sig_ref.longitude = data["longitude"] if "longitude" in data else None
sig_ref.activation_score = data["points"] if "points" in data else None
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WWBOTA":
response = URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
sig_ref.grid = data["locator"] if "locator" in data else None
sig_ref.latitude = data["lat"] if "lat" in data else None
sig_ref.longitude = data["long"] if "long" in data else None
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
sig_ref.grid = data["locator"] if "locator" in data else None
# For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
# the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
# and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
# just use the single point we have.
if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
"lat2") is not None and data.get("lng2") is not None:
sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
else:
sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WWFF":
lookup_data = DATA_STORE.sigrefs["WWFF"][ref_id] if ref_id in DATA_STORE.sigrefs["WWFF"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("WWFF database did not contain data for ref %s", ref_id)
elif sig.upper() == "SIOTA":
lookup_data = DATA_STORE.sigrefs["SIOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["SIOTA"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("SIOTA database did not contain data for ref %s", ref_id)
elif sig.upper() == "WOTA":
lookup_data = DATA_STORE.sigrefs["WOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["WOTA"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("WOTA database did not contain data for ref %s", ref_id)
elif sig.upper() == "ZLOTA":
lookup_data = DATA_STORE.sigrefs["ZLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["ZLOTA"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("ZLOTA database did not contain data for ref %s", ref_id)
elif sig.upper() == "BOTA":
if not sig_ref.name:
sig_ref.name = sig_ref.id
sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
elif sig.upper() == "LLOTA":
lookup_data = DATA_STORE.sigrefs["LLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["LLOTA"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("LLOTA database did not contain data for ref %s", ref_id)
elif sig.upper() == "DME":
# Zero-pad to 5 digits to match our source data
lookup_data = DATA_STORE.sigrefs["DME"][ref_id.zfill(5)] if ref_id.zfill(5) in DATA_STORE.sigrefs["DME"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("DME database did not contain data for ref %s", ref_id)
elif sig.upper() == "TOTA":
lookup_data = DATA_STORE.sigrefs["TOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["TOTA"] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("TOTA database did not contain data for ref %s", ref_id)
elif sig.upper() == "WWTOTA":
if not sig_ref.name:
sig_ref.name = sig_ref.id
sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + str(sig_ref.name)
# If the SIG is HEMA or KRMNPA, we have no current lookup for this so just skip it.
if sig.upper() == "HEMA" or sig.upper() == "KRMNPA":
return sig_ref
# If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
# calculate all the information we are going to get directly. So handle those cases first
elif sig.upper() == "TILES":
# Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
if not sig_ref.name:
@@ -208,10 +76,54 @@ def populate_sig_ref_info(sig_ref):
except:
logging.warning("Invalid lat/lon received for WAB/WAI reference")
except ConnectionError:
logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when looking up sig_ref info for " + sig + " ref " + ref_id)
elif sig.upper() == "BOTA":
# For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
if not sig_ref.name:
sig_ref.name = sig_ref.id
sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
# OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
# this SIG. If so, check for the reference data and use that.
elif sig in DATA_STORE.sigrefs:
lookup_data = DATA_STORE.sigrefs[sig][ref_id] if ref_id in DATA_STORE.sigrefs[sig] else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("%s database did not contain data for ref %s", sig, ref_id)
elif False:
# TODO remove
# OK, this is not a SIG we have stored data for. Maybe it's of a type we can query information for live.
if sig.upper() == "IOTA":
response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
sig_ref.grid = data["locator"] if "locator" in data else None
# For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
# the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
# and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
# just use the single point we have.
if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
"lat2") is not None and data.get("lng2") is not None:
sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
else:
sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
else:
logging.warning(f"Tried to look up a SIG called %s but Spothole does not know what that is.", sig)
except Exception:
logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
return sig_ref
core/status_reporter.py
+2 -1
View File
@@ -77,7 +77,8 @@ class StatusReporter:
DATA_STORE.status_data["sig_ref_data_providers"] = list(
map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
"reference_count": p.reference_count},
self._sig_ref_data_providers))
DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
"last_api_access": self._web_server.web_server_metrics[