w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 02:21:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 7 #118

Browse Source
This commit is contained in:
Ian Renton
2026-08-01 10:34:20 +01:00
parent b7d2cf0fdb
commit 41aecc6007
10 changed files with 121 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/lookup_helper.py → core/call_lookup_helper.py
+4 -4
View File
@@ -329,7 +329,7 @@ class LookupHelper:
def get_flag_for_dxcc(self, dxcc):
"""Get an emoji flag for a given DXCC entity ID"""
dxcc_data = DATA_STORE.dxcc_data[dxcc]
dxcc_data = DATA_STORE.dxcc_data[dxcc] if dxcc in DATA_STORE.dxcc_data else None
return dxcc_data["flag"] if dxcc_data else None
def infer_name_from_callsign_online_lookup(self, call, credentials=None):
@@ -621,9 +621,9 @@ class LookupHelper:
def _get_dxcc_data_for_callsign(self, call) -> dict | None:
"""Utility method to get generic DXCC data from our lookup table, if we can find it"""
for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]:
if entry["_prefixRegexCompiled"].match(call):
return entry
for pattern, entity_code in DATA_STORE.dxcc_lookup_by_call_regex:
if pattern.match(call):
return DATA_STORE.dxcc_data[entity_code]
return None
def stop(self):
core/data_store.py
+12
View File
@@ -1,4 +1,5 @@
import logging
import re
from pathlib import Path
import diskcache
@@ -23,6 +24,7 @@ class DataStore:
self.spots = None
self.callsigns = None
self.dxcc_data = None
self.dxcc_lookup_by_call_regex = []
self.sigrefs = None
self.status_data = None
self._status = None
@@ -46,6 +48,7 @@ class DataStore:
# Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
# a regular basis but there's no need for a TTL since old data is better than no data.
self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data")
self.regenerateCallRegexToDXCCEntityMap()
# For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers
# of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
@@ -72,6 +75,15 @@ class DataStore:
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
def regenerateCallRegexToDXCCEntityMap(self):
"""DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC
entity it belongs to. But getting every DXCC entity data object out of DiskCache, iterating, compiling its regex
and testing the callsign every time is expensive. So instead we build a separate in-memory lookup of compiled
regex against DXCC entity code, as a list of tuples we can iterate through."""
for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]:
self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
def close(self):
self.spots.close()
self.alerts.close()
core/sig_lookup_helper.py
+93
View File
@@ -0,0 +1,93 @@
import logging
from pyhamtools.locator import locator_to_latlong, latlong_to_locator
from core.data_store import DATA_STORE
from core.geo_utils import wab_wai_square_to_lat_lon
def populate_sig_ref_info(sig_ref):
"""Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
SIG we are getting data for.
Note there is currently no support for KRMNPA location lookup, see issue #61."""
if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "":
logging.debug("Failed to look up sig_ref info, sig or id were not set.")
return sig_ref
sig = sig_ref.sig
ref_id = sig_ref.id
try:
### FUDGES ###
#
# DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all
# activators add leading zeros.
if sig.upper() == "DME":
ref_id = ref_id.zfill(5)
# KRMNPA & SANPCPA fudge. These don't have their own reference system, they just use VKFF references, so pretend
# the sig is WWFF and carry on
if sig.upper() == "KRMNPA" or sig.upper() == "SANPCPA":
sig = "WWFF"
### SKIPS ###
#
# If the SIG is HEMA, we have no current lookup for this so just skip the lookup here.
if sig.upper() == "HEMA":
return sig_ref
### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ###
#
# If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
# calculate all the information we are going to get directly.
if sig.upper() == "TILES":
# Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
if not sig_ref.name:
sig_ref.name = sig_ref.id
if not sig_ref.grid:
sig_ref.grid = sig_ref.id
if sig_ref.grid and not sig_ref.latitude:
ll = locator_to_latlong(str(sig_ref.grid))
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
return sig_ref
elif sig.upper() == "WAB" or sig.upper() == "WAI":
ll = wab_wai_square_to_lat_lon(ref_id)
if ll:
sig_ref.name = ref_id
try:
sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
except:
logging.warning("Invalid lat/lon received for WAB/WAI reference")
return sig_ref
elif sig.upper() == "BOTA":
# For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
if not sig_ref.name:
sig_ref.name = sig_ref.id
sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
return sig_ref
### ACTUAL LOOKUP ###
#
# OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
# this SIG. If so, check for the reference data and use that.
key = sig + ":" + ref_id
lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
if lookup_data:
# Copy new sig ref data into existing object where data was previously missing
for key, value in lookup_data.__dict__.items():
if value is not None and sig_ref.__dict__.get(key) is None:
sig_ref.__dict__[key] = value
else:
logging.warning("%s database did not contain data for ref %s", sig, ref_id)
except Exception:
logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
return sig_ref
core/sig_utils.py
-90
View File
@@ -1,10 +1,4 @@
import logging
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from core.constants import SIGS
from core.data_store import DATA_STORE
from core.geo_utils import wab_wai_square_to_lat_lon
def get_ref_regex_for_sig(sig):
@@ -26,89 +20,5 @@ def get_sig_name_from_comment_name(sig):
return None
def populate_sig_ref_info(sig_ref):
"""Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
SIG we are getting data for.
Note there is currently no support for KRMNPA location lookup, see issue #61."""
if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "":
logging.debug("Failed to look up sig_ref info, sig or id were not set.")
return sig_ref
sig = sig_ref.sig
ref_id = sig_ref.id
try:
### FUDGES ###
#
# DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all
# activators add leading zeros.
if sig.upper() == "DME":
ref_id = ref_id.zfill(5)
# KRMNPA & SANPCPA fudge. These don't have their own reference system, they just use VKFF references, so pretend
# the sig is WWFF and carry on
if sig.upper() == "KRMNPA" or sig.upper() == "SANPCPA":
sig = "WWFF"
### SKIPS ###
#
# If the SIG is HEMA, we have no current lookup for this so just skip the lookup here.
if sig.upper() == "HEMA":
return sig_ref
### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ###
#
# If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
# calculate all the information we are going to get directly.
if sig.upper() == "TILES":
# Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
if not sig_ref.name:
sig_ref.name = sig_ref.id
if not sig_ref.grid:
sig_ref.grid = sig_ref.id
if sig_ref.grid and not sig_ref.latitude:
ll = locator_to_latlong(str(sig_ref.grid))
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
elif sig.upper() == "WAB" or sig.upper() == "WAI":
ll = wab_wai_square_to_lat_lon(ref_id)
if ll:
sig_ref.name = ref_id
try:
sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
except:
logging.warning("Invalid lat/lon received for WAB/WAI reference")
elif sig.upper() == "BOTA":
# For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
if not sig_ref.name:
sig_ref.name = sig_ref.id
sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
### ACTUAL LOOKUP ###
#
# OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
# this SIG. If so, check for the reference data and use that.
key = sig + ":" + ref_id
lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
if lookup_data:
# Copy new sig ref data into existing object where data was previously missing
for key, value in lookup_data.__dict__.items():
if value is not None and sig_ref.__dict__.get(key) is None:
sig_ref.__dict__[key] = value
else:
logging.warning("%s database did not contain data for ref %s", sig, ref_id)
except Exception:
logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
return sig_ref
# Regex matching any SIG's "comment name", i.e. how it may be referred to in spot comments
ANY_SIG_REGEX = r"(" + r"|".join(n for s in SIGS for n in s.comment_names) + r")"