Refactor of caching & data storage part 14 #118

This commit is contained in:
Ian Renton
2026-08-03 19:25:25 +01:00
parent 50ab27e4a2
commit f82d611b7e
15 changed files with 144 additions and 142 deletions
+23 -11
View File
@@ -251,32 +251,44 @@ sig-ref-data-providers:
# Callsign data providers to use. These allow Spothole to provide information about callsigns, either from static
# lookups or from web-based services such as QRZ.
# lookups or from web-based services such as QRZ. Note that use of QRZ and HamQTH is *on behalf of the user*, and
# requires the user to enter their credentials into the website UI or provide them in an API call. Spothole will not
# look up all calls via QRZ/HamQTH using the server owner's credentials, as this is against their usage policy.
callsign-data-providers:
- class: "CountryFiles"
- class: "QRZ"
enabled: true
# Callsign data providers can often provide conflicting data, e.g. a home location from QRZ vs the centre of the
# DXCC entity from a country file lookup. The priority flag sets which source "wins" in the event of conflict.
# Lower numbers take priority over higher numbers. Generally then, QRZ/HamQTH should have low numbers as they are
# likely to have the most accurate data.
priority: 1
# No server-side credentials for QRZ. Users must provide their own as per QRZ policy.
- class: "ClublogXML"
- class: "HamQTH"
enabled: true
# API key for Clublog to look up information. Required in order to enable this provider. You will need to request
# one via their helpdesk portal if you want to use callsign lookups from Clublog.
api-key: ""
priority: 2
# No server-side credentials for HamQTH. Users must provide their own.
- class: "ClublogAPI"
# Querying the Clublog API directly doesn't provide any more data than the XML version, it just provides slightly
# more up-to-date information in the rare case that the prefix data changes, at a significant cost of looking up
# every callsign via an API call. Normally left disabled but it exists as an option.
enabled: false
# API key for Clublog to look up information. Required in order to enable this provider.
priority: 3
# API key for Clublog to look up information. Required in order to enable this provider. Unlike QRZ and HamQTH,
# Clublog uses an API key issued to Spothole, not to the end user.
api-key: ""
- class: "QRZ"
- class: "ClublogXML"
enabled: true
# No server-side credentials for QRZ. Users must provide their own as per QRZ policy.
priority: 4
# API key for Clublog to look up information. Required in order to enable this provider. You will need to request
# one via their helpdesk portal if you want to use callsign lookups from Clublog.
api-key: ""
- class: "HamQTH"
- class: "CountryFiles"
priority: 5
enabled: true
# No server-side credentials for HamQTH. Users must provide their own.
# Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week
+20 -27
View File
@@ -1,21 +1,4 @@
import gzip
import logging
import urllib.parse
from datetime import timedelta
import xmltodict
from diskcache import Cache
from pyhamtools import LookupLib, Callinfo, callinfo
from pyhamtools.exceptions import APIKeyMissingError
from pyhamtools.locator import latlong_to_locator
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests_cache import CachedSession
from core.config import config
from core.constants import HTTP_HEADERS, HAMQTH_PRG
from core.data_providers import DATA_PROVIDERS
from core.data_store import DATA_STORE
from core.url_data_cache import URLDataCache
from data.callsign import Callsign
@@ -24,14 +7,24 @@ def get_call_info(callsign, lookup_credentials):
lookup_credentials is an optional object that carries the user's QRZ.com/HamQTH credentials, if they provided them,
to enable lookup using those providers."""
callsign = Callsign(call=callsign)
for p in DATA_PROVIDERS.callsign_data_providers:
# Get new lookup data
data = p.lookup(callsign, lookup_credentials)
if data:
# Merge in turn, replacing any existing content where we have it.
for key, value in data.__dict__.items():
if value is not None:
callsign.__dict__[key] = value
callsign_data = Callsign(call=callsign)
return callsign
if callsign:
# Sort callsign providers by priority order, so we query the highest priority (lowest numbers) first, and only
# query other providers for data we are missing as we go along.
for p in sorted(DATA_PROVIDERS.callsign_data_providers, key=lambda p: p.priority):
if p.enabled:
# Get new lookup data
data = p.lookup(callsign, lookup_credentials)
if data:
# If we have new data for fields that were previously unpopulated, add them in
for key, value in data.__dict__.items():
if value is not None and callsign_data.__dict__.get(key) is None:
callsign_data.__dict__[key] = value
# If callsign data is fully populated, avoid looking up using other providers as their data will not
# be used
if callsign_data.fully_populated():
break
return callsign_data
+24 -18
View File
@@ -84,23 +84,29 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
"""Utility function to take the data provided by a PyHamTools CallInfo object and populate our own Callsign data
object from it"""
home_call = callinfo.get_homecall(callsign)
data = callinfo.get_all()
try:
home_call = callinfo.get_homecall(callsign)
data = callinfo.get_all(callsign)
country = data["country"] if "country" in data else None
dxcc_id = data["adif"] if "adif" in data else None
continent = data["continent"] if "continent" in data else None
cq_zone = data["cqz"] if "cqz" in data else None
itu_zone = data["ituz"] if "ituz" in data else None
lat = float(data["latitude"]) if "latitude" in data else None
lon = float(data["longitude"]) if "longitude" in data else None
country = data["country"] if "country" in data else None
dxcc_id = data["adif"] if "adif" in data else None
continent = data["continent"] if "continent" in data else None
cq_zone = data["cqz"] if "cqz" in data else None
itu_zone = data["ituz"] if "ituz" in data else None
lat = float(data["latitude"]) if "latitude" in data else None
lon = float(data["longitude"]) if "longitude" in data else None
return Callsign(call=callsign,
home_call=home_call,
country=country,
dxcc_id=dxcc_id,
continent=continent,
cq_zone=cq_zone,
itu_zone=itu_zone,
latitude=lat,
longitude=lon)
return Callsign(call=callsign,
home_call=home_call,
country=country,
dxcc_id=dxcc_id,
continent=continent,
cq_zone=cq_zone,
itu_zone=itu_zone,
latitude=lat,
longitude=lon,
location_source="DXCC")
except ValueError:
# Unknown callsign, can't look anything up, return a Callsign object with basic data so that gets cached
return Callsign(call=callsign)
+9 -10
View File
@@ -7,7 +7,7 @@ from datetime import datetime, timedelta
import pytz
from core.call_lookup_helper import lookup_helper
from core.call_lookup_helper import get_call_info
from core.sig_lookup_helper import populate_missing_sig_ref_info
from core.utils import get_flag_for_dxcc
@@ -86,16 +86,17 @@ class Alert:
# DX country, continent, zones etc. from callsign. CQ/ITU zone are better looked up with a location but we don't
# have a real location for alerts.
call_info = get_call_info(self.dx_calls[0], credentials)
if self.dx_calls and self.dx_calls[0] and not self.dx_country:
self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_calls[0], credentials)
self.dx_country = call_info.country
if self.dx_calls and self.dx_calls[0] and not self.dx_continent:
self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_calls[0], credentials)
self.dx_continent = call_info.continent
if self.dx_calls and self.dx_calls[0] and not self.dx_cq_zone:
self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_calls[0], credentials)
self.dx_cq_zone = call_info.cq_zone
if self.dx_calls and self.dx_calls[0] and not self.dx_itu_zone:
self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_calls[0], credentials)
self.dx_itu_zone = call_info.itu_zone
if self.dx_calls and self.dx_calls[0] and not self.dx_dxcc_id:
self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_calls[0], credentials)
self.dx_dxcc_id = call_info.dxcc_id
if self.dx_dxcc_id and not self.dx_flag:
self.dx_flag = get_flag_for_dxcc(self.dx_dxcc_id)
@@ -124,12 +125,10 @@ class Alert:
self_copy.received_time_iso = ""
self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
# DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data
# from the actual alerting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon
# instead of the one from the park reference they're at.
# DX operator name lookup, using QRZ.com/HamQTH.
if self.dx_calls and not self.dx_names:
self.dx_names = list(
map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c, credentials), self.dx_calls))
map(lambda c: get_call_info(c, credentials).name, self.dx_calls))
except Exception as e:
logging.error("Exception while inferring missing data from spot", e, exc_info=True)
+9
View File
@@ -35,4 +35,13 @@ class Callsign:
cq_zone: int | None = None
# ITU zone in which the callsign indicates they are operating
itu_zone: int | None = None
# Location source. This can be "HOME QTH" or "DXCC" depending on which provider gave us a location
location_source: str | None = None
def fully_populated(self):
"""Utility method to indicate that the callsign data is fully populated. Multiple providers can return data for
a callsign, and we try them in sequence until we have all the data we can, in which case there's no point
querying any other providers."""
return self.home_call is not None and self.name is not None and self.qth is not None and self.grid is not None\
and self.latitude is not None and self.longitude is not None and self.country is not None and self.continent\
is not None and self.dxcc_id is not None and self.cq_zone is not None and self.itu_zone is not None
+23 -53
View File
@@ -12,7 +12,7 @@ from pyhamtools.locator import locator_to_latlong, latlong_to_locator
from core.config import MAX_SPOT_AGE
from core.constants import MODE_ALIASES, PROPAGATION_MODES
from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
from core.call_lookup_helper import lookup_helper
from core.call_lookup_helper import get_call_info
from core.sig_utils import ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
from core.sig_lookup_helper import populate_missing_sig_ref_info
from core.utils import infer_band_from_freq, infer_mode_from_comment, \
@@ -173,12 +173,13 @@ class Spot:
self.dx_ssid = split[1]
# DX country, continent etc. from callsign
dx_call_info = get_call_info(self.dx_call, credentials)
if self.dx_call and not self.dx_country:
self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_call, credentials)
self.dx_country = dx_call_info.country
if self.dx_call and not self.dx_continent:
self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_call, credentials)
self.dx_continent = dx_call_info.continent
if self.dx_call and not self.dx_dxcc_id:
self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_call, credentials)
self.dx_dxcc_id = dx_call_info.dxcc_id
if self.dx_dxcc_id and not self.dx_flag:
self.dx_flag = get_flag_for_dxcc(self.dx_dxcc_id)
@@ -205,14 +206,15 @@ class Spot:
# Spotter country, continent, zones etc. from callsign.
# DE call with no digits, or APRS servers starting "T2" are not things we can look up location for
de_call_info = get_call_info(self.de_call, credentials)
if self.de_call and any(char.isdigit() for char in self.de_call) and not (
self.de_call.startswith("T2") and self.source == "APRS-IS"):
if not self.de_country:
self.de_country = lookup_helper.infer_country_from_callsign(self.de_call, credentials)
self.de_country = de_call_info.country
if not self.de_continent:
self.de_continent = lookup_helper.infer_continent_from_callsign(self.de_call, credentials)
self.de_continent = de_call_info.continent
if not self.de_dxcc_id:
self.de_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.de_call, credentials)
self.de_dxcc_id = de_call_info.dxcc_id
if self.de_dxcc_id and not self.de_flag:
self.de_flag = get_flag_for_dxcc(self.de_dxcc_id)
@@ -359,18 +361,16 @@ class Spot:
self_copy.received_time_iso = ""
self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
# DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data
# from the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon
# DX operator details lookup. This should be the last resort compared to taking the data from the actual
# spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon or DXCC lat/lon
# instead of the one from the park reference they're at.
if self.dx_call and not self.dx_name:
self.dx_name = lookup_helper.infer_name_from_callsign_online_lookup(self.dx_call, credentials)
self.dx_name = dx_call_info.name
if self.dx_call and not self.dx_latitude:
latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.dx_call, credentials)
if latlon:
self.dx_latitude = latlon[0]
self.dx_longitude = latlon[1]
self.dx_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.dx_call, credentials)
self.dx_location_source = "HOME QTH"
self.dx_latitude = dx_call_info.latitude
self.dx_longitude = dx_call_info.longitude
self.dx_grid = dx_call_info.grid
self.dx_location_source = dx_call_info.location_source
# Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable string,
# otherwise see what they have set on an online lookup service.
@@ -380,39 +380,19 @@ class Spot:
qth += " " + self.sig_refs[0].name
self.dx_qth = qth
else:
self.dx_qth = lookup_helper.infer_qth_from_callsign_online_lookup(self.dx_call, credentials)
# Last resort for getting a DX position, use the DXCC entity.
if self.dx_call and not self.dx_latitude:
latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.dx_call)
if latlon:
self.dx_latitude = latlon[0]
self.dx_longitude = latlon[1]
self.dx_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.dx_call)
self.dx_location_source = "DXCC"
# It looks like we can sometimes get a string into lat/lon, so try to parse as float, reject if not valid
if isinstance(self.dx_latitude, str) or isinstance(self.dx_longitude, str):
try:
self.dx_latitude = float(str(self.dx_latitude))
self.dx_longitude = float(str(self.dx_longitude))
except (TypeError, ValueError):
logging.warning("Received non-numeric strings in lat/lon (" + str(self.dx_latitude) + ", " + str(
self.dx_longitude) + ") for call " + str(self.dx_call) + ", rejecting it")
self.dx_latitude = None
self.dx_longitude = None
self.dx_qth = dx_call_info.qth
# CQ and ITU zone lookup, preferably from location but failing that, from callsign
if not self.dx_cq_zone:
if self.dx_latitude:
self.dx_cq_zone = lat_lon_to_cq_zone(self.dx_latitude, self.dx_longitude)
elif self.dx_call:
self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_call, credentials)
self.dx_cq_zone = dx_call_info.cq_zone
if not self.dx_itu_zone:
if self.dx_latitude:
self.dx_itu_zone = lat_lon_to_itu_zone(self.dx_latitude, self.dx_longitude)
elif self.dx_call:
self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_call, credentials)
self.dx_itu_zone = dx_call_info.itu_zone
# DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
# is likely at home.
@@ -424,21 +404,11 @@ class Spot:
# DE with no digits and APRS servers starting "T2" are not things we can look up location for
if self.de_call and any(char.isdigit() for char in self.de_call) and not (
self.de_call.startswith("T2") and self.source == "APRS-IS"):
# DE operator position lookup, using QRZ.com/HamQTH.
# DE operator location lookup
if not self.de_latitude:
latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.de_call, credentials)
if latlon:
self.de_latitude = latlon[0]
self.de_longitude = latlon[1]
self.de_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.de_call, credentials)
# Last resort for getting a DE position, use the DXCC entity.
if not self.de_latitude:
latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.de_call)
if latlon:
self.de_latitude = latlon[0]
self.de_longitude = latlon[1]
self.de_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.de_call)
self.de_latitude = de_call_info.latitude
self.de_longitude = de_call_info.longitude
self.de_grid = de_call_info.grid
except Exception as e:
logging.error("Exception while inferring missing data from spot", e, exc_info=True)
@@ -18,6 +18,9 @@ class APIQueryCallsignDataProvider(CallsignDataProvider):
""" Set up the provider."""
super().__init__(name, provider_config, storage)
if self.enabled:
self.status = "Ready"
def start(self):
pass
@@ -15,6 +15,7 @@ class CallsignDataProvider:
self.name = name
self.enabled = provider_config["enabled"]
self.priority = int(provider_config["priority"])
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
self.status = "Not Started" if self.enabled else "Disabled"
self._storage = storage
@@ -38,13 +39,16 @@ class CallsignDataProvider:
possible. Data is cached internally for a set period of 30 days to avoid the need to request data from servers
each time."""
if callsign in self._storage:
return self._storage[callsign]
if self.enabled:
if callsign in self._storage:
return self._storage[callsign]
else:
c = self._perform_new_lookup(callsign, lookup_credentials)
if c:
self._storage.set(callsign, c, expire=DATA_STORE.CALLSIGN_DATA_TTL_SEC)
return c
else:
c = self._perform_new_lookup(callsign, lookup_credentials)
if c:
self._storage.set(callsign, c, expire=DATA_STORE.CALLSIGN_DATA_TTL_SEC)
return c
return None
def _perform_new_lookup(self, callsign, lookup_credentials):
+6 -5
View File
@@ -28,16 +28,17 @@ class ClublogAPI(APIQueryCallsignDataProvider):
super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
self.status = "Ready"
def _perform_new_lookup(self, callsign, lookup_credentials):
callsign_data = Callsign(call=callsign)
try:
callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
if self._callinfo:
callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
else:
return None
except Exception as e:
self.status = "Error"
+4 -1
View File
@@ -49,4 +49,7 @@ class ClublogXML(FileDownloadCallsignDataProvider):
return False
def _perform_new_lookup(self, callsign, lookup_credentials):
return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
if self._callinfo:
return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
else:
return None
+4 -1
View File
@@ -30,4 +30,7 @@ class CountryFiles(FileDownloadCallsignDataProvider):
return False
def _perform_new_lookup(self, callsign, lookup_credentials):
return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
if self._callinfo:
return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
else:
return None
@@ -24,6 +24,9 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
self._stop_event = Event()
self._url_data_cache = URLDataCache("callsigndata_" + name)
if self.enabled:
self.status = "Ready"
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting.
+3 -3
View File
@@ -27,11 +27,10 @@ class HamQTH(APIQueryCallsignDataProvider):
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/hamqth-creds",
expire_after=timedelta(minutes=55))
self.status = "Ready"
def _perform_new_lookup(self, callsign, lookup_credentials):
# If we don't have HamQTH credentials, skip this lookup
if not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
if not lookup_credentials or not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
or lookup_credentials.hamqth_session_key):
return None
@@ -134,4 +133,5 @@ class HamQTH(APIQueryCallsignDataProvider):
grid=grid,
dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cq"]) if "cq" in data else None,
itu_zone=int(data["itu"]) if "itu" in data else None)
itu_zone=int(data["itu"]) if "itu" in data else None,
location_source="HOME QTH")
+3 -3
View File
@@ -25,11 +25,10 @@ class QRZ(APIQueryCallsignDataProvider):
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/qrz-creds",
expire_after=timedelta(minutes=55))
self.status = "Ready"
def _perform_new_lookup(self, callsign, lookup_credentials):
# If we don't have QRZ credentials, skip this lookup
if not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
if not lookup_credentials or not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
or lookup_credentials.qrz_session_key):
return None
@@ -153,4 +152,5 @@ class QRZ(APIQueryCallsignDataProvider):
grid=grid,
dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
itu_zone=int(data["ituzone"]) if "ituzone" in data else None)
itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
location_source="HOME QTH")
-4
View File
@@ -4,7 +4,6 @@ import os
import signal
import sys
from core.call_lookup_helper import lookup_helper
from core.config import SERVER_OWNER_CALLSIGN, LOG_LEVEL
from core.constants import SOFTWARE_VERSION
from core.data_providers import DATA_PROVIDERS
@@ -49,9 +48,6 @@ if __name__ == '__main__':
# Set up data store
DATA_STORE.setup()
# Set up lookup helper
lookup_helper.start()
# Set up and start data providers
DATA_PROVIDERS.setup()
DATA_PROVIDERS.start()