w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-05 18:11:41 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Improve use of URL cache #118

Browse Source
This commit is contained in:
Ian Renton
2026-08-01 08:06:44 +01:00
parent fe10943ecc
commit c472872e10
5 changed files with 41 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/data_store.py
+9 -9
View File
@@ -4,17 +4,17 @@ from pathlib import Path
import diskcache
from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
from core.constants import SIGS
from core.live_data_cache import LiveDataCache
from data.solar_conditions import SolarConditions
CACHE_DIR = "./cache/"
class DataStore:
"""Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
lookup data using different caching strategies for each."""
def __init__(self):
self._CACHE_DIR = "./cache"
self._MAX_SPOT_COUNT = 100000
self._MAX_ALERT_COUNT = 100000
self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
@@ -29,15 +29,15 @@ class DataStore:
self._solar = None
def setup(self):
Path(self._CACHE_DIR).mkdir(parents=True, exist_ok=True)
Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
# Standard disk cache for solar data and status data, but each cache contains only a single object which we
# expose to the wider application
self._solar = diskcache.Cache(self._CACHE_DIR + "/solar")
self._solar = diskcache.Cache(CACHE_DIR + "solar")
if "solar_conditions" not in self._solar:
self._solar.add("solar_conditions", SolarConditions())
self.solar_conditions = self._solar.get("solar_conditions")
self._status = diskcache.Cache(self._CACHE_DIR + "/status")
self._status = diskcache.Cache(CACHE_DIR + "status")
if "status_data" not in self._status:
self._status.add("status_data", {})
self.status_data = self._status.get("status_data")
@@ -46,25 +46,25 @@ class DataStore:
# but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
# and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
# so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs")
self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
# we've seen, so we rely on them timing out and this triggering another lookup.
self.callsigns = diskcache.Cache(self._CACHE_DIR + "/callsigns")
self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns")
logging.info(f"Loaded data for %d callsigns.", len(self.callsigns))
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
# specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
# before the spots and alerts are live in the system.
self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
snapshot_dir=self._CACHE_DIR + "/spots",
snapshot_dir=CACHE_DIR + "spots",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys()))
self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
snapshot_dir=self._CACHE_DIR + "/alerts",
snapshot_dir=CACHE_DIR + "alerts",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
core/lookup_helper.py
+6 -5
View File
@@ -17,7 +17,7 @@ from requests_cache import CachedSession
from core.config import config
from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
from core.url_data_cache import URL_DATA_CACHE
from core.url_data_cache import URLDataCache
# QRZ XML field names differ from pyhamtools' normalised names; map them here.
_QRZ_FIELD_MAP = {
@@ -29,6 +29,7 @@ _QRZ_FIELD_MAP = {
}
_QRZ_INT_FIELDS = {"adif", "cqz", "ituz"}
_QRZ_FLOAT_FIELDS = {"latitude", "longitude"}
_URL_DATA_CACHE = URLDataCache("callsign_lookup")
def _normalize_qrz_data(raw):
@@ -142,7 +143,7 @@ class LookupHelper:
try:
logging.info("Downloading Country-files.com cty.plist...")
response = URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
response = _URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
headers=HTTP_HEADERS)
if response.ok:
@@ -167,7 +168,7 @@ class LookupHelper:
try:
logging.info("Downloading dxcc.json...")
response = URL_DATA_CACHE.get(
response = _URL_DATA_CACHE.get(
"https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
headers=HTTP_HEADERS)
@@ -515,7 +516,7 @@ class LookupHelper:
for lookup_call in calls_to_try:
try:
response = URL_DATA_CACHE.get(
response = _URL_DATA_CACHE.get(
self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
headers=HTTP_HEADERS, timeout=10)
if response.ok:
@@ -593,7 +594,7 @@ class LookupHelper:
for lookup_call in calls_to_try:
try:
response = URL_DATA_CACHE.get(
response = _URL_DATA_CACHE.get(
self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
if response.ok:
core/url_data_cache.py
+17 -15
View File
@@ -3,21 +3,23 @@ from datetime import timedelta
from requests_cache import CachedSession
# Cache for "semi-static" data retrieved from a URL. This is a layet of caching in addition to the normal caching of
# spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints frequently when
# e.g. restarting Spothole many times during testing.
_session = CachedSession("cache/semi_static_urls", expire_after=timedelta(days=1),
from core.data_store import CACHE_DIR
class URLDataCache(CachedSession):
"""Cache for "semi-static" data retrieved from a URL. This is an extra layer of caching in addition to the normal
caching of spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints
frequently when e.g. restarting Spothole many times during testing. It should only be used for data that isn't
expected to change on a sub-daily basis, so never for spots & alerts. It also implements a lock to allow it to be
used across multiple threads, though note that URL lookups will block each other this way, so it is still better to
create one of these objects per thread if possible."""
_lock = threading.Lock()
def __init__(self, name):
super().__init__(CACHE_DIR + "urls/" + name, expire_after=timedelta(days=1),
allowable_codes=(200, 400, 401, 403, 404))
_lock = threading.Lock()
class _ThreadSafeSession:
"""Wraps CachedSession with a lock to prevent concurrent SQLite access across threads. This allows a single object
to be used freely across the application."""
def get(self, *args, **kwargs):
with _lock:
return _session.get(*args, **kwargs)
# Global object
URL_DATA_CACHE = _ThreadSafeSession()
with self._lock:
return super().get(*args, **kwargs)
sigrefdataproviders/file_download_sig_ref_data_provider.py
+3 -2
View File
@@ -7,7 +7,7 @@ from requests import ReadTimeout
from requests.exceptions import ConnectionError, ConnectTimeout
from core.constants import HTTP_HEADERS
from core.url_data_cache import URL_DATA_CACHE
from core.url_data_cache import URLDataCache
from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
@@ -21,6 +21,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
self._poll_interval = poll_interval
self._thread = None
self._stop_event = Event()
self._url_data_cache = URLDataCache("sigrefdata-" + sig_name)
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
@@ -44,7 +45,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
# Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
# caching, but just so continual restarts of the software during testing don't hammer the servers.
logging.debug("Downloading " + self.sig_name + " SIG ref data...")
http_response = URL_DATA_CACHE.get(self._url, headers=HTTP_HEADERS)
http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
# Check response code was good
if http_response.ok:
# Pass off to the subclass for processing
spotproviders/gma.py
+6 -5
View File
@@ -4,7 +4,7 @@ from datetime import datetime
import pytz
from core.constants import HTTP_HEADERS
from core.url_data_cache import URL_DATA_CACHE
from core.url_data_cache import URLDataCache
from data.sig_ref import SIGRef
from data.spot import Spot
from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -21,12 +21,13 @@ class GMA(HTTPSpotProvider):
def __init__(self, provider_config):
# Ensure there is an API key in our config, and set up the query URL using it. If no key is provided,
# disable this spot provider.
self.api_key = provider_config.get("api-key", "")
if self.api_key == "":
self._api_key = provider_config.get("api-key", "")
if self._api_key == "":
provider_config["enabled"] = False
logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
self._url_data_cache = URLDataCache("GMA")
super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self.api_key, self.POLL_INTERVAL_SEC)
super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self._api_key, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
@@ -56,7 +57,7 @@ class GMA(HTTPSpotProvider):
# GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
if "REF" in source_spot:
try:
ref_response = URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
headers=HTTP_HEADERS)
# Sometimes this is blank even if it's a 200 response, so handle that
if ref_response.ok and ref_response.text is not None and ref_response.text != "":