Improve use of URL cache #118

core/data_store.py

@@ -4,17 +4,17 @@ from pathlib import Path
 import diskcache
 
 from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
-from core.constants import SIGS
 from core.live_data_cache import LiveDataCache
 from data.solar_conditions import SolarConditions
 
+CACHE_DIR = "./cache/"
+
 
 class DataStore:
     """Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
     lookup data using different caching strategies for each."""
 
     def __init__(self):
-        self._CACHE_DIR = "./cache"
         self._MAX_SPOT_COUNT = 100000
         self._MAX_ALERT_COUNT = 100000
         self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
@@ -29,15 +29,15 @@ class DataStore:
         self._solar = None
 
     def setup(self):
-        Path(self._CACHE_DIR).mkdir(parents=True, exist_ok=True)
+        Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
 
         # Standard disk cache for solar data and status data, but each cache contains only a single object which we
         # expose to the wider application
-        self._solar = diskcache.Cache(self._CACHE_DIR + "/solar")
+        self._solar = diskcache.Cache(CACHE_DIR + "solar")
         if "solar_conditions" not in self._solar:
             self._solar.add("solar_conditions", SolarConditions())
         self.solar_conditions = self._solar.get("solar_conditions")
-        self._status = diskcache.Cache(self._CACHE_DIR + "/status")
+        self._status = diskcache.Cache(CACHE_DIR + "status")
         if "status_data" not in self._status:
             self._status.add("status_data", {})
         self.status_data = self._status.get("status_data")
@@ -46,25 +46,25 @@ class DataStore:
         # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
         # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
         # so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
-        self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs")
+        self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
         logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
 
         # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
         # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
         # we've seen, so we rely on them timing out and this triggering another lookup.
-        self.callsigns = diskcache.Cache(self._CACHE_DIR + "/callsigns")
+        self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns")
         logging.info(f"Loaded data for %d callsigns.", len(self.callsigns))
 
         # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
         # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
         # before the spots and alerts are live in the system.
         self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
-                                   snapshot_dir=self._CACHE_DIR + "/spots",
+                                   snapshot_dir=CACHE_DIR + "spots",
                                    snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
         logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys()))
 
         self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
-                                    snapshot_dir=self._CACHE_DIR + "/alerts",
+                                    snapshot_dir=CACHE_DIR + "alerts",
                                     snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
         logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
 

core/lookup_helper.py

@@ -17,7 +17,7 @@ from requests_cache import CachedSession
 from core.config import config
 from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
     HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
-from core.url_data_cache import URL_DATA_CACHE
+from core.url_data_cache import URLDataCache
 
 # QRZ XML field names differ from pyhamtools' normalised names; map them here.
 _QRZ_FIELD_MAP = {
@@ -29,6 +29,7 @@ _QRZ_FIELD_MAP = {
 }
 _QRZ_INT_FIELDS = {"adif", "cqz", "ituz"}
 _QRZ_FLOAT_FIELDS = {"latitude", "longitude"}
+_URL_DATA_CACHE = URLDataCache("callsign_lookup")
 
 
 def _normalize_qrz_data(raw):
@@ -142,7 +143,7 @@ class LookupHelper:
 
         try:
             logging.info("Downloading Country-files.com cty.plist...")
-            response = URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
+            response = _URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
                                           headers=HTTP_HEADERS)
 
             if response.ok:
@@ -167,7 +168,7 @@ class LookupHelper:
 
         try:
             logging.info("Downloading dxcc.json...")
-            response = URL_DATA_CACHE.get(
+            response = _URL_DATA_CACHE.get(
                 "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
                 headers=HTTP_HEADERS)
 
@@ -515,7 +516,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = URL_DATA_CACHE.get(
+                response = _URL_DATA_CACHE.get(
                     self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
                     headers=HTTP_HEADERS, timeout=10)
                 if response.ok:
@@ -593,7 +594,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = URL_DATA_CACHE.get(
+                response = _URL_DATA_CACHE.get(
                     self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                         lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
                 if response.ok:

core/url_data_cache.py

@@ -3,21 +3,23 @@ from datetime import timedelta
 
 from requests_cache import CachedSession
 
-# Cache for "semi-static" data retrieved from a URL. This is a layet of caching in addition to the normal caching of
+from core.data_store import CACHE_DIR
-# spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints frequently when
+
-# e.g. restarting Spothole many times during testing.
+
-_session = CachedSession("cache/semi_static_urls", expire_after=timedelta(days=1),
+class URLDataCache(CachedSession):
+    """Cache for "semi-static" data retrieved from a URL. This is an extra layer of caching in addition to the normal
+    caching of spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints
+    frequently when e.g. restarting Spothole many times during testing. It should only be used for data that isn't
+    expected to change on a sub-daily basis, so never for spots & alerts. It also implements a lock to allow it to be
+    used across multiple threads, though note that URL lookups will block each other this way, so it is still better to
+    create one of these objects per thread if possible."""
+
+    _lock = threading.Lock()
+
+    def __init__(self, name):
+        super().__init__(CACHE_DIR + "urls/" + name, expire_after=timedelta(days=1),
                          allowable_codes=(200, 400, 401, 403, 404))
-_lock = threading.Lock()
-
-
-class _ThreadSafeSession:
-    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads. This allows a single object
-    to be used freely across the application."""
 
     def get(self, *args, **kwargs):
-        with _lock:
+        with self._lock:
-            return _session.get(*args, **kwargs)
+            return super().get(*args, **kwargs)
-
-# Global object
-URL_DATA_CACHE = _ThreadSafeSession()

sigrefdataproviders/file_download_sig_ref_data_provider.py

@@ -7,7 +7,7 @@ from requests import ReadTimeout
 from requests.exceptions import ConnectionError, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
-from core.url_data_cache import URL_DATA_CACHE
+from core.url_data_cache import URLDataCache
 from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
 
 
@@ -21,6 +21,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         self._poll_interval = poll_interval
         self._thread = None
         self._stop_event = Event()
+        self._url_data_cache = URLDataCache("sigrefdata-" + sig_name)
 
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
@@ -44,7 +45,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
             # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
             # caching, but just so continual restarts of the software during testing don't hammer the servers.
             logging.debug("Downloading " + self.sig_name + " SIG ref data...")
-            http_response = URL_DATA_CACHE.get(self._url, headers=HTTP_HEADERS)
+            http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
             # Check response code was good
             if http_response.ok:
                 # Pass off to the subclass for processing

spotproviders/gma.py

@@ -4,7 +4,7 @@ from datetime import datetime
 import pytz
 
 from core.constants import HTTP_HEADERS
-from core.url_data_cache import URL_DATA_CACHE
+from core.url_data_cache import URLDataCache
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -21,12 +21,13 @@ class GMA(HTTPSpotProvider):
     def __init__(self, provider_config):
         # Ensure there is an API key in our config, and set up the query URL using it. If no key is provided,
         # disable this spot provider.
-        self.api_key = provider_config.get("api-key", "")
+        self._api_key = provider_config.get("api-key", "")
-        if self.api_key == "":
+        if self._api_key == "":
             provider_config["enabled"] = False
             logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
+        self._url_data_cache = URLDataCache("GMA")
 
-        super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self.api_key, self.POLL_INTERVAL_SEC)
+        super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self._api_key, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -56,7 +57,7 @@ class GMA(HTTPSpotProvider):
                 # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
                 if "REF" in source_spot:
                     try:
-                        ref_response = URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
+                        ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
                                                           headers=HTTP_HEADERS)
                         # Sometimes this is blank even if it's a 200 response, so handle that
                         if ref_response.ok and ref_response.text is not None and ref_response.text != "":