diff --git a/core/data_store.py b/core/data_store.py index 33c900a..0511dfe 100644 --- a/core/data_store.py +++ b/core/data_store.py @@ -4,17 +4,17 @@ from pathlib import Path import diskcache from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE -from core.constants import SIGS from core.live_data_cache import LiveDataCache from data.solar_conditions import SolarConditions +CACHE_DIR = "./cache/" + class DataStore: """Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign lookup data using different caching strategies for each.""" def __init__(self): - self._CACHE_DIR = "./cache" self._MAX_SPOT_COUNT = 100000 self._MAX_ALERT_COUNT = 100000 self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300 @@ -29,15 +29,15 @@ class DataStore: self._solar = None def setup(self): - Path(self._CACHE_DIR).mkdir(parents=True, exist_ok=True) + Path(CACHE_DIR).mkdir(parents=True, exist_ok=True) # Standard disk cache for solar data and status data, but each cache contains only a single object which we # expose to the wider application - self._solar = diskcache.Cache(self._CACHE_DIR + "/solar") + self._solar = diskcache.Cache(CACHE_DIR + "solar") if "solar_conditions" not in self._solar: self._solar.add("solar_conditions", SolarConditions()) self.solar_conditions = self._solar.get("solar_conditions") - self._status = diskcache.Cache(self._CACHE_DIR + "/status") + self._status = diskcache.Cache(CACHE_DIR + "status") if "status_data" not in self._status: self._status.add("status_data", {}) self.status_data = self._status.get("status_data") @@ -46,25 +46,25 @@ class DataStore: # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference, # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts, # so we have an ugly "SIG:ref" syntax for keys to keep it a single level. - self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs") + self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs") logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs)) # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup. # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign # we've seen, so we rely on them timing out and this triggering another lookup. - self.callsigns = diskcache.Cache(self._CACHE_DIR + "/callsigns") + self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns") logging.info(f"Loaded data for %d callsigns.", len(self.callsigns)) # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache # before the spots and alerts are live in the system. self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE, - snapshot_dir=self._CACHE_DIR + "/spots", + snapshot_dir=CACHE_DIR + "spots", snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys())) self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE, - snapshot_dir=self._CACHE_DIR + "/alerts", + snapshot_dir=CACHE_DIR + "alerts", snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys())) diff --git a/core/lookup_helper.py b/core/lookup_helper.py index 09d84d8..370a82a 100644 --- a/core/lookup_helper.py +++ b/core/lookup_helper.py @@ -17,7 +17,7 @@ from requests_cache import CachedSession from core.config import config from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \ HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES -from core.url_data_cache import URL_DATA_CACHE +from core.url_data_cache import URLDataCache # QRZ XML field names differ from pyhamtools' normalised names; map them here. _QRZ_FIELD_MAP = { @@ -29,6 +29,7 @@ _QRZ_FIELD_MAP = { } _QRZ_INT_FIELDS = {"adif", "cqz", "ituz"} _QRZ_FLOAT_FIELDS = {"latitude", "longitude"} +_URL_DATA_CACHE = URLDataCache("callsign_lookup") def _normalize_qrz_data(raw): @@ -142,7 +143,7 @@ class LookupHelper: try: logging.info("Downloading Country-files.com cty.plist...") - response = URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist", + response = _URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist", headers=HTTP_HEADERS) if response.ok: @@ -167,7 +168,7 @@ class LookupHelper: try: logging.info("Downloading dxcc.json...") - response = URL_DATA_CACHE.get( + response = _URL_DATA_CACHE.get( "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json", headers=HTTP_HEADERS) @@ -515,7 +516,7 @@ class LookupHelper: for lookup_call in calls_to_try: try: - response = URL_DATA_CACHE.get( + response = _URL_DATA_CACHE.get( self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call), headers=HTTP_HEADERS, timeout=10) if response.ok: @@ -593,7 +594,7 @@ class LookupHelper: for lookup_call in calls_to_try: try: - response = URL_DATA_CACHE.get( + response = _URL_DATA_CACHE.get( self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus( lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS) if response.ok: diff --git a/core/url_data_cache.py b/core/url_data_cache.py index 7119e96..d93dbcb 100644 --- a/core/url_data_cache.py +++ b/core/url_data_cache.py @@ -3,21 +3,23 @@ from datetime import timedelta from requests_cache import CachedSession -# Cache for "semi-static" data retrieved from a URL. This is a layet of caching in addition to the normal caching of -# spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints frequently when -# e.g. restarting Spothole many times during testing. -_session = CachedSession("cache/semi_static_urls", expire_after=timedelta(days=1), +from core.data_store import CACHE_DIR + + +class URLDataCache(CachedSession): + """Cache for "semi-static" data retrieved from a URL. This is an extra layer of caching in addition to the normal + caching of spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints + frequently when e.g. restarting Spothole many times during testing. It should only be used for data that isn't + expected to change on a sub-daily basis, so never for spots & alerts. It also implements a lock to allow it to be + used across multiple threads, though note that URL lookups will block each other this way, so it is still better to + create one of these objects per thread if possible.""" + + _lock = threading.Lock() + + def __init__(self, name): + super().__init__(CACHE_DIR + "urls/" + name, expire_after=timedelta(days=1), allowable_codes=(200, 400, 401, 403, 404)) -_lock = threading.Lock() - - -class _ThreadSafeSession: - """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads. This allows a single object - to be used freely across the application.""" def get(self, *args, **kwargs): - with _lock: - return _session.get(*args, **kwargs) - -# Global object -URL_DATA_CACHE = _ThreadSafeSession() + with self._lock: + return super().get(*args, **kwargs) \ No newline at end of file diff --git a/sigrefdataproviders/file_download_sig_ref_data_provider.py b/sigrefdataproviders/file_download_sig_ref_data_provider.py index 8920fe3..d070cc7 100644 --- a/sigrefdataproviders/file_download_sig_ref_data_provider.py +++ b/sigrefdataproviders/file_download_sig_ref_data_provider.py @@ -7,7 +7,7 @@ from requests import ReadTimeout from requests.exceptions import ConnectionError, ConnectTimeout from core.constants import HTTP_HEADERS -from core.url_data_cache import URL_DATA_CACHE +from core.url_data_cache import URLDataCache from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider @@ -21,6 +21,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider): self._poll_interval = poll_interval self._thread = None self._stop_event = Event() + self._url_data_cache = URLDataCache("sigrefdata-" + sig_name) def start(self): # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between @@ -44,7 +45,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider): # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for # caching, but just so continual restarts of the software during testing don't hammer the servers. logging.debug("Downloading " + self.sig_name + " SIG ref data...") - http_response = URL_DATA_CACHE.get(self._url, headers=HTTP_HEADERS) + http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS) # Check response code was good if http_response.ok: # Pass off to the subclass for processing diff --git a/spotproviders/gma.py b/spotproviders/gma.py index 35e2a2a..6c589bd 100644 --- a/spotproviders/gma.py +++ b/spotproviders/gma.py @@ -4,7 +4,7 @@ from datetime import datetime import pytz from core.constants import HTTP_HEADERS -from core.url_data_cache import URL_DATA_CACHE +from core.url_data_cache import URLDataCache from data.sig_ref import SIGRef from data.spot import Spot from spotproviders.http_spot_provider import HTTPSpotProvider @@ -21,12 +21,13 @@ class GMA(HTTPSpotProvider): def __init__(self, provider_config): # Ensure there is an API key in our config, and set up the query URL using it. If no key is provided, # disable this spot provider. - self.api_key = provider_config.get("api-key", "") - if self.api_key == "": + self._api_key = provider_config.get("api-key", "") + if self._api_key == "": provider_config["enabled"] = False logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.") + self._url_data_cache = URLDataCache("GMA") - super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self.api_key, self.POLL_INTERVAL_SEC) + super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self._api_key, self.POLL_INTERVAL_SEC) def _http_response_to_spots(self, http_response): new_spots = [] @@ -56,7 +57,7 @@ class GMA(HTTPSpotProvider): # GMA doesn't give what programme (SIG) the reference is for until we separately look it up. if "REF" in source_spot: try: - ref_response = URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"], + ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"], headers=HTTP_HEADERS) # Sometimes this is blank even if it's a 200 response, so handle that if ref_response.ok and ref_response.text is not None and ref_response.text != "":