From 459d999f57cbdf24fe249908b5d12b7bfd5c2104 Mon Sep 17 00:00:00 2001 From: Ian Renton Date: Mon, 3 Aug 2026 22:15:29 +0100 Subject: [PATCH] Refactor of caching & data storage part 16 #118 --- core/data_providers.py | 19 +++++++------- core/status_reporter.py | 3 ++- core/utils.py | 7 ++++- .../callsigndata/callsign_data_provider.py | 1 + providers/callsigndata/clublogapi.py | 1 + providers/callsigndata/clublogxml.py | 22 +++++++++++++--- providers/callsigndata/countryfiles.py | 20 +++++++++++--- providers/callsigndata/hamqth.py | 20 +++++++------- providers/callsigndata/qrz.py | 4 ++- server/handlers/api/lookups.py | 26 +++---------------- server/sse_broadcaster.py | 7 +++-- server/webserver.py | 5 ++++ static/apidocs/openapi.yml | 12 +++++++-- static/js/status.js | 1 + 14 files changed, 93 insertions(+), 55 deletions(-) diff --git a/core/data_providers.py b/core/data_providers.py index 91ee3cc..89a6792 100644 --- a/core/data_providers.py +++ b/core/data_providers.py @@ -29,15 +29,7 @@ class DataProviders: def start(self): - for p in self.spot_providers: - if p.enabled: - p.start() - for p in self.alert_providers: - if p.enabled: - p.start() - for p in self.solar_condition_providers: - if p.enabled: - p.start() + # Start data providers before spot/alert providers so the lookup data is there already for incoming spots for p in self.static_data_providers: if p.enabled: p.start() @@ -47,6 +39,15 @@ class DataProviders: for p in self.callsign_data_providers: if p.enabled: p.start() + for p in self.spot_providers: + if p.enabled: + p.start() + for p in self.alert_providers: + if p.enabled: + p.start() + for p in self.solar_condition_providers: + if p.enabled: + p.start() def stop(self): for sp in self.spot_providers: diff --git a/core/status_reporter.py b/core/status_reporter.py index f2219e4..47211d5 100644 --- a/core/status_reporter.py +++ b/core/status_reporter.py @@ -84,7 +84,8 @@ class StatusReporter: DATA_STORE.status_data["callsign_data_providers"] = list( map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, "last_updated": p.last_update_time.replace( - tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, + tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0, + "lookup_count": p.lookup_count}, DATA_PROVIDERS.callsign_data_providers)) DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"], "last_api_access": WEB_SERVER.web_server_metrics[ diff --git a/core/utils.py b/core/utils.py index bfd5586..bd6db9c 100644 --- a/core/utils.py +++ b/core/utils.py @@ -2,6 +2,7 @@ import logging import simplejson from pyhamtools.frequency import freq_to_band +from pyhamtools.locator import latlong_to_locator from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES from core.data_store import DATA_STORE @@ -95,6 +96,9 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo): itu_zone = data["ituz"] if "ituz" in data else None lat = float(data["latitude"]) if "latitude" in data else None lon = float(data["longitude"]) if "longitude" in data else None + grid = None + if lat and lon: + grid = latlong_to_locator(lat, lon) return Callsign(call=callsign, home_call=home_call, @@ -105,8 +109,9 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo): itu_zone=itu_zone, latitude=lat, longitude=lon, + grid=grid, location_source="DXCC") - except ValueError: + except (KeyError, ValueError): # Unknown callsign, can't look anything up, return a Callsign object with basic data so that gets cached return Callsign(call=callsign) diff --git a/providers/callsigndata/callsign_data_provider.py b/providers/callsigndata/callsign_data_provider.py index 6281a51..05d15f2 100644 --- a/providers/callsigndata/callsign_data_provider.py +++ b/providers/callsigndata/callsign_data_provider.py @@ -19,6 +19,7 @@ class CallsignDataProvider: self.priority = int(provider_config["priority"]) self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC) self.status = "Not Started" if self.enabled else "Disabled" + self.lookup_count = 0 self._storage = storage def start(self): diff --git a/providers/callsigndata/clublogapi.py b/providers/callsigndata/clublogapi.py index 135e8a4..c0f39c4 100644 --- a/providers/callsigndata/clublogapi.py +++ b/providers/callsigndata/clublogapi.py @@ -37,6 +37,7 @@ class ClublogAPI(APIQueryCallsignDataProvider): callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo) self.status = "OK" self.last_update_time = datetime.now(pytz.UTC) + self.lookup_count += 1 else: return None diff --git a/providers/callsigndata/clublogxml.py b/providers/callsigndata/clublogxml.py index c964e28..f5c1b26 100644 --- a/providers/callsigndata/clublogxml.py +++ b/providers/callsigndata/clublogxml.py @@ -1,10 +1,13 @@ import gzip import logging +from datetime import datetime +import pytz from pyhamtools import LookupLib, Callinfo from core.data_store import DATA_STORE from core.utils import get_callsign_object_from_pyhamtools_callinfo +from data.callsign import Callsign from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider @@ -49,7 +52,18 @@ class ClublogXML(FileDownloadCallsignDataProvider): return False def _perform_new_lookup(self, callsign, lookup_credentials): - if self._callinfo: - return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo) - else: - return None + callsign_data = Callsign(call=callsign) + + try: + if self._callinfo: + callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo) + self.status = "OK" + self.lookup_count += 1 + else: + return None + + except Exception as e: + self.status = "Error" + logging.error("Exception when looking up data from Clublog XML data", e, exc_info=True) + + return callsign_data diff --git a/providers/callsigndata/countryfiles.py b/providers/callsigndata/countryfiles.py index 76ffaba..1c647bb 100644 --- a/providers/callsigndata/countryfiles.py +++ b/providers/callsigndata/countryfiles.py @@ -4,6 +4,7 @@ from pyhamtools import LookupLib, Callinfo from core.data_store import DATA_STORE from core.utils import get_callsign_object_from_pyhamtools_callinfo +from data.callsign import Callsign from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider @@ -30,7 +31,18 @@ class CountryFiles(FileDownloadCallsignDataProvider): return False def _perform_new_lookup(self, callsign, lookup_credentials): - if self._callinfo: - return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo) - else: - return None + callsign_data = Callsign(call=callsign) + + try: + if self._callinfo: + callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo) + self.status = "OK" + self.lookup_count += 1 + else: + return None + + except Exception as e: + self.status = "Error" + logging.error("Exception when looking up data from Country file", e, exc_info=True) + + return callsign_data diff --git a/providers/callsigndata/hamqth.py b/providers/callsigndata/hamqth.py index 9216d27..596f513 100644 --- a/providers/callsigndata/hamqth.py +++ b/providers/callsigndata/hamqth.py @@ -1,7 +1,8 @@ import logging import urllib.parse -from datetime import timedelta +from datetime import timedelta, datetime +import pytz import xmltodict from pyhamtools import callinfo from requests import ConnectTimeout, ReadTimeout @@ -32,14 +33,14 @@ class HamQTH(APIQueryCallsignDataProvider): # If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because # # someone might provide credentials next time around. if not lookup_credentials or not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password) - or lookup_credentials.hamqth_session_key): + or lookup_credentials.hamqth_session_id): return None try: # Obtain session key from credentials, by looking it up from username & password if necessary. - session_key = None - if lookup_credentials.hamqth_session_key: - session_key = lookup_credentials.hamqth_session_key + session_id = None + if lookup_credentials.hamqth_session_id: + session_id = lookup_credentials.hamqth_session_id elif lookup_credentials.hamqth_username and lookup_credentials.hamqth_password: try: session_data = self._CREDENTIALS_CACHE.get( @@ -48,7 +49,7 @@ class HamQTH(APIQueryCallsignDataProvider): headers=HTTP_HEADERS).content dict_data = xmltodict.parse(session_data) if "session_id" in dict_data["HamQTH"]["session"]: - session_key = str(dict_data["HamQTH"]["session"]["session_id"]) + session_id = str(dict_data["HamQTH"]["session"]["session_id"]) else: # Log this failure at debug level only, not our problem if user entered the wrong password. logging.debug("HamQTH login details incorrect, failed to look up with HamQTH.") @@ -57,7 +58,7 @@ class HamQTH(APIQueryCallsignDataProvider): logging.error("Exception when getting HamQTH session key") return None - if not session_key: + if not session_id: return None # Try the call as given, then fall back to the base call (strips /P, /M etc.) @@ -73,13 +74,14 @@ class HamQTH(APIQueryCallsignDataProvider): for lookup_call in calls_to_try: try: response = self._URL_DATA_CACHE.get( - self._HAMQTH_BASE_URL + "?id=" + session_key + "&callsign=" + urllib.parse.quote_plus( + self._HAMQTH_BASE_URL + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus( lookup_call) + "&prg=" + self._PRG, headers=HTTP_HEADERS, timeout=10) if response.ok: self.status = "OK" self.last_update_time = datetime.now(pytz.UTC) - data = xmltodict.parse(response.content)["HamQTH"]["search"] + self.lookup_count += 1 # Found data, convert it to our object and return it + data = xmltodict.parse(response.content)["HamQTH"]["search"] return self.hamqth_response_to_callsign(callsign, data) elif not response.from_cache: diff --git a/providers/callsigndata/qrz.py b/providers/callsigndata/qrz.py index 839ef83..a1ea0f5 100644 --- a/providers/callsigndata/qrz.py +++ b/providers/callsigndata/qrz.py @@ -1,7 +1,8 @@ import logging import urllib.parse -from datetime import timedelta +from datetime import timedelta, datetime +import pytz import xmltodict from pyhamtools import callinfo from requests import ConnectTimeout, ReadTimeout @@ -80,6 +81,7 @@ class QRZ(APIQueryCallsignDataProvider): if "Callsign" in qrz_response: self.status = "OK" self.last_update_time = datetime.now(pytz.UTC) + self.lookup_count += 1 # Found data, convert it to our object and return it return self.qrz_response_to_callsign(callsign, qrz_response.get("Callsign")) diff --git a/server/handlers/api/lookups.py b/server/handlers/api/lookups.py index 785576b..5b497e1 100644 --- a/server/handlers/api/lookups.py +++ b/server/handlers/api/lookups.py @@ -8,15 +8,15 @@ import tornado from tornado import httputil from tornado.web import Application +from core.call_lookup_helper import get_call_info from core.constants import SIGS from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone from core.prometheus_metrics_handler import api_requests_counter -from core.sig_utils import get_ref_regex_for_sig from core.sig_lookup_helper import populate_missing_sig_ref_info +from core.sig_utils import get_ref_regex_for_sig from core.utils import safe_json_dumps from data.lookup_credentials import extract_credentials from data.sig_ref import SIGRef -from data.spot import Spot class APILookupCallHandler(tornado.web.RequestHandler): @@ -45,27 +45,9 @@ class APILookupCallHandler(tornado.web.RequestHandler): if "call" in query_params.keys(): call = str(query_params.get("call")).upper() if re.match(r"^[A-Z0-9/\-]*$", call): - # Take the callsign, make a "fake spot" so we can run infer_missing() on it, then repack the - # resulting data in the correct way for the API response. credentials = extract_credentials(query_params) - fake_spot = Spot(dx_call=call) - fake_spot.infer_missing(credentials) - data = { - "call": call, - "name": fake_spot.dx_name, - "qth": fake_spot.dx_qth, - "country": fake_spot.dx_country, - "flag": fake_spot.dx_flag, - "continent": fake_spot.dx_continent, - "dxcc_id": fake_spot.dx_dxcc_id, - "cq_zone": fake_spot.dx_cq_zone, - "itu_zone": fake_spot.dx_itu_zone, - "grid": fake_spot.dx_grid, - "latitude": fake_spot.dx_latitude, - "longitude": fake_spot.dx_longitude, - "location_source": fake_spot.dx_location_source - } - self.write(safe_json_dumps(data)) + callsign_data = get_call_info(call, credentials) + self.write(safe_json_dumps(callsign_data)) else: self.write(safe_json_dumps("Error - '" + call + "' does not look like a valid callsign.")) diff --git a/server/sse_broadcaster.py b/server/sse_broadcaster.py index a538117..b52a2dd 100644 --- a/server/sse_broadcaster.py +++ b/server/sse_broadcaster.py @@ -11,6 +11,9 @@ class SSEBroadcaster: def __init__(self): self._handlers = set() self._lock = threading.Lock() + self._loop = None + + def bind_to_web_server_loop(self): self._loop = IOLoop.current() def register(self, handler): @@ -22,9 +25,9 @@ class SSEBroadcaster: self._handlers.discard(handler) def publish(self, value): - self._loop.add_callback(self._fan_out, value) + self._loop.add_callback(self._broadcast, value) - def _fan_out(self, value): + def _broadcast(self, value): with self._lock: handlers = list(self._handlers) for handler in handlers: diff --git a/server/webserver.py b/server/webserver.py index fbc80eb..7e32325 100644 --- a/server/webserver.py +++ b/server/webserver.py @@ -1,6 +1,7 @@ import asyncio import logging import os +import threading import tornado from tornado.web import StaticFileHandler @@ -61,6 +62,10 @@ class WebServer: async def _start_inner(self): """Start method (async). Sets up the Tornado application.""" + # Bind the SSE broadcasters to the web server's loop, so they fire correctly + self._spot_broadcaster.bind_to_web_server_loop() + self._alert_broadcaster.bind_to_web_server_loop() + # Prepare a list of common arguments that are passed in to every API & page handler. This is just a basic thing # to avoid copy-pasting the same thing to every route declaration below. handler_opts = {"web_server_metrics": self.web_server_metrics} diff --git a/static/apidocs/openapi.yml b/static/apidocs/openapi.yml index 5b735ab..2c60731 100644 --- a/static/apidocs/openapi.yml +++ b/static/apidocs/openapi.yml @@ -325,7 +325,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/CallLookup' + $ref: '#/components/schemas/CallsignData' '422': description: Validation error e.g. callsign missing or format incorrect content: @@ -1791,6 +1791,10 @@ components: The last time at which this provider received data, UTC seconds since UNIX epoch. If this is zero, the provider has never updated. example: 1759579508 + lookup_count: + type: number + description: The number of callsign lookups performed using this provider since the server was started. + example: 1234 SpotList: type: array @@ -1962,12 +1966,16 @@ components: on this server. example: true - CallLookup: + CallsignData: type: object properties: call: type: string description: Callsign, as provided to the API + example: DL/M0TRT/P + home_call: + type: string + description: The "home" call, without prefixes or suffixes example: M0TRT name: type: string diff --git a/static/js/status.js b/static/js/status.js index 237f638..5994265 100644 --- a/static/js/status.js +++ b/static/js/status.js @@ -65,6 +65,7 @@ function loadStatus() {
${p["name"]}
Status: ${p["status"]}
Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}
+
Lookups: ${(p["enabled"] && p["lookup_count"] > 0) ? p["lookup_count"] : "N/A"}
`); }); });