From 11a236e668dc75667c5112ee092e0145207c489a Mon Sep 17 00:00:00 2001 From: Ian Renton Date: Sun, 2 Aug 2026 10:31:31 +0100 Subject: [PATCH] Refactor of caching & data storage part 10 #118 --- config-example.yml | 45 ++++++---- core/config.py | 40 ++------- core/data_providers.py | 72 +++++++++++++++ core/data_store.py | 4 +- core/status_reporter.py | 72 +++++++-------- data/callsign.py | 10 +-- .../callsigndata/callsign_data_provider.py | 36 ++++++++ providers/callsigndata/clublogxml.py | 68 ++++++++++++++ providers/callsigndata/countryfiles.py | 48 ++++++++++ .../file_download_callsign_data_provider.py | 85 ++++++++++++++++++ server/webserver.py | 5 ++ spothole.py | 90 ++++--------------- static/apidocs/openapi.yml | 29 +++++- static/js/status.js | 9 ++ templates/status.html | 9 ++ 15 files changed, 452 insertions(+), 170 deletions(-) create mode 100644 core/data_providers.py create mode 100644 providers/callsigndata/callsign_data_provider.py create mode 100644 providers/callsigndata/clublogxml.py create mode 100644 providers/callsigndata/countryfiles.py create mode 100644 providers/callsigndata/file_download_callsign_data_provider.py diff --git a/config-example.yml b/config-example.yml index dd02187..158d966 100644 --- a/config-example.yml +++ b/config-example.yml @@ -166,7 +166,23 @@ alert-providers: enabled: true -# Static reference data providers to use. This allows Spothole to download data such as mapping between callsign +# Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band +# conditions, etc.) and make it available via the /api/v1/solar endpoint. +solar-condition-providers: + - class: "HamQSL" + enabled: true + + - class: "NOAA3dayForecast" + enabled: true + + - class: "GIROIonosonde" + enabled: true + + - class: "KC2GProp" + enabled: true + + +# Static reference data providers to use. These allow Spothole to download data such as mapping between callsign # prefixes and DXCC entities. static-data-providers: - class: "K0SWE" @@ -179,7 +195,7 @@ static-data-providers: enabled: true -# SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF +# SIG reference data providers to use. These allow Spothole to download, for example, the WWFF directory that maps WWFF # park IDs to their name and location. sig-ref-data-providers: - class: "POTA" @@ -233,30 +249,25 @@ sig-ref-data-providers: - class: "Toilets" enabled: true -# Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band -# conditions, etc.) and make it available via the /api/v1/solar endpoint. -solar-condition-providers: - - class: "HamQSL" + +# Callsign data providers to use. These allow Spothole to provide information about callsigns, either from static +# lookups or from web-based services such as QRZ. +callsign-data-providers: + - class: "CountryFiles" enabled: true - - class: "NOAA3dayForecast" - enabled: true + - class: "ClublogXML" + enabled: false + # API key for Clublog to look up information. Required in order to enable this provider. You will need to request + # one via their helpdesk portal if you want to use callsign lookups from Clublog. + clublog-api-key: "" - - class: "GIROIonosonde" - enabled: true - - - class: "KC2GProp" - enabled: true # Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week # for alerts. max-spot-age-sec: 3600 max-alert-age-sec: 604800 -# API key for Clublog to look up information. Optional. You sill need to request one via their helpdesk portal if you -# want to use callsign lookups from Clublog. -clublog-api-key: "" - # Allow submitting spots to the Spothole API? allow-spotting: true diff --git a/core/config.py b/core/config.py index 6c99ef4..804b4a8 100644 --- a/core/config.py +++ b/core/config.py @@ -36,41 +36,11 @@ if ALLOW_SPOTTING: WEB_UI_OPTIONS["spot-providers-enabled-by-default"].append("API") -def get_spot_provider_from_config(config_providers_entry): - """Utility method to get a spot provider based on the class specified in its config entry.""" +def create_provider_from_config(package, config_providers_entry): + """Utility method to get a provider based on the class specified in its config entry. You must also provide the + package to look for it in, as there are several types of provider. e.g. package "providers.spot", where the config + entry is for a POTA spot provider.""" - module = importlib.import_module('providers.spot.' + config_providers_entry["class"].lower()) - provider_class = getattr(module, config_providers_entry["class"]) - return provider_class(config_providers_entry) - - -def get_alert_provider_from_config(config_providers_entry): - """Utility method to get an alert provider based on the class specified in its config entry.""" - - module = importlib.import_module('providers.alert.' + config_providers_entry["class"].lower()) - provider_class = getattr(module, config_providers_entry["class"]) - return provider_class(config_providers_entry) - - -def get_solar_conditions_provider_from_config(config_providers_entry): - """Utility method to get a solar conditions provider based on the class specified in its config entry.""" - - module = importlib.import_module('providers.solarconditions.' + config_providers_entry["class"].lower()) - provider_class = getattr(module, config_providers_entry["class"]) - return provider_class(config_providers_entry) - - -def get_static_data_provider_from_config(config_providers_entry): - """Utility method to get a static reference data provider based on the class specified in its config entry.""" - - module = importlib.import_module('providers.staticdata.' + config_providers_entry["class"].lower()) - provider_class = getattr(module, config_providers_entry["class"]) - return provider_class(config_providers_entry) - - -def get_sig_ref_data_provider_from_config(config_providers_entry): - """Utility method to get a SIG reference data provider based on the class specified in its config entry.""" - - module = importlib.import_module('providers.sigrefdata.' + config_providers_entry["class"].lower()) + module = importlib.import_module(package + "." + config_providers_entry["class"].lower()) provider_class = getattr(module, config_providers_entry["class"]) return provider_class(config_providers_entry) diff --git a/core/data_providers.py b/core/data_providers.py new file mode 100644 index 0000000..91ee3cc --- /dev/null +++ b/core/data_providers.py @@ -0,0 +1,72 @@ +from core.config import config, create_provider_from_config + + +class DataProviders: + """Global object for storing data providers.""" + + def __init__(self): + self.spot_providers = [] + self.alert_providers = [] + self.solar_condition_providers = [] + self.static_data_providers = [] + self.sig_ref_data_providers = [] + self.callsign_data_providers = [] + + + def setup(self): + for entry in config["spot-providers"]: + self.spot_providers.append(create_provider_from_config("providers.spot", entry)) + for entry in config["alert-providers"]: + self.alert_providers.append(create_provider_from_config("providers.alert", entry)) + for entry in config.get("solar-condition-providers", []): + self.solar_condition_providers.append(create_provider_from_config("providers.solarconditions", entry)) + for entry in config.get("static-data-providers", []): + self.static_data_providers.append(create_provider_from_config("providers.staticdata", entry)) + for entry in config.get("sig-ref-data-providers", []): + self.sig_ref_data_providers.append(create_provider_from_config("providers.sigrefdata", entry)) + for entry in config.get("callsign-data-providers", []): + self.callsign_data_providers.append(create_provider_from_config("providers.callsigndata", entry)) + + + def start(self): + for p in self.spot_providers: + if p.enabled: + p.start() + for p in self.alert_providers: + if p.enabled: + p.start() + for p in self.solar_condition_providers: + if p.enabled: + p.start() + for p in self.static_data_providers: + if p.enabled: + p.start() + for p in self.sig_ref_data_providers: + if p.enabled: + p.start() + for p in self.callsign_data_providers: + if p.enabled: + p.start() + + def stop(self): + for sp in self.spot_providers: + if sp.enabled: + sp.stop() + for ap in self.alert_providers: + if ap.enabled: + ap.stop() + for scp in self.solar_condition_providers: + if scp.enabled: + scp.stop() + for srdp in self.sig_ref_data_providers: + if srdp.enabled: + srdp.stop() + for sdp in self.static_data_providers: + if sdp.enabled: + sdp.stop() + for cdp in self.callsign_data_providers: + if cdp.enabled: + cdp.stop() + +# Global object +DATA_PROVIDERS = DataProviders() \ No newline at end of file diff --git a/core/data_store.py b/core/data_store.py index 3e8faa0..e2a72e0 100644 --- a/core/data_store.py +++ b/core/data_store.py @@ -28,9 +28,9 @@ class DataStore: self.dxcc_data = None self.dxcc_lookup_by_call_regex = [] self.sigrefs = None - self.status_data = None + self.status_data = {} self._status = None - self.solar_conditions = None + self.solar_conditions = {} self._solar = None # ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be # caches, they can just be straight objects diff --git a/core/status_reporter.py b/core/status_reporter.py index d21802d..f2219e4 100644 --- a/core/status_reporter.py +++ b/core/status_reporter.py @@ -7,24 +7,19 @@ import pytz from core.config import SERVER_OWNER_CALLSIGN from core.constants import SOFTWARE_VERSION +from core.data_providers import DATA_PROVIDERS from core.data_store import DATA_STORE from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge +from server.webserver import WEB_SERVER class StatusReporter: """Provides a timed update of the application's status data.""" - def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers, - static_data_providers, sig_ref_data_providers): + def __init__(self, run_interval): """Constructor""" self._run_interval = run_interval - self._web_server = web_server - self._spot_providers = spot_providers - self._alert_providers = alert_providers - self._solar_condition_providers = solar_condition_providers - self._static_data_providers = static_data_providers - self._sig_ref_data_providers = sig_ref_data_providers self._thread = None self._stop_event = Event() self._startup_time = datetime.now(pytz.UTC) @@ -59,45 +54,50 @@ class StatusReporter: DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values()) DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values()) DATA_STORE.status_data["spot_providers"] = list( - map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, - "last_updated": p.last_update_time.replace( - tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0, - "last_spot": p.last_spot_time.replace( - tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0}, - self._spot_providers)) + map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, + "last_updated": p.last_update_time.replace( + tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0, + "last_spot": p.last_spot_time.replace( + tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0}, + DATA_PROVIDERS.spot_providers)) DATA_STORE.status_data["alert_providers"] = list( - map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, - "last_updated": p.last_update_time.replace( - tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, - self._alert_providers)) + map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, + "last_updated": p.last_update_time.replace( + tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, + DATA_PROVIDERS.alert_providers)) DATA_STORE.status_data["solar_condition_providers"] = list( - map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, - "last_updated": p.last_update_time.replace( - tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, - self._solar_condition_providers)) + map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, + "last_updated": p.last_update_time.replace( + tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, + DATA_PROVIDERS.solar_condition_providers)) DATA_STORE.status_data["static_data_providers"] = list( map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, "last_updated": p.last_update_time.replace( tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, - self._static_data_providers)) + DATA_PROVIDERS.static_data_providers)) DATA_STORE.status_data["sig_ref_data_providers"] = list( map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status, "last_updated": p.last_update_time.replace( tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0, "reference_count": p.reference_count}, - self._sig_ref_data_providers)) - DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"], - "last_api_access": self._web_server.web_server_metrics[ - "last_api_access_time"].replace( - tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[ - "last_api_access_time"] else 0, - "api_access_count": self._web_server.web_server_metrics["api_access_counter"], - "last_page_access": self._web_server.web_server_metrics[ - "last_page_access_time"].replace( - tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[ - "last_page_access_time"] else 0, - "page_access_count": self._web_server.web_server_metrics[ - "page_access_counter"]} + DATA_PROVIDERS.sig_ref_data_providers)) + DATA_STORE.status_data["callsign_data_providers"] = list( + map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, + "last_updated": p.last_update_time.replace( + tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, + DATA_PROVIDERS.callsign_data_providers)) + DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"], + "last_api_access": WEB_SERVER.web_server_metrics[ + "last_api_access_time"].replace( + tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[ + "last_api_access_time"] else 0, + "api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"], + "last_page_access": WEB_SERVER.web_server_metrics[ + "last_page_access_time"].replace( + tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[ + "last_page_access_time"] else 0, + "page_access_count": WEB_SERVER.web_server_metrics[ + "page_access_counter"]} # Update Prometheus metrics memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss) diff --git a/data/callsign.py b/data/callsign.py index 8121ed3..0a7aef7 100644 --- a/data/callsign.py +++ b/data/callsign.py @@ -26,13 +26,13 @@ class Callsign: # the centre of the country they're operating in if no other data is available. longitude : float | None = None # Country in which the callsign indicates they are operating - dx_country: str | None = None + country: str | None = None # Continent in which the callsign indicates they are operating - dx_continent: str | None = None + continent: str | None = None # DXCC ID in which the callsign indicates they are operating - dx_dxcc_id: int | None = None + dxcc_id: int | None = None # CQ zone in which the callsign indicates they are operating - dx_cq_zone: int | None = None + cq_zone: int | None = None # ITU zone in which the callsign indicates they are operating - dx_itu_zone: int | None = None + itu_zone: int | None = None diff --git a/providers/callsigndata/callsign_data_provider.py b/providers/callsigndata/callsign_data_provider.py new file mode 100644 index 0000000..dc84e2c --- /dev/null +++ b/providers/callsigndata/callsign_data_provider.py @@ -0,0 +1,36 @@ +from datetime import datetime + +import pytz + + +class CallsignDataProvider: + """Generic callsign reference data provider class. Subclasses of this set up the various mechanisms via which + Spothole can look up data for callsigns.""" + + def __init__(self, name, provider_config): + """Constructor""" + + self.name = name + self.enabled = provider_config["enabled"] + self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC) + self.status = "Not Started" if self.enabled else "Disabled" + + def start(self): + """Start the provider. This should return immediately after spawning threads to access remote resources, if + needed.""" + + raise NotImplementedError("Subclasses must implement this method") + + def stop(self): + """Stop any threads and prepare for application shutdown""" + + raise NotImplementedError("Subclasses must implement this method") + + def lookup(self, callsign, lookup_credentials): + """Looks up data for the provided callsign. Takes a LookupCredentials object, which provides any credentials + that have been provided by the user for this session (QRZ.com/HamQTH) to allow us to look up using those + services on the user's behalf. (Clublog is looked up using an API key owned by the server and provided in its + config file, so users need not provide their own.) Returns a Callsign object with as much data populated as + possible.""" + + raise NotImplementedError("Subclasses must implement this method") diff --git a/providers/callsigndata/clublogxml.py b/providers/callsigndata/clublogxml.py new file mode 100644 index 0000000..2891b25 --- /dev/null +++ b/providers/callsigndata/clublogxml.py @@ -0,0 +1,68 @@ +import gzip +import logging + +from pyhamtools import LookupLib, Callinfo + +from data.callsign import Callsign +from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider + + +class ClublogXML(FileDownloadCallsignDataProvider): + """Callsign data provider for ClubLog's Country File, which provides basic callsign to DXCC entity mapping.""" + + POLL_INTERVAL_DAYS = 30 + DATA_URL = "https://cdn.clublog.org/cty.php" + CACHE_PATH_ZIPPED = "cache/cty.xml.gz" + CACHE_PATH_UNZIPPED = "cache/cty.xml" + _callinfo = None + + def __init__(self, provider_config): + # API key required for this provider + self._api_key = provider_config.get("api-key", "") + if self._api_key == "": + provider_config["enabled"] = False + logging.warning( + "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.") + + super().__init__("Clublog XML", provider_config, self.DATA_URL + "?api=" + self._api_key, + self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS) + + def _handle_file(self, path): + try: + # The download from Clublog is gzipped so we need to uncompress that and re-save as a separate file that + # the LookupLib can actually use. + with gzip.open(path, "rb") as uncompressed: + file_content = uncompressed.read() + assert isinstance(file_content, bytes) + with open(self.CACHE_PATH_UNZIPPED, "wb") as f: + f.write(file_content) + f.flush() + + # Now load the data + lookuplib = LookupLib(lookuptype="clublogxml", filename=self.CACHE_PATH_UNZIPPED) + self._callinfo = Callinfo(lookuplib) + return True + + except Exception as e: + logging.error("Exception when loading Clublog XML.", e, exc_info=True) + return False + + def lookup(self, callsign, lookup_credentials): + # Lookup credentials are not required for this source. + # Lat/lon will only be centre of country or capital city from this source + ll = self._callinfo.get_lat_long(callsign) + lat = None + lon = None + if ll and "latitude" in ll and "longitude" in ll: + lat = float(ll["latitude"]) + lon = float(ll["longitude"]) + + return Callsign(call=callsign, + home_call=self._callinfo.get_homecall(callsign), + country=self._callinfo.get_country_name(callsign), + dxcc_id=self._callinfo.get_adif_id(callsign), + continent=self._callinfo.get_continent(callsign), + cq_zone=self._callinfo.get_cqz(callsign), + itu_zone=self._callinfo.get_ituz(callsign), + latitude=lat, + longitude=lon) diff --git a/providers/callsigndata/countryfiles.py b/providers/callsigndata/countryfiles.py new file mode 100644 index 0000000..3a6d370 --- /dev/null +++ b/providers/callsigndata/countryfiles.py @@ -0,0 +1,48 @@ +import logging + +from pyhamtools import LookupLib, Callinfo + +from data.callsign import Callsign +from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider + + +class CountryFiles(FileDownloadCallsignDataProvider): + """Callsign data provider for Country-files.com, which provides basic callsign to DXCC entity mapping.""" + + POLL_INTERVAL_DAYS = 30 + DATA_URL = "https://www.country-files.com/cty/cty.plist" + CACHE_PATH = "cache/cty.plist" + _callinfo = None + + def __init__(self, provider_config): + super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS) + + def _handle_file(self, path): + try: + lookuplib = LookupLib(lookuptype="countryfile", filename=path) + self._callinfo = Callinfo(lookuplib) + return True + + except Exception as e: + logging.error("Exception when loading Country Files cty.plist.", e, exc_info=True) + return False + + def lookup(self, callsign, lookup_credentials): + # Lookup credentials are not required for this source. + # Lat/lon will only be centre of country or capital city from this source + ll = self._callinfo.get_lat_long(callsign) + lat = None + lon = None + if ll and "latitude" in ll and "longitude" in ll: + lat = float(ll["latitude"]) + lon = float(ll["longitude"]) + + return Callsign(call=callsign, + home_call=self._callinfo.get_homecall(callsign), + country=self._callinfo.get_country_name(callsign), + dxcc_id=self._callinfo.get_adif_id(callsign), + continent=self._callinfo.get_continent(callsign), + cq_zone=self._callinfo.get_cqz(callsign), + itu_zone=self._callinfo.get_ituz(callsign), + latitude=lat, + longitude=lon) diff --git a/providers/callsigndata/file_download_callsign_data_provider.py b/providers/callsigndata/file_download_callsign_data_provider.py new file mode 100644 index 0000000..0c562ce --- /dev/null +++ b/providers/callsigndata/file_download_callsign_data_provider.py @@ -0,0 +1,85 @@ +import logging +from datetime import datetime +from threading import Thread, Event + +import pytz +from requests import ReadTimeout +from requests.exceptions import ConnectionError, ConnectTimeout + +from core.constants import HTTP_HEADERS +from core.url_data_cache import URLDataCache +from providers.callsigndata.callsign_data_provider import CallsignDataProvider + + +class FileDownloadCallsignDataProvider(CallsignDataProvider): + """Generic static reference data provider class for providers that fetch their data from the web by downloading a + file.""" + + def __init__(self, name, provider_config, url, cache_file_path, poll_interval): + """ Set up the provider, note poll_interval is in *days*.""" + super().__init__(name, provider_config) + self._url = url + self._cache_file_path = cache_file_path + self._poll_interval = poll_interval + self._thread = None + self._stop_event = Event() + self._url_data_cache = URLDataCache("callsigndata_" + name) + + def start(self): + # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between + # subsequent polls, so start() returns immediately and the application can continue starting. + logging.info( + "Set up query of " + self.name + " callsign reference data every " + str(self._poll_interval) + " days.") + self._thread = Thread(target=self._run, daemon=True) + self._thread.start() + + def stop(self): + self._stop_event.set() + + def _run(self): + while True: + self._poll() + if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24): + break + + def _poll(self): + try: + # Request the file. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for + # caching, but just so continual restarts of the software during testing don't hammer the servers. + logging.debug("Downloading " + self.name + " callsign reference data...") + http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS) + # Check response code was good + if http_response.ok: + # Save the data to a local file + with open(self._cache_file_path, "wb") as f: + f.write(http_response.content) + f.flush() + # Pass off to the subclass for processing + ok = self._handle_file(self._cache_file_path) + if ok: + self.status = "OK" + self.last_update_time = datetime.now(pytz.UTC) + logging.info("Updated callsign reference data from " + self.name) + else: + self.status = "Error" + logging.warning(f"Error updating callsign reference data from {self.name}.") + + else: + self.status = "Error" + logging.warning(f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}.") + + except ConnectionError: + self.status = "Error" + logging.warning(f"Connection error when downloading callsign reference data from {self.name}.") + except (ConnectTimeout, ReadTimeout): + self.status = "Error" + logging.warning(f"Timeout when downloading callsign reference data from {self.name}.") + except Exception: + self.status = "Error" + logging.exception("Exception in callsign reference data provider (" + self.name + ")") + self._stop_event.wait(timeout=1) + + def _handle_file(self, path): + """Handle an updated file downloaded from the server. Return true if successful, false otherwise.""" + + raise NotImplementedError("Subclasses must implement this method") diff --git a/server/webserver.py b/server/webserver.py index 131b11d..fbc80eb 100644 --- a/server/webserver.py +++ b/server/webserver.py @@ -43,6 +43,7 @@ class WebServer: "status": "Starting" } + def setup(self): # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately DATA_STORE.spots.add_listener(self._spot_broadcaster.publish) DATA_STORE.alerts.add_listener(self._alert_broadcaster.publish) @@ -145,3 +146,7 @@ def request_log(handler): f'{handler.get_status()} {request.request_time():.2f}ms | ' f'Ref: {referrer} | UA: {user_agent}' ) + + +# Global object +WEB_SERVER = WebServer() \ No newline at end of file diff --git a/spothole.py b/spothole.py index c2cfc74..2c124a9 100644 --- a/spothole.py +++ b/spothole.py @@ -4,49 +4,25 @@ import os import signal import sys -from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_data_provider_from_config, \ - get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config, \ - get_static_data_provider_from_config -from core.constants import SOFTWARE_VERSION -from core.data_store import DATA_STORE from core.call_lookup_helper import lookup_helper +from core.config import SERVER_OWNER_CALLSIGN, LOG_LEVEL +from core.constants import SOFTWARE_VERSION +from core.data_providers import DATA_PROVIDERS +from core.data_store import DATA_STORE from core.status_reporter import StatusReporter -from server.webserver import WebServer +from server.webserver import WEB_SERVER # Globals -web_server = None -spot_providers = [] -alert_providers = [] -solar_condition_providers = [] -static_data_providers = [] -sig_ref_data_providers = [] -cleanup_timer = None run = True - def shutdown(_signum=None, _frame=None): """Shutdown function""" global run logging.info("Stopping program...") - if web_server: - web_server.stop() - for sp in spot_providers: - if sp.enabled: - sp.stop() - for ap in alert_providers: - if ap.enabled: - ap.stop() - for scp in solar_condition_providers: - if scp.enabled: - scp.stop() - for srdp in sig_ref_data_providers: - if srdp.enabled: - srdp.stop() - for srdp in static_data_providers: - if srdp.enabled: - srdp.stop() + WEB_SERVER.stop() + DATA_PROVIDERS.stop() DATA_STORE.close() os._exit(0) @@ -76,54 +52,20 @@ if __name__ == '__main__': # Set up lookup helper lookup_helper.start() - # Set up web server - web_server = WebServer() + # Set up and start data providers + DATA_PROVIDERS.setup() + DATA_PROVIDERS.start() - # Fetch, set up and start spot providers - for entry in config["spot-providers"]: - spot_providers.append(get_spot_provider_from_config(entry)) - for p in spot_providers: - if p.enabled: - p.start() - - # Fetch, set up and start alert providers - for entry in config["alert-providers"]: - alert_providers.append(get_alert_provider_from_config(entry)) - for p in alert_providers: - if p.enabled: - p.start() - - # Fetch, set up and start solar conditions providers - for entry in config.get("solar-condition-providers", []): - solar_condition_providers.append(get_solar_conditions_provider_from_config(entry)) - for p in solar_condition_providers: - if p.enabled: - p.start() - - # Fetch, set up and start static reference data providers - for entry in config.get("static-data-providers", []): - static_data_providers.append(get_static_data_provider_from_config(entry)) - for p in static_data_providers: - if p.enabled: - p.start() - - # Fetch, set up and start SIG reference data providers - for entry in config.get("sig-ref-data-providers", []): - sig_ref_data_providers.append(get_sig_ref_data_provider_from_config(entry)) - for p in sig_ref_data_providers: - if p.enabled: - p.start() - - # Set up status reporter - status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers, - alert_providers=alert_providers, static_data_providers=static_data_providers, - sig_ref_data_providers=sig_ref_data_providers, - solar_condition_providers=solar_condition_providers, run_interval=5) + # Set up and start status reporter + status_reporter = StatusReporter(run_interval=5) status_reporter.start() + # Set up the web server + WEB_SERVER.setup() + logging.info("Startup complete.") # Run the web server. This is the blocking call that keeps the application running in the main thread, so this must # be the last thing we do. web_server.stop() triggers an await condition in the web server which finishes the main # thread. - web_server.start() + WEB_SERVER.start() diff --git a/static/apidocs/openapi.yml b/static/apidocs/openapi.yml index f5ecf4f..cc3db7e 100644 --- a/static/apidocs/openapi.yml +++ b/static/apidocs/openapi.yml @@ -23,7 +23,7 @@ info: * Added `comment_names` to SIGs in the `/options`, to reflect how they might be referred to in spot comments where it differs from their `name`. * Added `propagation_mode` field to spots - * Added `sig_ref_data_providers` and `static_data_providers` to status and removed `cleanup` + * Added `sig_ref_data_providers`, `static_data_providers` and `callsign_data_providers` to status and removed `cleanup` ### 1.3 @@ -1769,6 +1769,28 @@ components: description: The number of references fetched using this provider. example: 1234 + CallsignDataProviderStatus: + type: object + properties: + sig_name: + type: string + description: The name of the provider. + example: Country Files + enabled: + type: boolean + description: Whether the provider is enabled or not. + example: true + status: + type: string + description: The status of the provider. + example: OK + last_updated: + type: number + description: > + The last time at which this provider received data, UTC seconds since UNIX epoch. If this + is zero, the provider has never updated. + example: 1759579508 + SpotList: type: array items: @@ -1876,6 +1898,11 @@ components: description: An array of all the SIG reference data providers. items: $ref: '#/components/schemas/SIGRefDataProviderStatus' + callsign_data_providers: + type: array + description: An array of all the callsign data providers. + items: + $ref: '#/components/schemas/CallsignDataProviderStatus' Options: type: object diff --git a/static/js/status.js b/static/js/status.js index 9ba0f78..237f638 100644 --- a/static/js/status.js +++ b/static/js/status.js @@ -58,6 +58,15 @@ function loadStatus() {
References: ${(p["enabled"] && p["reference_count"] > 0) ? p["reference_count"] : "N/A"}
`); }); + + jsonData["callsign_data_providers"].forEach(p => { + $("#callsign-data-providers-status-container").append(` +
+
${p["name"]}
+
Status: ${p["status"]}
+
Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}
+
`); + }); }); } diff --git a/templates/status.html b/templates/status.html index 2fbee72..f396a9d 100644 --- a/templates/status.html +++ b/templates/status.html @@ -72,6 +72,15 @@ +
+
+ Callsign Data Providers +
+
+ +
+
+