w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 10:31:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 10 #118

Browse Source
This commit is contained in:
Ian Renton
2026-08-02 10:31:31 +01:00
parent 2157bf114e
commit 11a236e668
15 changed files with 452 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/config.py
+5 -35
View File
@@ -36,41 +36,11 @@ if ALLOW_SPOTTING:
WEB_UI_OPTIONS["spot-providers-enabled-by-default"].append("API")
def get_spot_provider_from_config(config_providers_entry):
"""Utility method to get a spot provider based on the class specified in its config entry."""
def create_provider_from_config(package, config_providers_entry):
"""Utility method to get a provider based on the class specified in its config entry. You must also provide the
package to look for it in, as there are several types of provider. e.g. package "providers.spot", where the config
entry is for a POTA spot provider."""
module = importlib.import_module('providers.spot.' + config_providers_entry["class"].lower())
provider_class = getattr(module, config_providers_entry["class"])
return provider_class(config_providers_entry)
def get_alert_provider_from_config(config_providers_entry):
"""Utility method to get an alert provider based on the class specified in its config entry."""
module = importlib.import_module('providers.alert.' + config_providers_entry["class"].lower())
provider_class = getattr(module, config_providers_entry["class"])
return provider_class(config_providers_entry)
def get_solar_conditions_provider_from_config(config_providers_entry):
"""Utility method to get a solar conditions provider based on the class specified in its config entry."""
module = importlib.import_module('providers.solarconditions.' + config_providers_entry["class"].lower())
provider_class = getattr(module, config_providers_entry["class"])
return provider_class(config_providers_entry)
def get_static_data_provider_from_config(config_providers_entry):
"""Utility method to get a static reference data provider based on the class specified in its config entry."""
module = importlib.import_module('providers.staticdata.' + config_providers_entry["class"].lower())
provider_class = getattr(module, config_providers_entry["class"])
return provider_class(config_providers_entry)
def get_sig_ref_data_provider_from_config(config_providers_entry):
"""Utility method to get a SIG reference data provider based on the class specified in its config entry."""
module = importlib.import_module('providers.sigrefdata.' + config_providers_entry["class"].lower())
module = importlib.import_module(package + "." + config_providers_entry["class"].lower())
provider_class = getattr(module, config_providers_entry["class"])
return provider_class(config_providers_entry)
core/data_providers.py
+72
View File
@@ -0,0 +1,72 @@
from core.config import config, create_provider_from_config
class DataProviders:
"""Global object for storing data providers."""
def __init__(self):
self.spot_providers = []
self.alert_providers = []
self.solar_condition_providers = []
self.static_data_providers = []
self.sig_ref_data_providers = []
self.callsign_data_providers = []
def setup(self):
for entry in config["spot-providers"]:
self.spot_providers.append(create_provider_from_config("providers.spot", entry))
for entry in config["alert-providers"]:
self.alert_providers.append(create_provider_from_config("providers.alert", entry))
for entry in config.get("solar-condition-providers", []):
self.solar_condition_providers.append(create_provider_from_config("providers.solarconditions", entry))
for entry in config.get("static-data-providers", []):
self.static_data_providers.append(create_provider_from_config("providers.staticdata", entry))
for entry in config.get("sig-ref-data-providers", []):
self.sig_ref_data_providers.append(create_provider_from_config("providers.sigrefdata", entry))
for entry in config.get("callsign-data-providers", []):
self.callsign_data_providers.append(create_provider_from_config("providers.callsigndata", entry))
def start(self):
for p in self.spot_providers:
if p.enabled:
p.start()
for p in self.alert_providers:
if p.enabled:
p.start()
for p in self.solar_condition_providers:
if p.enabled:
p.start()
for p in self.static_data_providers:
if p.enabled:
p.start()
for p in self.sig_ref_data_providers:
if p.enabled:
p.start()
for p in self.callsign_data_providers:
if p.enabled:
p.start()
def stop(self):
for sp in self.spot_providers:
if sp.enabled:
sp.stop()
for ap in self.alert_providers:
if ap.enabled:
ap.stop()
for scp in self.solar_condition_providers:
if scp.enabled:
scp.stop()
for srdp in self.sig_ref_data_providers:
if srdp.enabled:
srdp.stop()
for sdp in self.static_data_providers:
if sdp.enabled:
sdp.stop()
for cdp in self.callsign_data_providers:
if cdp.enabled:
cdp.stop()
# Global object
DATA_PROVIDERS = DataProviders()
core/data_store.py
+2 -2
View File
@@ -28,9 +28,9 @@ class DataStore:
self.dxcc_data = None
self.dxcc_lookup_by_call_regex = []
self.sigrefs = None
self.status_data = None
self.status_data = {}
self._status = None
self.solar_conditions = None
self.solar_conditions = {}
self._solar = None
# ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
# caches, they can just be straight objects
core/status_reporter.py
+36 -36
View File
@@ -7,24 +7,19 @@ import pytz
from core.config import SERVER_OWNER_CALLSIGN
from core.constants import SOFTWARE_VERSION
from core.data_providers import DATA_PROVIDERS
from core.data_store import DATA_STORE
from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge
from server.webserver import WEB_SERVER
class StatusReporter:
"""Provides a timed update of the application's status data."""
def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers,
static_data_providers, sig_ref_data_providers):
def __init__(self, run_interval):
"""Constructor"""
self._run_interval = run_interval
self._web_server = web_server
self._spot_providers = spot_providers
self._alert_providers = alert_providers
self._solar_condition_providers = solar_condition_providers
self._static_data_providers = static_data_providers
self._sig_ref_data_providers = sig_ref_data_providers
self._thread = None
self._stop_event = Event()
self._startup_time = datetime.now(pytz.UTC)
@@ -59,45 +54,50 @@ class StatusReporter:
DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
DATA_STORE.status_data["spot_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
"last_spot": p.last_spot_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
self._spot_providers))
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
"last_spot": p.last_spot_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
DATA_PROVIDERS.spot_providers))
DATA_STORE.status_data["alert_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
self._alert_providers))
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
DATA_PROVIDERS.alert_providers))
DATA_STORE.status_data["solar_condition_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
self._solar_condition_providers))
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
DATA_PROVIDERS.solar_condition_providers))
DATA_STORE.status_data["static_data_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
self._static_data_providers))
DATA_PROVIDERS.static_data_providers))
DATA_STORE.status_data["sig_ref_data_providers"] = list(
map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
"reference_count": p.reference_count},
self._sig_ref_data_providers))
DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
"last_api_access": self._web_server.web_server_metrics[
"last_api_access_time"].replace(
tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
"last_api_access_time"] else 0,
"api_access_count": self._web_server.web_server_metrics["api_access_counter"],
"last_page_access": self._web_server.web_server_metrics[
"last_page_access_time"].replace(
tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
"last_page_access_time"] else 0,
"page_access_count": self._web_server.web_server_metrics[
"page_access_counter"]}
DATA_PROVIDERS.sig_ref_data_providers))
DATA_STORE.status_data["callsign_data_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
DATA_PROVIDERS.callsign_data_providers))
DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"],
"last_api_access": WEB_SERVER.web_server_metrics[
"last_api_access_time"].replace(
tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[
"last_api_access_time"] else 0,
"api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"],
"last_page_access": WEB_SERVER.web_server_metrics[
"last_page_access_time"].replace(
tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[
"last_page_access_time"] else 0,
"page_access_count": WEB_SERVER.web_server_metrics[
"page_access_counter"]}
# Update Prometheus metrics
memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)