diff --git a/config-example.yml b/config-example.yml index 3a63665..81a56c5 100644 --- a/config-example.yml +++ b/config-example.yml @@ -116,21 +116,30 @@ spot-providers: - class: "XOTA" # xOTA sources have a "name" property so you can define what programme it is for, since xOTA is generic software # for running "something-on-the-air" programmes. - name: "39C3 TOTA" + name: "C3 TOTA" enabled: false url: "wss://39c3.totawatch.de/api/spot/live" - # Fixed SIG for all spots from a provider & location CSV are currently only a feature for the "XOTA" provider, - # the software found at https://github.com/nischu/xOTA/. This is because this is a generic backend for xOTA - # programmes and so different URLs provide different programmes. + # For the "XOTA" provider, a SIG must be set menually here because xOTA is a generic backend for xOTA + # programmes and so different URLs potentially provide different programmes. sig: "TOTA" - locations-csv: "datafiles/39c3-tota.csv" + # For Toilets on the Air, we prefix the SIG references (T-01 etc) with some characters that define the conference: + # C3, EH or HOPE - so we can look up the correct locations in our database, because each conference starts from T-01 + # but refers to a toilet in a different building (or continent!) + sig-ref-prefix: "C3" - class: "XOTA" name: "EH23 TOTA" enabled: false url: "wss://eh23.totawatch.de/api/spot/live" sig: "TOTA" - locations-csv: "datafiles/eh23-tota.csv" + sig-ref-prefix: "EH" + + - class: "XOTA" + name: "HOPE26 TOTA" + enabled: false + url: "wss://hope-26.totawatch.de/api/spot/live" + sig: "TOTA" + sig-ref-prefix: "HOPE" # Alert providers to use. Same setup as the spot providers list above. @@ -157,6 +166,30 @@ alert-providers: enabled: true +# SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF +# park IDs to their name and location. +sig-ref-data-providers: + - class: "WWFF" + enabled: true + + - class: "WOTA" + enabled: true + + - class: "SIOTA" + enabled: true + + - class: "ZLOTA" + enabled: true + + - class: "LLOTA" + enabled: true + + - class: "DME" + enabled: true + + - class: "TOTA" + enabled: true + # Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band # conditions, etc.) and make it available via the /api/v1/solar endpoint. solar-condition-providers: diff --git a/core/config.py b/core/config.py index 4cb43c8..b3f6fe9 100644 --- a/core/config.py +++ b/core/config.py @@ -1,3 +1,4 @@ +import importlib import logging import os @@ -33,3 +34,35 @@ WEB_UI_OPTIONS["spot-providers-enabled-by-default"] = [p["name"] for p in config # one of our proviers. We set that to also be enabled by default. if ALLOW_SPOTTING: WEB_UI_OPTIONS["spot-providers-enabled-by-default"].append("API") + + +def get_spot_provider_from_config(config_providers_entry): + """Utility method to get a spot provider based on the class specified in its config entry.""" + + module = importlib.import_module('spotproviders.' + config_providers_entry["class"].lower()) + provider_class = getattr(module, config_providers_entry["class"]) + return provider_class(config_providers_entry) + + +def get_alert_provider_from_config(config_providers_entry): + """Utility method to get an alert provider based on the class specified in its config entry.""" + + module = importlib.import_module('alertproviders.' + config_providers_entry["class"].lower()) + provider_class = getattr(module, config_providers_entry["class"]) + return provider_class(config_providers_entry) + + +def get_solar_conditions_provider_from_config(config_providers_entry): + """Utility method to get a solar conditions provider based on the class specified in its config entry.""" + + module = importlib.import_module('solarconditionsproviders.' + config_providers_entry["class"].lower()) + provider_class = getattr(module, config_providers_entry["class"]) + return provider_class(config_providers_entry) + + +def get_sig_ref_data_provider_from_config(config_providers_entry): + """Utility method to get a SIG reference data provider based on the class specified in its config entry.""" + + module = importlib.import_module('sigrefdataproviders.' + config_providers_entry["class"].lower()) + provider_class = getattr(module, config_providers_entry["class"]) + return provider_class(config_providers_entry) diff --git a/core/data_store.py b/core/data_store.py index b78affd..7e23e27 100644 --- a/core/data_store.py +++ b/core/data_store.py @@ -12,57 +12,52 @@ class DataStore: lookup data using different caching strategies for each.""" def __init__(self): - cache_dir = "./cache" + self.CACHE_DIR = "./cache" self.MAX_SPOT_COUNT = 10000 self.MAX_ALERT_COUNT = 10000 self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300 self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60 - Path(cache_dir).mkdir(parents=True, exist_ok=True) - - # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval - self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE, - snapshot_dir=cache_dir + "/spots", - snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) - self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE, - snapshot_dir=cache_dir + "/alerts", - snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) + def setup(self): + Path(self.CACHE_DIR).mkdir(parents=True, exist_ok=True) # Standard disk cache for solar data and status data, but each cache contains only a single object which we # expose to the wider application - self.solar = diskcache.Cache(cache_dir + "/solar") + self.solar = diskcache.Cache(self.CACHE_DIR + "/solar") if "solar_conditions" not in self.solar: self.solar.add("solar_conditions", SolarConditions()) self.solar_conditions = self.solar.get("solar_conditions") - self.status = diskcache.Cache(cache_dir + "/status") + self.status = diskcache.Cache(self.CACHE_DIR + "/status") if "status_data" not in self.status: self.status.add("status_data", {}) self.status_data = self.status.get("status_data") - # Standard disk caches for SIG ref data. Separate provider threads will repopulate these on a regular basis - # but there's no need for a TTL since old data is better than no data. - self.sigrefs_wwff = diskcache.Cache(cache_dir + "/sigrefs/wwff") - self.sigrefs_siota = diskcache.Cache(cache_dir + "/sigrefs/siota") - self.sigrefs_wota = diskcache.Cache(cache_dir + "/sigrefs/wota") - self.sigrefs_zlota = diskcache.Cache(cache_dir + "/sigrefs/zlota") - self.sigrefs_llota = diskcache.Cache(cache_dir + "/sigrefs/llota") - self.sigrefs_dme = diskcache.Cache(cache_dir + "/sigrefs/dme") + # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis + # but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG + # name and then reference ID, with the final value being a SIGRef object. + self.sigrefs = diskcache.Cache(self.CACHE_DIR + "/sigrefs") # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup. # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign # we've seen, so we rely on them timing out and this triggering another lookup. - self.callsigns = diskcache.Cache(cache_dir + "/callsigns") + self.callsigns = diskcache.Cache(self.CACHE_DIR + "/callsigns") + + # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We + # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache + # before the spots and alerts are live in the system. + self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE, + snapshot_dir=self.CACHE_DIR + "/spots", + snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) + self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE, + snapshot_dir=self.CACHE_DIR + "/alerts", + snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) def close(self): self.spots.close() self.alerts.close() self.solar.close() self.status.close() - self.sigrefs_wwff.close() - self.sigrefs_siota.close() - self.sigrefs_wota.close() - self.sigrefs_zlota.close() - self.sigrefs_llota.close() + self.sigrefs.close() self.callsigns.close() # Global object diff --git a/core/sig_utils.py b/core/sig_utils.py index 82f7809..2c16c9d 100644 --- a/core/sig_utils.py +++ b/core/sig_utils.py @@ -1,4 +1,3 @@ -import csv import logging from pyhamtools.locator import latlong_to_locator, locator_to_latlong @@ -9,11 +8,6 @@ from core.data_store import DATA_STORE from core.geo_utils import wab_wai_square_to_lat_lon from core.url_data_cache import URL_DATA_CACHE -# Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data -# file in Spothole and load it on startup. -with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f: - for row in csv.DictReader(_f, delimiter=";"): - DATA_STORE.sigrefs_dme.add(row["COD_INE"][:5], row) def get_ref_regex_for_sig(sig): """Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned.""" @@ -35,12 +29,15 @@ def get_sig_name_from_comment_name(sig): def populate_sig_ref_info(sig_ref): - """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object which - must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. + """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object + which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes + use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which + SIG we are getting data for. Note there is currently no support for KRMNPA location lookup, see issue #61.""" - if sig_ref.sig is None or sig_ref.id is None: - logging.warning("Failed to look up sig_ref info, sig or id were not set.") + if sig_ref.sig is None or sig_ref.id is None or sig_ref.id == "": + logging.debug("Failed to look up sig_ref info, sig or id were not set.") + return sig_ref sig = sig_ref.sig or "" ref_id = sig_ref.id @@ -123,95 +120,36 @@ def populate_sig_ref_info(sig_ref): logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) elif sig.upper() == "WWFF": - response = URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv", - headers=HTTP_HEADERS) - if response.ok: - if not bool(DATA_STORE.sigrefs_wwff) or not response.from_cache: - # New data from WWFF, update our internal map - for row in csv.DictReader(response.content.decode().splitlines()): - DATA_STORE.sigrefs_wwff.add(row["reference"], row) - row = DATA_STORE.sigrefs_wwff.get(ref_id) - if row: - sig_ref.name = row["name"] if "name" in row else None - sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id - sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None - sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None - sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[ - "longitude"] != "-" else None - elif not response.from_cache: - logging.warning("WWFF database did not contain data for ref %s", ref_id) - elif not response.from_cache: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + lookup_data = DATA_STORE.sigrefs["WWFF"][ref_id] if ref_id in DATA_STORE.sigrefs["WWFF"] else None + if lookup_data: + # Copy new sig ref data into existing object + sig_ref.__dict__.update(lookup_data.__dict__) + else: + logging.warning("WWFF database did not contain data for ref %s", ref_id) elif sig.upper() == "SIOTA": - response = URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv", - headers=HTTP_HEADERS) - if response.ok: - if not bool(DATA_STORE.sigrefs_siota) or not response.from_cache: - # New data from SIOTA, update our internal map - for row in csv.DictReader(response.content.decode().splitlines()): - DATA_STORE.sigrefs_siota.add(row["SILO_CODE"], row) - row = DATA_STORE.sigrefs_siota.get(ref_id) - if row: - sig_ref.name = row["NAME"] if "NAME" in row else None - sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None - sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None - sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None - elif not response.from_cache: - logging.warning("SIOTA database did not contain data for ref %s", ref_id) - elif not response.from_cache: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + lookup_data = DATA_STORE.sigrefs["SIOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["SIOTA"] else None + if lookup_data: + # Copy new sig ref data into existing object + sig_ref.__dict__.update(lookup_data.__dict__) + else: + logging.warning("SIOTA database did not contain data for ref %s", ref_id) elif sig.upper() == "WOTA": - response = URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json", - headers=HTTP_HEADERS) - if response.ok: - data = response.json() - if data: - if not bool(DATA_STORE.sigrefs_wota) or not response.from_cache: - # New data from WOTA, update our internal map - for feature in data.get("features", []): - DATA_STORE.sigrefs_wota.add(feature["properties"]["wotaId"], feature) - feature = DATA_STORE.sigrefs_wota.get(ref_id) - if feature: - sig_ref.name = feature["properties"]["title"] - # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be - # added to them - sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id - if ref_id.upper().startswith("LDO-"): - number = int(ref_id.upper().replace("LDO-", "")) - sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214) - sig_ref.grid = feature["properties"]["qthLocator"] - sig_ref.latitude = feature["geometry"]["coordinates"][1] - sig_ref.longitude = feature["geometry"]["coordinates"][0] - elif not response.from_cache: - logging.warning("Malformed response looking up %s ref %s", sig, ref_id) - elif not response.from_cache: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + lookup_data = DATA_STORE.sigrefs["WOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["WOTA"] else None + if lookup_data: + # Copy new sig ref data into existing object + sig_ref.__dict__.update(lookup_data.__dict__) + else: + logging.warning("WOTA database did not contain data for ref %s", ref_id) elif sig.upper() == "ZLOTA": - response = URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS) - if response.ok: - data = response.json() - if isinstance(data, list): - if not bool(DATA_STORE.sigrefs_zlota) or not response.from_cache: - # New data from ZLOTA, update our internal map - for ref in data: - DATA_STORE.sigrefs_zlota.add(ref["code"], ref) - ref = DATA_STORE.sigrefs_zlota.get(ref_id) - if ref: - sig_ref.name = ref["name"] - sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_") - try: - sig_ref.grid = latlong_to_locator(ref["y"], ref["x"], 6) - except: - logging.debug("Invalid lat/lon received for reference") - sig_ref.latitude = ref["y"] - sig_ref.longitude = ref["x"] - elif not response.from_cache: - logging.warning("Malformed response looking up %s ref %s", sig, ref_id) - elif not response.from_cache: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + lookup_data = DATA_STORE.sigrefs["ZLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["ZLOTA"] else None + if lookup_data: + # Copy new sig ref data into existing object + sig_ref.__dict__.update(lookup_data.__dict__) + else: + logging.warning("ZLOTA database did not contain data for ref %s", ref_id) elif sig.upper() == "BOTA": if not sig_ref.name: @@ -219,27 +157,29 @@ def populate_sig_ref_info(sig_ref): sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-") elif sig.upper() == "LLOTA": - response = URL_DATA_CACHE.get("https://llota.app/api/public/references", - headers=HTTP_HEADERS) - if response.ok: - data = response.json() - if isinstance(data, list): - if not bool(DATA_STORE.sigrefs_llota) or not response.from_cache: - # New data from LLOTA, update our internal map - for ref in data: - DATA_STORE.sigrefs_llota.add(ref["reference_code"], ref) - ref = DATA_STORE.sigrefs_llota.get(ref_id) - if ref: - sig_ref.name = str(ref["name"]) - sig_ref.url = "https://llota.app/list/ref/" + ref_id - sig_ref.grid = str(ref["grid_locator"]) - ll = locator_to_latlong(sig_ref.grid) - sig_ref.latitude = ll[0] - sig_ref.longitude = ll[1] - elif not response.from_cache: - logging.warning("Malformed response looking up %s ref %s", sig, ref_id) - elif not response.from_cache: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + lookup_data = DATA_STORE.sigrefs["LLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["LLOTA"] else None + if lookup_data: + # Copy new sig ref data into existing object + sig_ref.__dict__.update(lookup_data.__dict__) + else: + logging.warning("LLOTA database did not contain data for ref %s", ref_id) + + elif sig.upper() == "DME": + # Zero-pad to 5 digits to match our source data + lookup_data = DATA_STORE.sigrefs["DME"][ref_id.zfill(5)] if ref_id.zfill(5) in DATA_STORE.sigrefs["DME"] else None + if lookup_data: + # Copy new sig ref data into existing object + sig_ref.__dict__.update(lookup_data.__dict__) + else: + logging.warning("DME database did not contain data for ref %s", ref_id) + + elif sig.upper() == "TOTA": + lookup_data = DATA_STORE.sigrefs["TOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["TOTA"] else None + if lookup_data: + # Copy new sig ref data into existing object + sig_ref.__dict__.update(lookup_data.__dict__) + else: + logging.warning("TOTA database did not contain data for ref %s", ref_id) elif sig.upper() == "WWTOTA": if not sig_ref.name: @@ -268,23 +208,6 @@ def populate_sig_ref_info(sig_ref): except: logging.warning("Invalid lat/lon received for WAB/WAI reference") - elif sig.upper() == "DME": - # Zero-pad to 5 digits to match our source data - row = DATA_STORE.sigrefs_dme.get(ref_id.zfill(5)) - if row: - sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"] - sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get( - "LATITUD_ETRS89_REGCAN95") else None - sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get( - "LONGITUD_ETRS89_REGCAN95") else None - if sig_ref.latitude and sig_ref.longitude: - try: - sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6) - except Exception: - logging.warning("Invalid lat/lon received for DME reference") - else: - logging.warning("DME database did not contain data for ref %s", ref_id) - except ConnectionError: logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id) except (ConnectTimeout, ReadTimeout): diff --git a/core/status_reporter.py b/core/status_reporter.py index 5da9421..82ada84 100644 --- a/core/status_reporter.py +++ b/core/status_reporter.py @@ -14,7 +14,8 @@ from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alert class StatusReporter: """Provides a timed update of the application's status data.""" - def __init__(self, run_interval, web_server,spot_providers, alert_providers, solar_condition_providers): + def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers, + sig_ref_data_providers): """Constructor""" self._run_interval = run_interval @@ -22,6 +23,7 @@ class StatusReporter: self._spot_providers = spot_providers self._alert_providers = alert_providers self._solar_condition_providers = solar_condition_providers + self._sig_ref_data_providers = sig_ref_data_providers self._thread = None self._stop_event = Event() self._startup_time = datetime.now(pytz.UTC) @@ -72,6 +74,11 @@ class StatusReporter: "last_updated": p.last_update_time.replace( tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, self._solar_condition_providers)) + DATA_STORE.status_data["sig_ref_data_providers"] = list( + map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status, + "last_updated": p.last_update_time.replace( + tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, + self._sig_ref_data_providers)) DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"], "last_api_access": self._web_server.web_server_metrics[ "last_api_access_time"].replace( diff --git a/datafiles/39c3-tota.csv b/datafiles/39c3-tota.csv deleted file mode 100644 index 15f6ac6..0000000 --- a/datafiles/39c3-tota.csv +++ /dev/null @@ -1,18 +0,0 @@ -ref,lat,lon -T-01,53.56278090617755,9.984341869295505 -T-02,53.562383404176416,9.98551893027115 -T-03,53.56170184391514,9.985416035619778 -T-04,53.562026534393176,9.986372919078974 -T-11,53.56284641242506,9.98475590239655 -T-12,53.562431705517035,9.98551675702443 -T-13,53.56223704898424,9.985774520335664 -T-14,53.5617893512591,9.986344302837976 -T-21,53.56284641242506,9.98475590239655 -T-22,53.56245816412497,9.985456089490567 -T-23,53.56199560857136,9.985636761412673 -T-24,53.5617893512591,9.986344302837976 -T-31,53.56247470064887,9.985611427551902 -T-32,53.5617893512591,9.986344302837976 -T-41,53.56245039134992,9.985486136112701 -T-91,53.56147934973529,9.984626806439744 -T-92,53.561396810300735,9.987553052152899 \ No newline at end of file diff --git a/datafiles/eh23-tota.csv b/datafiles/eh23-tota.csv deleted file mode 100644 index 8a50df8..0000000 --- a/datafiles/eh23-tota.csv +++ /dev/null @@ -1,13 +0,0 @@ -ref,lat,lon -T-01,50.3636495,7.5584857 -T-02,50.3636495,7.5584857 -T-03,50.3636495,7.5584857 -T-11,50.3636495,7.5584857 -T-13,50.3636495,7.5584857 -T-14,50.3636495,7.5584857 -T-21,50.3636495,7.5584857 -T-31,50.3636495,7.5584857 -T-33,50.3636495,7.5584857 -T-34,50.3636495,7.5584857 -T-41,50.3636495,7.5584857 -T-51,50.3636495,7.5584857 \ No newline at end of file diff --git a/datafiles/tota.csv b/datafiles/tota.csv new file mode 100644 index 0000000..49e85db --- /dev/null +++ b/datafiles/tota.csv @@ -0,0 +1,30 @@ +ref,lat,lon +C3 T-01,53.56278090617755,9.984341869295505 +C3 T-02,53.562383404176416,9.98551893027115 +C3 T-03,53.56170184391514,9.985416035619778 +C3 T-04,53.562026534393176,9.986372919078974 +C3 T-11,53.56284641242506,9.98475590239655 +C3 T-12,53.562431705517035,9.98551675702443 +C3 T-13,53.56223704898424,9.985774520335664 +C3 T-14,53.5617893512591,9.986344302837976 +C3 T-21,53.56284641242506,9.98475590239655 +C3 T-22,53.56245816412497,9.985456089490567 +C3 T-23,53.56199560857136,9.985636761412673 +C3 T-24,53.5617893512591,9.986344302837976 +C3 T-31,53.56247470064887,9.985611427551902 +C3 T-32,53.5617893512591,9.986344302837976 +C3 T-41,53.56245039134992,9.985486136112701 +C3 T-91,53.56147934973529,9.984626806439744 +C3 T-92,53.561396810300735,9.987553052152899 +EH T-01,50.3636495,7.5584857 +EH T-02,50.3636495,7.5584857 +EH T-03,50.3636495,7.5584857 +EH T-11,50.3636495,7.5584857 +EH T-13,50.3636495,7.5584857 +EH T-14,50.3636495,7.5584857 +EH T-21,50.3636495,7.5584857 +EH T-31,50.3636495,7.5584857 +EH T-33,50.3636495,7.5584857 +EH T-34,50.3636495,7.5584857 +EH T-41,50.3636495,7.5584857 +EH T-51,50.3636495,7.5584857 \ No newline at end of file diff --git a/sigrefdataproviders/dme.py b/sigrefdataproviders/dme.py new file mode 100644 index 0000000..b2a8e47 --- /dev/null +++ b/sigrefdataproviders/dme.py @@ -0,0 +1,34 @@ +import csv + +from pyhamtools.locator import latlong_to_locator + +from data.sig_ref import SIGRef +from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider + + +class DME(LocalFileSIGRefDataProvider): + """SIG ref data provider for Diploma Municipios de Espana""" + + SIG = "DME" + PATH = "datafiles/MUNICIPIOS.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.PATH) + + def _file_to_data(self, path): + new_data = {} + with open(path, encoding="latin-1") as _f: + for row in csv.DictReader(_f, delimiter=";"): + ref_id = row["COD_INE"][:5] + latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get( + "LATITUD_ETRS89_REGCAN95") else None + longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get( + "LONGITUD_ETRS89_REGCAN95") else None + + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, + name=row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"], + latitude=latitude, + longitude=longitude) + if latitude and longitude: + new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6) + return new_data diff --git a/sigrefdataproviders/file_download_sig_ref_data_provider.py b/sigrefdataproviders/file_download_sig_ref_data_provider.py new file mode 100644 index 0000000..9f46bc1 --- /dev/null +++ b/sigrefdataproviders/file_download_sig_ref_data_provider.py @@ -0,0 +1,75 @@ +import logging +from datetime import datetime +from threading import Thread, Event + +import pytz +import requests +from requests import ReadTimeout +from requests.exceptions import ConnectionError, ConnectTimeout + +from core.constants import HTTP_HEADERS +from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider + + +class FileDownloadSIGRefDataProvider(SIGRefDataProvider): + """Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file.""" + + def __init__(self, sig_name, provider_config, url, poll_interval): + super().__init__(sig_name, provider_config) + self._url = url + self._poll_interval = poll_interval + self._thread = None + self._stop_event = Event() + + def start(self): + # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between + # subsequent polls, so start() returns immediately and the application can continue starting. + logging.info( + "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " seconds.") + self._thread = Thread(target=self._run, daemon=True) + self._thread.start() + + def stop(self): + self._stop_event.set() + + def _run(self): + while True: + self._poll() + if self._stop_event.wait(timeout=self._poll_interval): + break + + def _poll(self): + try: + # Request data from API + logging.debug("Downloading " + self.sig_name + " SIG ref data...") + http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30)) + # Check response code was good + if http_response.ok: + # Pass off to the subclass for processing + new_data = self._http_response_to_data(http_response) + # Submit the new spots for processing. There might not be any spots for the less popular programs. + if new_data: + self._replace_data(new_data) + + self.status = "OK" + self.last_update_time = datetime.now(pytz.UTC) + logging.debug("Received SIG ref data for " + self.sig_name) + else: + self.status = "Error" + logging.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.") + + except ConnectionError: + logging.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.") + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.") + except Exception: + self.status = "Error" + logging.exception("Exception in HTTP SIG Ref Data Provider (" + self.sig_name + ")") + self._stop_event.wait(timeout=1) + + def _http_response_to_data(self, http_response): + """Convert an HTTP response returned by the server into SIG Ref data. The whole response is provided here so the + subclass implementations can check for HTTP status codes if necessary, and handle the response as JSON, CSV, + whatever the remote file actually is.""" + + raise NotImplementedError("Subclasses must implement this method") diff --git a/sigrefdataproviders/llota.py b/sigrefdataproviders/llota.py new file mode 100644 index 0000000..e9cc8ce --- /dev/null +++ b/sigrefdataproviders/llota.py @@ -0,0 +1,32 @@ +from pyhamtools.locator import locator_to_latlong + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class LLOTA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Lagos y Lagunas on the Air""" + + POLL_INTERVAL_SEC = 7 * 24 * 60 * 60 # 7 days + SIG = "LLOTA" + DATA_URL = "https://llota.app/api/public/references" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC) + + def _http_response_to_data(self, http_response): + new_data = {} + data = http_response.json() + if isinstance(data, list): + for ref in data: + ref_id = ref["reference_code"] + grid = str(ref["grid_locator"]) + ll = locator_to_latlong(grid) + + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]), + url="https://llota.app/list/ref/" + ref_id, + grid=grid, + latitude=ll[0], + longitude=ll[1]) + + return new_data diff --git a/sigrefdataproviders/local_file_sig_ref_data_provider.py b/sigrefdataproviders/local_file_sig_ref_data_provider.py new file mode 100644 index 0000000..ba427d3 --- /dev/null +++ b/sigrefdataproviders/local_file_sig_ref_data_provider.py @@ -0,0 +1,36 @@ +import logging +from datetime import datetime + +import pytz + +from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider + + +class LocalFileSIGRefDataProvider(SIGRefDataProvider): + """Generic SIG ref data provider class for providers that fetch their data from a local file on startup.""" + + def __init__(self, sig, provider_config, path): + super().__init__(sig, provider_config) + self._path = path + + def start(self): + logging.info("Loading " + self.sig_name + " SIG ref data from file.") + try: + new_data = self._file_to_data(self._path) + if new_data: + self._replace_data(new_data) + self.status = "OK" + self.last_update_time = datetime.now(pytz.UTC) + else: + logging.info("No new SIG ref data found for " + self.sig_name) + except Exception as e: + self.status = "Error" + logging.exception("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")") + + def stop(self): + pass + + def _file_to_data(self, path): + """Load a file on the given path and turn it into SIG Ref data.""" + + raise NotImplementedError("Subclasses must implement this method") diff --git a/sigrefdataproviders/sig_ref_data_provider.py b/sigrefdataproviders/sig_ref_data_provider.py index cf2e7f8..96e265b 100644 --- a/sigrefdataproviders/sig_ref_data_provider.py +++ b/sigrefdataproviders/sig_ref_data_provider.py @@ -2,19 +2,25 @@ from datetime import datetime import pytz +from core.data_store import DATA_STORE + class SIGRefDataProvider: """Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data.""" - def __init__(self, name, provider_config): + def __init__(self, sig_name, provider_config): """Constructor""" - self.name = name + self.sig_name = sig_name self.enabled = provider_config["enabled"] self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC) self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC) self.status = "Not Started" if self.enabled else "Disabled" + # Create an empty dict to store data if one doesn't already exist + if not sig_name in DATA_STORE.sigrefs: + DATA_STORE.sigrefs[sig_name] = {} + def start(self): """Start the provider. This should return immediately after spawning threads to access the remote resources""" @@ -26,3 +32,9 @@ class SIGRefDataProvider: """Stop any threads and prepare for application shutdown""" raise NotImplementedError("Subclasses must implement this method") + + def _replace_data(self, new_data): + """Replace all data for the named sig with the new data. new_data should be a map of reference ID to SIGRef + objects.""" + + DATA_STORE.sigrefs[self.sig_name] = new_data diff --git a/sigrefdataproviders/siota.py b/sigrefdataproviders/siota.py new file mode 100644 index 0000000..8d60272 --- /dev/null +++ b/sigrefdataproviders/siota.py @@ -0,0 +1,26 @@ +import csv + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class SIOTA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Silos on the Air""" + + POLL_INTERVAL_SEC = 30 * 24 * 60 * 60 # 30 days + SIG = "SIOTA" + DATA_URL = "https://www.silosontheair.com/data/silos.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode().splitlines()): + ref_id = row["SILO_CODE"] + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None, + grid=row["LOCATOR"] if "LOCATOR" in row else None, + latitude=float(row["LAT"]) if "LAT" in row else None, + longitude=float(row["LNG"]) if "LNG" in row else None) + + return new_data diff --git a/sigrefdataproviders/tota.py b/sigrefdataproviders/tota.py new file mode 100644 index 0000000..ad7b546 --- /dev/null +++ b/sigrefdataproviders/tota.py @@ -0,0 +1,24 @@ +import csv + +from data.sig_ref import SIGRef +from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider + + +class TOTA(LocalFileSIGRefDataProvider): + """SIG ref data provider for Toilets on the Air""" + + SIG = "TOTA" + PATH = "datafiles/tota.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.PATH) + + def _file_to_data(self, path): + new_data = {} + f = open(path) + csv_data = f.read() + dr = csv.DictReader(csv_data.splitlines()) + for row in dr: + new_data[row["ref"]] = SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]), + longitude=float(row["lon"])) + return new_data diff --git a/sigrefdataproviders/wota.py b/sigrefdataproviders/wota.py new file mode 100644 index 0000000..4e2b8a8 --- /dev/null +++ b/sigrefdataproviders/wota.py @@ -0,0 +1,31 @@ +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class WOTA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Wainwrights on the Air""" + + POLL_INTERVAL_SEC = 365 * 24 * 60 * 60 # 365 days + SIG = "WOTA" + DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC) + + def _http_response_to_data(self, http_response): + new_data = {} + for feature in http_response.json().get("features", []): + ref_id = feature["properties"]["wotaId"] + # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be + # added to them + url = "https://www.wota.org.uk/MM_" + ref_id + if ref_id.upper().startswith("LDO-"): + number = int(ref_id.upper().replace("LDO-", "")) + url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214) + + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url, + grid=feature["properties"]["qthLocator"], + latitude=feature["geometry"]["coordinates"][1], + longitude=feature["geometry"]["coordinates"][0]) + + return new_data diff --git a/sigrefdataproviders/wwff.py b/sigrefdataproviders/wwff.py new file mode 100644 index 0000000..492bc91 --- /dev/null +++ b/sigrefdataproviders/wwff.py @@ -0,0 +1,30 @@ +import csv + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class WWFF(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Worldwide Flora & Fauna""" + + POLL_INTERVAL_SEC = 30 * 24 * 60 * 60 # 30 days + SIG = "WWFF" + DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode().splitlines()): + ref_id = row["reference"] + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None, + url="https://wwff.co/directory/?showRef=" + ref_id, + grid=row["iaruLocator"] if "iaruLocator" in row and row[ + "iaruLocator"] != "-" else None, + latitude=float(row["latitude"]) if "latitude" in row and row[ + "latitude"] != "" and row["latitude"] != "-" else None, + longitude=float(row["longitude"]) if "longitude" in row and row[ + "longitude"] != "" and row["longitude"] != "-" else None) + + return new_data diff --git a/sigrefdataproviders/zlota.py b/sigrefdataproviders/zlota.py new file mode 100644 index 0000000..84546df --- /dev/null +++ b/sigrefdataproviders/zlota.py @@ -0,0 +1,33 @@ +from pyhamtools.locator import latlong_to_locator + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class ZLOTA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for New Zealand on the Air""" + + POLL_INTERVAL_SEC = 30 * 24 * 60 * 60 # 30 days + SIG = "ZLOTA" + DATA_URL = "https://ontheair.nz/assets/assets.json" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC) + + def _http_response_to_data(self, http_response): + new_data = {} + data = http_response.json() + if isinstance(data, list): + for ref in data: + ref_id = ref["code"] + latitude = ref["y"] + longitude = ref["x"] + + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"], + url="https://ontheair.nz/assets/" + ref_id.replace("/", "_"), + latitude=latitude, + longitude=longitude) + if latitude and longitude: + new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6) + + return new_data diff --git a/spothole.py b/spothole.py index ad9016a..02c0ee5 100644 --- a/spothole.py +++ b/spothole.py @@ -1,11 +1,11 @@ # Main script -import importlib import logging import os import signal import sys -from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL +from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_data_provider_from_config, \ + get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config from core.constants import SOFTWARE_VERSION from core.data_store import DATA_STORE from core.lookup_helper import lookup_helper @@ -17,6 +17,7 @@ web_server = None spot_providers = [] alert_providers = [] solar_condition_providers = [] +sig_ref_data_providers = [] cleanup_timer = None run = True @@ -38,34 +39,13 @@ def shutdown(_signum=None, _frame=None): for scp in solar_condition_providers: if scp.enabled: scp.stop() + for srdp in sig_ref_data_providers: + if srdp.enabled: + srdp.stop() DATA_STORE.close() os._exit(0) -def get_spot_provider_from_config(config_providers_entry): - """Utility method to get a spot provider based on the class specified in its config entry.""" - - module = importlib.import_module('spotproviders.' + config_providers_entry["class"].lower()) - provider_class = getattr(module, config_providers_entry["class"]) - return provider_class(config_providers_entry) - - -def get_alert_provider_from_config(config_providers_entry): - """Utility method to get an alert provider based on the class specified in its config entry.""" - - module = importlib.import_module('alertproviders.' + config_providers_entry["class"].lower()) - provider_class = getattr(module, config_providers_entry["class"]) - return provider_class(config_providers_entry) - - -def get_solar_conditions_provider_from_config(config_providers_entry): - """Utility method to get a solar conditions provider based on the class specified in its config entry.""" - - module = importlib.import_module('solarconditionsproviders.' + config_providers_entry["class"].lower()) - provider_class = getattr(module, config_providers_entry["class"]) - return provider_class(config_providers_entry) - - # Main function if __name__ == '__main__': # Set up logging @@ -85,6 +65,9 @@ if __name__ == '__main__': # Shut down gracefully on SIGINT signal.signal(signal.SIGINT, shutdown) + # Set up data store + DATA_STORE.setup() + # Set up lookup helper lookup_helper.start() @@ -112,9 +95,17 @@ if __name__ == '__main__': if p.enabled: p.start() + # Fetch, set up and start SIG reference data providers + for entry in config.get("sig-ref-data-providers", []): + sig_ref_data_providers.append(get_sig_ref_data_provider_from_config(entry)) + for p in sig_ref_data_providers: + if p.enabled: + p.start() + # Set up status reporter status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers, alert_providers=alert_providers, + sig_ref_data_providers=sig_ref_data_providers, solar_condition_providers=solar_condition_providers, run_interval=5) status_reporter.start() diff --git a/spotproviders/http_spot_provider.py b/spotproviders/http_spot_provider.py index 08eaa4f..b4c4d93 100644 --- a/spotproviders/http_spot_provider.py +++ b/spotproviders/http_spot_provider.py @@ -64,7 +64,7 @@ class HTTPSpotProvider(SpotProvider): logging.warning(f"Timeout when accessing {self.name} spots API.") except Exception: self.status = "Error" - logging.exception("Exception in HTTP JSON Spot Provider (" + self.name + ")") + logging.exception("Exception in HTTP Spot Provider (" + self.name + ")") self._stop_event.wait(timeout=1) def _http_response_to_spots(self, http_response): diff --git a/spotproviders/xota.py b/spotproviders/xota.py index 8513650..14e7526 100644 --- a/spotproviders/xota.py +++ b/spotproviders/xota.py @@ -1,6 +1,4 @@ -import csv import json -import logging from datetime import datetime import pytz @@ -12,10 +10,11 @@ from spotproviders.websocket_spot_provider import WebsocketSpotProvider class XOTA(WebsocketSpotProvider): """Spot provider for servers based on the "xOTA" software at https://github.com/nischu/xOTA/ - The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG and a reference - to a local CSV file with location information. This functionality is implemented for TOTA events, of which there are - several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA and hence which server - supplied the data, which is why the CSV location lookup is here and not in sig_utils.""" + The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG which we can + then use for lookups. This functionality is implemented for TOTA events, of which there are + several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA, which is why we also + provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3 might become + "C3 T-01". This allows us to provide location lookups for TOTA at several conferences.""" LOCATION_DATA = {} SIG = None @@ -23,26 +22,13 @@ class XOTA(WebsocketSpotProvider): def __init__(self, provider_config): name = provider_config["name"] if "name" in provider_config else "xOTA" super().__init__(name, provider_config, provider_config["url"]) - locations_csv = str(provider_config["locations-csv"]) if "locations-csv" in provider_config else None self.SIG = str(provider_config["sig"]) if "sig" in provider_config else None - - # Load location data - if locations_csv: - try: - f = open(locations_csv) - csv_data = f.read() - dr = csv.DictReader(csv_data.splitlines()) - for row in dr: - self.LOCATION_DATA[row["ref"]] = {"lat": row["lat"], "lon": row["lon"]} - except: - logging.exception("Could not look up location data for XOTA source.") + self._sig_ref_prefix = str(provider_config["sig-ref-prefix"]) if "sig-ref-prefix" in provider_config else "" def _ws_message_to_spot(self, b): string = b.decode("utf-8") source_spot = json.loads(string) - ref_id = source_spot["reference"]["title"] - lat = float(self.LOCATION_DATA[ref_id]["lat"]) if ref_id in self.LOCATION_DATA else None - lon = float(self.LOCATION_DATA[ref_id]["lon"]) if ref_id in self.LOCATION_DATA else None + ref_id = self._sig_ref_prefix + " " + source_spot["reference"]["title"] spot = Spot(source=self.name, source_id=source_spot["id"], dx_call=source_spot["stationCallSign"].upper(), @@ -50,10 +36,7 @@ class XOTA(WebsocketSpotProvider): mode=source_spot["mode"].upper(), sig=self.SIG, sig_refs=[ - SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"], latitude=lat, - longitude=lon)], + SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"])], time=datetime.now(pytz.UTC).timestamp(), - dx_latitude=lat, - dx_longitude=lon, qrt=source_spot["state"] != "active") return spot diff --git a/static/apidocs/openapi.yml b/static/apidocs/openapi.yml index c6d8b8d..abe3b6b 100644 --- a/static/apidocs/openapi.yml +++ b/static/apidocs/openapi.yml @@ -23,6 +23,7 @@ info: * Added `comment_names` to SIGs in the `/options`, to reflect how they might be referred to in spot comments where it differs from their `name`. * Added `propagation_mode` field to spots + * Added `sig_ref_data_providers` to status and removed `cleanup` ### 1.3 @@ -1720,6 +1721,28 @@ components: is zero, the provider has never updated. example: 1759579508 + SIGRefDataProviderStatus: + type: object + properties: + sig_name: + type: string + description: The name of the SIG. + example: WWFF + enabled: + type: boolean + description: Whether the provider is enabled or not. + example: true + status: + type: string + description: The status of the provider. + example: OK + last_updated: + type: number + description: > + The last time at which this provider received data, UTC seconds since UNIX epoch. If this + is zero, the provider has never updated. + example: 1759579508 + SpotList: type: array items: @@ -1787,17 +1810,6 @@ components: type: integer description: Number of alerts currently in the system. example: 123 - "cleanup": - type: object - properties: - status: - type: string - description: The status of the cleanup thread - example: OK - last_ran: - type: number - description: The last time the cleanup operation ran, UTC seconds since UNIX epoch. - example: 1759579508 "webserver": type: object properties: @@ -1828,6 +1840,11 @@ components: description: An array of all the solar conditions providers. items: $ref: '#/components/schemas/SolarConditionsProviderStatus' + sig_ref_data_providers: + type: array + description: An array of all the SIG reference data providers. + items: + $ref: '#/components/schemas/SIGRefDataProviderStatus' Options: type: object diff --git a/static/js/status.js b/static/js/status.js index 33f5f82..2d8a977 100644 --- a/static/js/status.js +++ b/static/js/status.js @@ -12,9 +12,6 @@ function loadStatus() { $("#web-server-last-api").text(moment.unix(jsonData["webserver"]["last_api_access"]).utc().fromNow()); $("#web-server-last-page").text(moment.unix(jsonData["webserver"]["last_page_access"]).utc().fromNow()); - $("#cleanup-status").text(jsonData["cleanup"]["status"]); - $("#cleanup-last-ran").text(moment.unix(jsonData["cleanup"]["last_ran"]).utc().fromNow()); - jsonData["spot_providers"].forEach(p => { $("#spot-providers-status-container").append(`