From 64a7b27887b66a70c83813fff4721cd4e471c301 Mon Sep 17 00:00:00 2001 From: Ian Renton Date: Fri, 15 May 2026 18:25:54 +0100 Subject: [PATCH] Support fetching ionosonde data for FoF2 and MUF display on the Conditions page --- config-example.yml | 4 + data/solar_conditions.py | 2 + datafiles/didbase-stations.csv | 42 +++++ server/handlers/pagetemplate.py | 6 +- server/webserver.py | 3 +- solarconditionsproviders/giroionosonde.py | 122 +++++++++++++ templates/about.html | 4 +- templates/add_spot.html | 4 +- templates/alerts.html | 4 +- templates/bands.html | 6 +- templates/base.html | 8 +- templates/conditions.html | 25 ++- templates/map.html | 6 +- templates/spots.html | 6 +- templates/status.html | 4 +- webassets/apidocs/openapi.yml | 44 +++++ webassets/js/conditions.js | 211 +++++++++++++++++++++- 17 files changed, 473 insertions(+), 28 deletions(-) create mode 100644 datafiles/didbase-stations.csv create mode 100644 solarconditionsproviders/giroionosonde.py diff --git a/config-example.yml b/config-example.yml index ec81825..efe5400 100644 --- a/config-example.yml +++ b/config-example.yml @@ -183,6 +183,10 @@ solar-condition-providers: class: "NOAA3dayForecast" name: "NOAA 3-day Forecast" enabled: true + - + class: "GIROIonosonde" + name: "GIRO Ionosonde Data" + enabled: true # Port to open the local web server on web-server-port: 8080 diff --git a/data/solar_conditions.py b/data/solar_conditions.py index aff0d5a..07c56a5 100644 --- a/data/solar_conditions.py +++ b/data/solar_conditions.py @@ -161,6 +161,8 @@ class SolarConditions: blackout_forecast_r1r2: dict = None # NOAA Radio Blackout (R3 or greater) probability forecast, keyed by UNIX timestamp of start of day UTC blackout_forecast_r3_or_greater: dict = None + # Ionosonde measurements from LGDC, list of dicts with keys: ursi, name, fof2, muf + ionosonde_data: list = None # Derived values (populated by infer_descriptions()) # HF radio blackout risk description, derived from xray diff --git a/datafiles/didbase-stations.csv b/datafiles/didbase-stations.csv new file mode 100644 index 0000000..29e575a --- /dev/null +++ b/datafiles/didbase-stations.csv @@ -0,0 +1,42 @@ +AA343,"Almaty, Kazakhstan" +AL945,"Alpena, United States" +AT138,"Athens, Greece" +AU930,"Austin, United States" +BR52P,"Brisbane, Australia" +BVJ03,"Boa Vista, Brazil" +CAJ2M,"Cachoeira Paulista, Brazil" +CB53N,"Canberra, Australia" +CGK21,"Campo Grande, Brazil" +DB049,"Dourbes, Belgium" +DW41K,"Darwin, Australia" +EA036,"El Arenosillo, Spain" +EB040,"Roquetes, Spain" +EG931,"Eglin Air Force Base, United States" +FF051,"Fairford, United Kingdom" +GA762,"Gakona, United States" +GM037,"Gibilmanna, Italy" +GR13L,"Grahamstown, South Africa" +HE13N,"Hermanus, South Africa" +HO54K,"Hobart, Australia" +IC437,"I-Cheon, South Korea" +IF843,"Idaho National Laboratory, United States" +JI91J,"Jicamarca, Peru" +JR055,"Juliusruh, Germany" +LAA38,"Lajes Terceira Island, Portugal" +LL721,"Lualualei, United States" +LM42B,"Learmonth, Australia" +LV12P,"Louisvale, South Africa" +MHJ45,"Millstone Hill, United States" +ML10L,"Malindi, Kenya" +NI135,"Nicosia, Cyprus" +NI63_,"Norfolk, Australia" +PA836,"Portt Arguello, United States" +PE43K,"Perth, Australia" +PF765,"Poker Flat, United States" +PQ052,"Pruhonice, Czechia" +RO041,"Rome, Italy" +SAA0K,"Saoluis, Brazil" +SO148,"Sopron, Hungary" +TR169,"Tromso, Norway" +TV51R,"Townsville, Australia" +VT139,"San Vito, Italy" diff --git a/server/handlers/pagetemplate.py b/server/handlers/pagetemplate.py index 39adc06..06540d3 100644 --- a/server/handlers/pagetemplate.py +++ b/server/handlers/pagetemplate.py @@ -11,11 +11,12 @@ from core.prometheus_metrics_handler import page_requests_counter class PageTemplateHandler(tornado.web.RequestHandler): """Handler for all HTML pages generated from templates""" - def initialize(self, template_name, web_server_metrics, has_hamqsl=False, has_noaa_forecast=False): + def initialize(self, template_name, web_server_metrics, has_hamqsl=False, has_noaa_forecast=False, has_giro_ionosonde=False): self._template_name = template_name self._web_server_metrics = web_server_metrics self._has_hamqsl = has_hamqsl self._has_noaa_forecast = has_noaa_forecast + self._has_giro_ionosonde = has_giro_ionosonde def get(self): # Metrics @@ -27,4 +28,5 @@ class PageTemplateHandler(tornado.web.RequestHandler): # Load named template, and provide variables used in templates self.render(self._template_name + ".html", software_version=SOFTWARE_VERSION, allow_spotting=ALLOW_SPOTTING, web_ui_options=WEB_UI_OPTIONS, baseurl=BASE_URL, current_path=self.request.path, - has_hamqsl=self._has_hamqsl, has_noaa_forecast=self._has_noaa_forecast) \ No newline at end of file + has_hamqsl=self._has_hamqsl, has_noaa_forecast=self._has_noaa_forecast, + has_giro_ionosonde=self._has_giro_ionosonde) \ No newline at end of file diff --git a/server/webserver.py b/server/webserver.py index a942895..93ac067 100644 --- a/server/webserver.py +++ b/server/webserver.py @@ -57,8 +57,9 @@ class WebServer: provider_classes = [type(p).__name__ for p in self._solar_condition_providers if p.enabled] has_hamqsl = "HamQSL" in provider_classes has_noaa_forecast = "NOAA3dayForecast" in provider_classes + has_giro_ionosonde = "GIROIonosonde" in provider_classes page_opts = {"web_server_metrics": self.web_server_metrics, "has_hamqsl": has_hamqsl, - "has_noaa_forecast": has_noaa_forecast} + "has_noaa_forecast": has_noaa_forecast, "has_giro_ionosonde": has_giro_ionosonde} app = tornado.web.Application([ # Routes for API calls diff --git a/solarconditionsproviders/giroionosonde.py b/solarconditionsproviders/giroionosonde.py new file mode 100644 index 0000000..5add621 --- /dev/null +++ b/solarconditionsproviders/giroionosonde.py @@ -0,0 +1,122 @@ +import csv +import logging +from datetime import datetime, timezone, timedelta +from threading import Thread, Event + +import pytz +import requests + +from core.constants import HTTP_HEADERS +from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider + +POLL_INTERVAL = 3600 # 1 hour +STATIONS_INDEX = "datafiles/didbase-stations.csv" +LGDC_URL = "https://lgdc.uml.edu/common/DIDBGetValues" +HISTORY_HOURS = 24 + + +class GIROIonosonde(SolarConditionsProvider): + """Solar conditions provider using ionosonde data from the GIRO Data Center. + Queries foF2 and MUF measurements for all stations in datafiles/didbase-stations.csv.""" + + def __init__(self, provider_config): + super().__init__(provider_config) + self._stations = self._load_stations() + self._thread = None + self._stop_event = Event() + + def _load_stations(self): + """Load the CSV file containing the list of URSIs and Station Names for currently active ionosondes.""" + + stations = [] + with open(STATIONS_INDEX, newline='') as f: + for row in csv.reader(f): + if len(row) >= 2: + stations.append({"ursi": row[0].strip(), "name": row[1].strip()}) + return stations + + def start(self): + logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.") + self._thread = Thread(target=self._run, daemon=True) + self._thread.start() + + def stop(self): + self._stop_event.set() + + def _run(self): + while True: + self._poll() + if self._stop_event.wait(timeout=POLL_INTERVAL): + break + + def _poll(self): + try: + logging.debug(f"Polling {self.name} ionosonde data...") + now = datetime.now(timezone.utc) + from_time = now - timedelta(hours=HISTORY_HOURS) + results = [] + + for station in self._stations: + if self._stop_event.is_set(): + break + ursi = station["ursi"] + name = station["name"] + entry = {"ursi": ursi, "name": name, "fof2": None, "muf": None} + try: + fof2, muf = self._fetch_station_data(ursi, from_time, now) + entry["fof2"] = fof2 + entry["muf"] = muf + if fof2 and muf: + results.append(entry) + except Exception: + logging.debug(f"Could not fetch ionosonde data for {ursi} ({name})") + + self.update_data({"ionosonde_data": results}) + self.status = "OK" + self.last_update_time = datetime.now(pytz.UTC) + logging.debug(f"Received ionosonde data for {len(results)} stations from {self.name}.") + + except Exception: + self.status = "Error" + logging.exception(f"Exception in GIRO Ionosonde data provider ({self.name})") + self._stop_event.wait(timeout=1) + + def _fetch_station_data(self, ursi, from_time, to_time): + """Fetch foF2 and MUF readings for a station. Returns (fof2_dict, muf_dict) keyed by UNIX timestamp.""" + + from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S") + to_str = to_time.strftime("%Y.%m.%d+%H:%M:%S") + url = ( + f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUFD&DMUF=3000&fromDate={from_str}&toDate={to_str}" + ) + response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15)) + if response.status_code != 200: + return None, None + return self._parse_all(response.text) + + @staticmethod + def _parse_all(text): + """Parse web server response and return (fof2_dict, muf_dict) keyed by UNIX timestamp.""" + + fof2_data = {} + muf_data = {} + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + # Data rows: timestamp CS foF2 QD MUFD QD + parts = line.split() + if len(parts) >= 5: + try: + ts = datetime.fromisoformat(parts[0].replace('Z', '+00:00')).timestamp() + except ValueError: + continue + try: + fof2_data[ts] = float(parts[2]) + except ValueError: + pass + try: + muf_data[ts] = float(parts[4]) + except ValueError: + pass + return fof2_data, muf_data diff --git a/templates/about.html b/templates/about.html index fae2e95..483e6a4 100644 --- a/templates/about.html +++ b/templates/about.html @@ -27,7 +27,7 @@

What data sources are supported?

Spothole can retrieve spots from: Telnet-based DX clusters, the Reverse Beacon Network (RBN), the APRS Internet Service (APRS-IS), POTA, SOTA, WWFF, GMA, WWBOTA, HEMA, Parks 'n' Peaks, ZLOTA, WOTA, LLOTA, WWTOTA, Tiles on the Air, the UK Packet Repeater Network, and any site based on the xOTA software by nischu.

Spothole can retrieve alerts from: NG3K, POTA, SOTA, WWFF, Parks 'n' Peaks, WOTA and BOTA.

-

Spothole can retrieve solar and propagation condition data from HamQSL.

+

Spothole can retrieve solar and propagation condition data from HamQSL, the NOAA Space Weather Prediction Center and the Lowell GIRO Data Center.

Spothole can also perform lookups for callsign data on behalf of the user from QRZ.com and HamQTH.

Note that the server owner has not necessarily enabled all these data sources. In particular it is common to disable RBN, to avoid the server being swamped with FT8 traffic, and to disable APRS-IS and UK Packet Net so that the server only displays stations where there is likely to be an operator physically present for a QSO.

Between the various data sources, the following Special Interest Groups (SIGs) are supported: Parks on the Air (POTA), Summits on the Air (SOTA), Worldwide Flora & Fauna (WWFF), Global Mountain Activity (GMA), Worldwide Bunkers on the Air (WWBOTA), HuMPs Excluding Marilyns Award (HEMA), Islands on the Air (IOTA), Mills on the Air (MOTA), the Amateur Radio Lighthouse Socirty (ARLHS), International Lighthouse Lightship Weekend (ILLW), Silos on the Air (SIOTA), World Castles Award (WCA), New Zealand on the Air (ZLOTA), Keith Roget Memorial National Parks Award (KRMNPA), Wainwrights on the Air (WOTA), Beaches on the Air (BOTA), Lagos y Lagunas On the Air (LLOTA), Towers on the Air (WWTOTA), Tiles on the Air, Worked All Britain (WAB), Worked All Ireland (WAI), and Toilets on the Air (TOTA).

@@ -69,7 +69,7 @@

This software is dedicated to the memory of Tom G1PJB, SK, a friend and colleague who sadly passed away around the time I started writing it in Autumn 2025. I was looking forward to showing it to you when it was done.

- + {% end %} \ No newline at end of file diff --git a/templates/add_spot.html b/templates/add_spot.html index 875f992..7674326 100644 --- a/templates/add_spot.html +++ b/templates/add_spot.html @@ -69,8 +69,8 @@ - - + + {% end %} \ No newline at end of file diff --git a/templates/alerts.html b/templates/alerts.html index 1fbf306..da0d023 100644 --- a/templates/alerts.html +++ b/templates/alerts.html @@ -70,8 +70,8 @@ - - + + {% end %} \ No newline at end of file diff --git a/templates/bands.html b/templates/bands.html index c2fed6f..d9d79d4 100644 --- a/templates/bands.html +++ b/templates/bands.html @@ -76,9 +76,9 @@ - - - + + + {% end %} \ No newline at end of file diff --git a/templates/base.html b/templates/base.html index 3d0f179..83c3d42 100644 --- a/templates/base.html +++ b/templates/base.html @@ -24,7 +24,7 @@ Spothole - + @@ -52,9 +52,9 @@ integrity="sha384-L1eE4eD41kpBIWe2I0eHy+GnEUC4RIpcvibVW2JCminuPlTl+2Bc528iPdVMg5Dn" crossorigin="anonymous"> - - - + + + diff --git a/templates/conditions.html b/templates/conditions.html index 07ecf5a..285cb97 100644 --- a/templates/conditions.html +++ b/templates/conditions.html @@ -137,7 +137,7 @@ {% if has_noaa_forecast %}
- Forecast + Solar Weather Forecast
@@ -173,6 +173,25 @@
{% end %} +{% if has_giro_ionosonde %} +
+
+ Critical Frequency and Maximum Usable Frequency +
+
+
+ + +
+
+ +
Data from the Lowell GIRO Data Center.
+
+
+{% end %} +
DX Opportunities @@ -230,8 +249,8 @@
- - + + diff --git a/templates/map.html b/templates/map.html index 3ea8bfb..c9e182e 100644 --- a/templates/map.html +++ b/templates/map.html @@ -94,9 +94,9 @@ - - - + + + {% end %} \ No newline at end of file diff --git a/templates/spots.html b/templates/spots.html index 862d458..a0261c9 100644 --- a/templates/spots.html +++ b/templates/spots.html @@ -104,9 +104,9 @@ - - - + + + {% end %} \ No newline at end of file diff --git a/templates/status.html b/templates/status.html index abdc528..594d82f 100644 --- a/templates/status.html +++ b/templates/status.html @@ -59,8 +59,8 @@
- - + + diff --git a/webassets/apidocs/openapi.yml b/webassets/apidocs/openapi.yml index 0b3cc25..4655e3f 100644 --- a/webassets/apidocs/openapi.yml +++ b/webassets/apidocs/openapi.yml @@ -13,6 +13,10 @@ info: ## Changelog + ### 1.4 + + * `/solar` response now includes `ionosonde_data`, a list of ionosonde station measurements (foF2 and MUF) sourced from the GIRO Data Center. + ### 1.3 * `/spots`, `/spots/stream`, `/alerts`, `/alerts/stream`, and `/lookup/call` now accept optional QRZ.com and HamQTH credentials as query parameters. When supplied, returned data is enriched with operator name, home location etc. from those services. @@ -1683,6 +1687,46 @@ components: type: string description: Electron flux impact description, derived from electron flux level. example: "No impact" + ionosonde_data: + type: array + nullable: true + description: > + Ionosonde measurements from the GIRO Data Center, covering active stations listed in the + system. Only stations for which data was successfully retrieved are included. Null if the + GIROIonosonde provider has not yet completed its first poll. + items: + $ref: '#/components/schemas/IonosondeStation' + + IonosondeStation: + type: object + description: Ionosonde measurement data for a single station, covering approximately the last 24 hours. + properties: + ursi: + type: string + description: URSI code identifying the ionosonde station. + example: DB049 + name: + type: string + description: Human-readable name of the ionosonde station. + example: Dourbes + fof2: + type: object + nullable: true + description: F2 layer critical frequency (foF2) measurements in MHz, keyed by UNIX timestamp (UTC seconds since epoch) of each measurement. + additionalProperties: + type: number + example: + "1747267201.0": 7.45 + "1747267501.0": 7.50 + muf: + type: object + nullable: true + description: Maximum Usable Frequency (MUF) for a 3000 km path in MHz, keyed by UNIX timestamp (UTC seconds since epoch) of each measurement. + additionalProperties: + type: number + example: + "1747267201.0": 21.66 + "1747267501.0": 21.80 SolarConditionsProviderStatus: type: object diff --git a/webassets/js/conditions.js b/webassets/js/conditions.js index a924ec9..24730d5 100644 --- a/webassets/js/conditions.js +++ b/webassets/js/conditions.js @@ -1,8 +1,12 @@ // Cache for the full dxstats API response, so we can reload on the fly if the user changes the value of their continent // in the select box let dxStatsData = null; -// Forecast chart +// Kp forecast chart let kpChart = null; +// Cache for ionosonde data from the API +let ionosondeData = null; +// Ionosonde foF2/MUF chart +let ionosondeChart = null; // Load solar conditions function loadSolarConditions() { @@ -109,6 +113,14 @@ function loadSolarConditions() { electronFlux <= 100 ? 'bg-success-subtle' : electronFlux <= 1000 ? 'bg-warning-subtle' : 'bg-danger-subtle'); } + // Ionosonde + + if (jsonData.ionosonde_data && jsonData.ionosonde_data.length > 0) { + ionosondeData = jsonData.ionosonde_data; + populateIonosondeDropdown(ionosondeData); + renderIonosondeData(); + } + // Forecast renderKIndexForecast(jsonData.k_index_forecast); @@ -348,6 +360,203 @@ function renderBlackoutForecast(r1r2Data, r3Data) { .append(makeRow('R3 or greater', e => e.r3)); } +// Populate the ionosonde station dropdown and restore any saved selection +function populateIonosondeDropdown(data) { + const select = $('#ionosonde-station'); + const savedUrsi = localStorage.getItem('#ionosonde-station:value'); + const savedValue = savedUrsi ? JSON.parse(savedUrsi) : null; + select.empty(); + data.forEach(function (station) { + select.append($('