Workaround to fetch ionosonde data from KC2G since the GIRO data source often seems to be down.

solarconditionsproviders/giroionosonde.py

@@ -6,19 +6,21 @@ from threading import Thread, Event
 import pytz
 import requests
 
-from core.constants import HTTP_HEADERS, BANDS
+from core.constants import HTTP_HEADERS
+from solarconditionsproviders.ionosonde_utils import compute_band_states
 from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider
 
 POLL_INTERVAL = 3600  # 1 hour
 STATIONS_INDEX = "datafiles/didbase-stations.csv"
 LGDC_URL = "https://lgdc.uml.edu/common/DIDBGetValues"
 HISTORY_HOURS = 24
-HF_BANDS = [b for b in BANDS if b.is_ham_hf]
 
 
 class GIROIonosonde(SolarConditionsProvider):
     """Solar conditions provider using ionosonde data from the GIRO Data Center.
-    Queries foF2, MUF, and LUF measurements for all stations in datafiles/didbase-stations.csv."""
+    Queries foF2, MUF, and LUF measurements for all stations in datafiles/didbase-stations.csv.
+    Can run alongside KC2GProp: GIRO supplements KC2G's foF2/MUF data with LUF readings, and
+    stations from each source that the other does not cover are preserved."""
 
     def __init__(self, provider_config):
         super().__init__(provider_config)
@@ -27,8 +29,6 @@ class GIROIonosonde(SolarConditionsProvider):
         self._stop_event = Event()
 
     def _load_stations(self):
-        """Load the CSV file containing the list of URSIs and Station Names for currently active ionosondes."""
-
         stations = []
         with open(STATIONS_INDEX, newline='') as f:
             for row in csv.reader(f):
@@ -37,15 +37,19 @@ class GIROIonosonde(SolarConditionsProvider):
         return stations
 
     def setup(self, solar_conditions, solar_conditions_cache):
-        """Prepopulate the ionosonde_data map with known URSI and station names, so that the API exposes this structure
-        even before we actually have any data in it."""
+        """Pre-populate ionosonde_data with known station names for stations not already present,
+        so the station dropdown is available before the first poll. Does not overwrite existing
+        entries so KC2G cache data is preserved."""
 
         super().setup(solar_conditions, solar_conditions_cache)
-        self.update_data({"ionosonde_data": {
-            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None, "luf": None,
-                        "band_states": None}
-            for s in self._stations
-        }})
+        existing = solar_conditions.ionosonde_data or {}
+        new_entries = {
+            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
+                        "luf": None, "band_states": None}
+            for s in self._stations if s["ursi"] not in existing
+        }
+        if new_entries:
+            self.update_data({"ionosonde_data": {**existing, **new_entries}})
 
     def start(self):
         logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
@@ -63,9 +67,13 @@ class GIROIonosonde(SolarConditionsProvider):
 
     def _poll(self):
         try:
-            logging.debug(f"Polling GIRO ionosonde data...")
+            logging.debug("Polling GIRO ionosonde data...")
             now = datetime.now(timezone.utc)
             from_time = now - timedelta(hours=HISTORY_HOURS)
+            cutoff_ts = from_time.timestamp()
+
+            # Start from the existing ionosonde_data so stations provided by other providers
+            # (e.g. KC2GProp) are preserved for stations GIRO does not cover.
             ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
             updated_count = 0
 
@@ -76,11 +84,28 @@ class GIROIonosonde(SolarConditionsProvider):
                 name = station["name"]
                 try:
                     fof2, muf, luf = self._fetch_station_data(ursi, from_time, now)
-                    if fof2 and muf:
-                        band_states = self._compute_band_statess(fof2, muf, luf or {})
-                        ionosonde_data[ursi] = {"ursi": ursi, "name": name, "fof2": fof2, "muf": muf,
-                                                "luf": luf or None, "band_states": band_states}
-                        updated_count += 1
+                    if not fof2 or not muf:
+                        continue
+
+                    # Merge GIRO's readings into any existing data for this station.
+                    existing = ionosonde_data.get(ursi, {})
+                    merged_fof2 = {**{float(t): v for t, v in (existing.get("fof2") or {}).items()}, **fof2}
+                    merged_muf = {**{float(t): v for t, v in (existing.get("muf") or {}).items()}, **muf}
+                    merged_luf = dict(luf) if luf else {}
+
+                    merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts}
+                    merged_muf = {t: v for t, v in merged_muf.items() if t >= cutoff_ts}
+                    merged_luf = {t: v for t, v in merged_luf.items() if t >= cutoff_ts}
+
+                    band_states = compute_band_states(merged_fof2, merged_muf, merged_luf)
+                    ionosonde_data[ursi] = {
+                        "ursi": ursi, "name": name,
+                        "fof2": merged_fof2 or None,
+                        "muf": merged_muf or None,
+                        "luf": merged_luf or None,
+                        "band_states": band_states,
+                    }
+                    updated_count += 1
                 except Exception:
                     logging.warning(f"Could not fetch ionosonde data for {ursi} ({name})")
 
@@ -91,7 +116,7 @@ class GIROIonosonde(SolarConditionsProvider):
 
         except Exception:
             self.status = "Error"
-            logging.exception(f"Exception in GIRO Ionosonde data provider")
+            logging.exception("Exception in GIRO Ionosonde data provider")
             self._stop_event.wait(timeout=1)
 
     def _fetch_station_data(self, ursi, from_time, to_time):
@@ -105,40 +130,6 @@ class GIROIonosonde(SolarConditionsProvider):
             return None, None, None
         return self._parse_all(response.text)
 
-    @staticmethod
-    def _latest(d):
-        """Return the value with the highest timestamp key, or None if the dict is empty."""
-        return d[max(d.keys())] if d else None
-
-    @staticmethod
-    def _compute_band_statess(fof2_dict, muf_dict, luf_dict):
-        """Compute HF band states from the latest foF2, MUF and LUF values.
-
-        States:
-          Closed if band frequency is below LUF (if known) or above MUF
-          Short if band frequency is >= LUF and < foF2 (good for NVIS)
-          Long if band frequency is >= foF2 and < MUF (good for DX)
-        """
-
-        # We have a list of timestamped data for each value, but for this we only want the latest value
-        fof2 = GIROIonosonde._latest(fof2_dict)
-        muf = GIROIonosonde._latest(muf_dict)
-        luf = GIROIonosonde._latest(luf_dict)
-        if fof2 is None or muf is None:
-            return {}
-        band_states = {}
-
-        # Iterate over all ham HF bands, we don't care about the others at this point
-        for band in HF_BANDS:
-            freq = band.start_freq / 1000000
-            if freq > muf or (luf is not None and freq < luf):
-                band_states[band.name] = "Closed"
-            elif freq < fof2:
-                band_states[band.name] = "Short"
-            else:
-                band_states[band.name] = "Long"
-        return band_states
-
     @staticmethod
     def _parse_all(text):
         """Parse web server response and return (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""