Workaround to fetch ionosonde data from KC2G since the GIRO data source often seems to be down.

solarconditionsproviders/giroionosonde.py

@@ -6,19 +6,21 @@ from threading import Thread, Event
 import pytz
 import requests
 
-from core.constants import HTTP_HEADERS, BANDS
+from core.constants import HTTP_HEADERS
+from solarconditionsproviders.ionosonde_utils import compute_band_states
 from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider
 
 POLL_INTERVAL = 3600  # 1 hour
 STATIONS_INDEX = "datafiles/didbase-stations.csv"
 LGDC_URL = "https://lgdc.uml.edu/common/DIDBGetValues"
 HISTORY_HOURS = 24
-HF_BANDS = [b for b in BANDS if b.is_ham_hf]
 
 
 class GIROIonosonde(SolarConditionsProvider):
     """Solar conditions provider using ionosonde data from the GIRO Data Center.
-    Queries foF2, MUF, and LUF measurements for all stations in datafiles/didbase-stations.csv."""
+    Queries foF2, MUF, and LUF measurements for all stations in datafiles/didbase-stations.csv.
+    Can run alongside KC2GProp: GIRO supplements KC2G's foF2/MUF data with LUF readings, and
+    stations from each source that the other does not cover are preserved."""
 
     def __init__(self, provider_config):
         super().__init__(provider_config)
@@ -27,8 +29,6 @@ class GIROIonosonde(SolarConditionsProvider):
         self._stop_event = Event()
 
     def _load_stations(self):
-        """Load the CSV file containing the list of URSIs and Station Names for currently active ionosondes."""
-
         stations = []
         with open(STATIONS_INDEX, newline='') as f:
             for row in csv.reader(f):
@@ -37,15 +37,19 @@ class GIROIonosonde(SolarConditionsProvider):
         return stations
 
     def setup(self, solar_conditions, solar_conditions_cache):
-        """Prepopulate the ionosonde_data map with known URSI and station names, so that the API exposes this structure
-        even before we actually have any data in it."""
+        """Pre-populate ionosonde_data with known station names for stations not already present,
+        so the station dropdown is available before the first poll. Does not overwrite existing
+        entries so KC2G cache data is preserved."""
 
         super().setup(solar_conditions, solar_conditions_cache)
-        self.update_data({"ionosonde_data": {
-            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None, "luf": None,
-                        "band_states": None}
-            for s in self._stations
-        }})
+        existing = solar_conditions.ionosonde_data or {}
+        new_entries = {
+            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
+                        "luf": None, "band_states": None}
+            for s in self._stations if s["ursi"] not in existing
+        }
+        if new_entries:
+            self.update_data({"ionosonde_data": {**existing, **new_entries}})
 
     def start(self):
         logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
@@ -63,9 +67,13 @@ class GIROIonosonde(SolarConditionsProvider):
 
     def _poll(self):
         try:
-            logging.debug(f"Polling GIRO ionosonde data...")
+            logging.debug("Polling GIRO ionosonde data...")
             now = datetime.now(timezone.utc)
             from_time = now - timedelta(hours=HISTORY_HOURS)
+            cutoff_ts = from_time.timestamp()
+
+            # Start from the existing ionosonde_data so stations provided by other providers
+            # (e.g. KC2GProp) are preserved for stations GIRO does not cover.
             ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
             updated_count = 0
 
@@ -76,11 +84,28 @@ class GIROIonosonde(SolarConditionsProvider):
                 name = station["name"]
                 try:
                     fof2, muf, luf = self._fetch_station_data(ursi, from_time, now)
-                    if fof2 and muf:
-                        band_states = self._compute_band_statess(fof2, muf, luf or {})
-                        ionosonde_data[ursi] = {"ursi": ursi, "name": name, "fof2": fof2, "muf": muf,
-                                                "luf": luf or None, "band_states": band_states}
-                        updated_count += 1
+                    if not fof2 or not muf:
+                        continue
+
+                    # Merge GIRO's readings into any existing data for this station.
+                    existing = ionosonde_data.get(ursi, {})
+                    merged_fof2 = {**{float(t): v for t, v in (existing.get("fof2") or {}).items()}, **fof2}
+                    merged_muf = {**{float(t): v for t, v in (existing.get("muf") or {}).items()}, **muf}
+                    merged_luf = dict(luf) if luf else {}
+
+                    merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts}
+                    merged_muf = {t: v for t, v in merged_muf.items() if t >= cutoff_ts}
+                    merged_luf = {t: v for t, v in merged_luf.items() if t >= cutoff_ts}
+
+                    band_states = compute_band_states(merged_fof2, merged_muf, merged_luf)
+                    ionosonde_data[ursi] = {
+                        "ursi": ursi, "name": name,
+                        "fof2": merged_fof2 or None,
+                        "muf": merged_muf or None,
+                        "luf": merged_luf or None,
+                        "band_states": band_states,
+                    }
+                    updated_count += 1
                 except Exception:
                     logging.warning(f"Could not fetch ionosonde data for {ursi} ({name})")
 
@@ -91,7 +116,7 @@ class GIROIonosonde(SolarConditionsProvider):
 
         except Exception:
             self.status = "Error"
-            logging.exception(f"Exception in GIRO Ionosonde data provider")
+            logging.exception("Exception in GIRO Ionosonde data provider")
             self._stop_event.wait(timeout=1)
 
     def _fetch_station_data(self, ursi, from_time, to_time):
@@ -105,40 +130,6 @@ class GIROIonosonde(SolarConditionsProvider):
             return None, None, None
         return self._parse_all(response.text)
 
-    @staticmethod
-    def _latest(d):
-        """Return the value with the highest timestamp key, or None if the dict is empty."""
-        return d[max(d.keys())] if d else None
-
-    @staticmethod
-    def _compute_band_statess(fof2_dict, muf_dict, luf_dict):
-        """Compute HF band states from the latest foF2, MUF and LUF values.
-
-        States:
-          Closed if band frequency is below LUF (if known) or above MUF
-          Short if band frequency is >= LUF and < foF2 (good for NVIS)
-          Long if band frequency is >= foF2 and < MUF (good for DX)
-        """
-
-        # We have a list of timestamped data for each value, but for this we only want the latest value
-        fof2 = GIROIonosonde._latest(fof2_dict)
-        muf = GIROIonosonde._latest(muf_dict)
-        luf = GIROIonosonde._latest(luf_dict)
-        if fof2 is None or muf is None:
-            return {}
-        band_states = {}
-
-        # Iterate over all ham HF bands, we don't care about the others at this point
-        for band in HF_BANDS:
-            freq = band.start_freq / 1000000
-            if freq > muf or (luf is not None and freq < luf):
-                band_states[band.name] = "Closed"
-            elif freq < fof2:
-                band_states[band.name] = "Short"
-            else:
-                band_states[band.name] = "Long"
-        return band_states
-
     @staticmethod
     def _parse_all(text):
         """Parse web server response and return (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""

solarconditionsproviders/ionosonde_utils.py

@@ -0,0 +1,33 @@
+from core.constants import BANDS
+
+HF_BANDS = [b for b in BANDS if b.is_ham_hf]
+
+
+def _latest(d):
+    return d[max(d.keys())] if d else None
+
+
+def compute_band_states(fof2_dict, muf_dict, luf_dict):
+    """Compute HF band states from the latest foF2, MUF and LUF values.
+
+    States:
+      Closed if band frequency is above MUF or below LUF (if known)
+      Short  if band frequency is >= LUF and < foF2 (good for NVIS)
+      Long   if band frequency is >= foF2 and < MUF (good for DX)
+    """
+
+    fof2 = _latest(fof2_dict)
+    muf = _latest(muf_dict)
+    luf = _latest(luf_dict) if luf_dict else None
+    if fof2 is None or muf is None:
+        return {}
+    band_states = {}
+    for band in HF_BANDS:
+        freq = band.start_freq / 1_000_000
+        if freq > muf or (luf is not None and freq < luf):
+            band_states[band.name] = "Closed"
+        elif freq < fof2:
+            band_states[band.name] = "Short"
+        else:
+            band_states[band.name] = "Long"
+    return band_states

solarconditionsproviders/kc2gprop.py

@@ -0,0 +1,121 @@
+import logging
+from datetime import datetime, timezone, timedelta
+from threading import Thread, Event
+
+import pytz
+import requests
+
+from core.constants import HTTP_HEADERS
+from solarconditionsproviders.ionosonde_utils import compute_band_states
+from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider
+
+POLL_INTERVAL = 900  # 15 minutes
+KC2G_URL = "https://prop.kc2g.com/api/stations.json"
+HISTORY_HOURS = 24
+
+
+class KC2GProp(SolarConditionsProvider):
+    """Solar conditions provider using ionosonde data from prop.kc2g.com. The API returns only the latest reading per
+     station, so this provider polls every 15 minutes and accumulates a 24-hour time series by merging each new reading
+     into the persisted ionosonde_data, producing the same data structure as GIROIonosonde.
+
+     Can run alongside GIROIonosonde: KC2G provides foF2/MUF with good reliability, while GIRO supplements with LUF
+     readings. Stations from each source that the other does not cover are preserved."""
+
+    def __init__(self, provider_config):
+        super().__init__(provider_config)
+        self._thread = None
+        self._stop_event = Event()
+
+    def start(self):
+        logging.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=POLL_INTERVAL):
+                break
+
+    def _poll(self):
+        try:
+            logging.debug("Polling KC2G ionosonde data...")
+            response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
+            if response.status_code != 200:
+                logging.warning(f"KC2G ionosonde API returned HTTP {response.status_code}")
+                return
+
+            now = datetime.now(timezone.utc)
+            cutoff_ts = (now - timedelta(hours=HISTORY_HOURS)).timestamp()
+
+            # Start from existing ionosonde_data so the accumulated time series survives across polls and restarts and
+            # stations provided only by GIROIonosonde are not discarded
+            ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
+            updated_count = 0
+
+            for reading in response.json():
+                station = reading.get("station", {})
+                ursi = station.get("code")
+                name = station.get("name")
+                if not ursi or not name:
+                    continue
+
+                time_str = reading.get("time")
+                if not time_str:
+                    continue
+                try:
+                    ts = datetime.fromisoformat(time_str)
+                    if ts.tzinfo is None:
+                        ts = ts.replace(tzinfo=timezone.utc)
+                    ts_float = ts.timestamp()
+                except ValueError:
+                    continue
+
+                # Skip readings outside our history window (some stations have months-old data)
+                if ts_float < cutoff_ts:
+                    continue
+
+                fof2_val = reading.get("fof2")
+                muf_val = reading.get("mufd")
+                if fof2_val is None and muf_val is None:
+                    continue
+
+                # Merge this reading into the existing time series for the station.
+                existing = ionosonde_data.get(ursi, {})
+                fof2_dict = dict(existing.get("fof2") or {})
+                muf_dict = dict(existing.get("muf") or {})
+                # LUF is not available from KC2G; carry forward whatever GIRO has written.
+                luf_dict = {float(t): v for t, v in (existing.get("luf") or {}).items()}
+
+                fof2_dict[ts_float] = fof2_val
+                muf_dict[ts_float] = muf_val
+
+                # Trim all series to the 24-hour window.
+                fof2_dict = {t: v for t, v in fof2_dict.items() if t >= cutoff_ts}
+                muf_dict = {t: v for t, v in muf_dict.items() if t >= cutoff_ts}
+                luf_dict = {t: v for t, v in luf_dict.items() if t >= cutoff_ts}
+
+                band_states = compute_band_states(fof2_dict, muf_dict, luf_dict)
+                ionosonde_data[ursi] = {
+                    "ursi": ursi,
+                    "name": name,
+                    "fof2": fof2_dict or None,
+                    "muf": muf_dict or None,
+                    "luf": luf_dict or None,
+                    "band_states": band_states,
+                }
+                updated_count += 1
+
+            self.update_data({"ionosonde_data": ionosonde_data})
+            self.status = "OK"
+            self.last_update_time = datetime.now(pytz.UTC)
+            logging.debug(f"Updated KC2G ionosonde data for {updated_count} stations.")
+
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in KC2G ionosonde data provider")
+            self._stop_event.wait(timeout=1)