spothole/providers/solarconditions/giroionosonde.py

import csv
import logging
from datetime import datetime, timedelta, timezone
from threading import Event, Thread

import pytz
import requests
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout

from core.constants import HTTP_HEADERS
from providers.solarconditions.ionosonde_utils import compute_band_states
from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider

logger = logging.getLogger(__name__)

# Each station gets polled roughly once every two hours (7200 seconds). Note that to avoid a burst of requests to the
# server every two hours, the requests for data from each station are spaced out throughout the period, leading to one
# request being sent every 3-4 minutes.
POLL_INTERVAL = 7200
# To avoid looking up all stations in the GIRO system and working out which ones are providing live data, this has been
# manually determined and a CSV provided of all the stations that we can query for live data.
STATIONS_INDEX = "datafiles/didbase-stations.csv"
LGDC_URL = "https://lgdc.uml.edu/fastchar/getbest"
HISTORY_HOURS = 24


class GIROIonosonde(SolarConditionsProvider):
    """Solar conditions provider using ionosonde data from the GIRO Data Center.
    Queries foF2, MUF, and LUF measurements for all stations in datafiles/didbase-stations.csv.

    Designed to run alongside KC2GProp even though they produce similar data. GIRO has more stations and includes LUF
    data, but is less reliable and often offline."""

    def __init__(self, provider_config):
        super().__init__("GIRO Ionosonde Data", provider_config)
        self._stations = self._load_stations()
        self._thread = None
        self._stop_event = Event()

        # Pre-populate ionosonde_data with known station names for stations not already present,
        # so the station dropdown is available before the first poll. Does not overwrite existing
        # entries so KC2G cache data is preserved.
        existing = self._solar_conditions.ionosonde_data or {}
        new_entries = {
            s["ursi"]: {
                "ursi": s["ursi"],
                "name": s["name"],
                "fof2": None,
                "muf": None,
                "luf": None,
                "band_states": None,
            }
            for s in self._stations
            if s["ursi"] not in existing
        }
        if new_entries:
            self.update_data({"ionosonde_data": {**existing, **new_entries}})

    @staticmethod
    def _load_stations():
        stations = []
        with open(STATIONS_INDEX, newline="") as f:
            for row in csv.reader(f):
                if len(row) >= 2:
                    stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
        return stations

    def start(self):
        logger.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
        self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider")
        self._thread.start()

    def stop(self):
        self._stop_event.set()

    def _run(self):
        # Real interval at which we poll is the "once per hour" divided by the number of stations, so each one gets
        # polled once per hour, just not all at once
        interval = POLL_INTERVAL / len(self._stations)
        station_index = 0
        while True:
            self._poll_station(self._stations[station_index])
            station_index = (station_index + 1) % len(self._stations)
            if self._stop_event.wait(timeout=interval):
                break

    def _poll_station(self, station):
        ursi = station["ursi"]
        name = station["name"]
        try:
            logger.debug(f"Polling GIRO ionosonde data for {ursi} ({name})...")
            now = datetime.now(timezone.utc)
            from_time = now - timedelta(hours=HISTORY_HOURS)
            cutoff_ts = from_time.timestamp()

            fof2, muf, luf = self._fetch_station_data(ursi, from_time, now)
            if not fof2 or not muf:
                return

            # Start from the existing ionosonde_data so stations provided by other providers
            # (e.g. KC2GProp) are preserved for stations GIRO does not cover.
            ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})

            # Merge GIRO's readings into any existing data for this station.
            existing = ionosonde_data.get(ursi, {})
            merged_fof2 = {
                **{float(t): v for t, v in (existing.get("fof2") or {}).items()},
                **fof2,
            }
            merged_muf = {
                **{float(t): v for t, v in (existing.get("muf") or {}).items()},
                **muf,
            }
            merged_luf = dict(luf) if luf else {}

            merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts}
            merged_muf = {t: v for t, v in merged_muf.items() if t >= cutoff_ts}
            merged_luf = {t: v for t, v in merged_luf.items() if t >= cutoff_ts}

            band_states = compute_band_states(merged_fof2, merged_muf, merged_luf)
            ionosonde_data[ursi] = {
                "ursi": ursi,
                "name": name,
                "fof2": merged_fof2 or None,
                "muf": merged_muf or None,
                "luf": merged_luf or None,
                "band_states": band_states,
            }
            self.update_data({"ionosonde_data": ionosonde_data})
            self.status = "OK"
            self.last_update_time = datetime.now(pytz.UTC)
            logger.debug(f"Updated ionosonde data for {ursi} ({name}).")

        except Exception:
            self.status = "Error"
            logger.exception(f"Exception fetching GIRO ionosonde data for {ursi} ({name})")

    def _fetch_station_data(self, ursi, from_time, to_time):
        """Fetch foF2, MUF and LUF readings for a station. Returns (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""

        from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S")
        to_str = to_time.strftime("%Y.%m.%d+%H:%M:%S")
        url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUF%28D%29,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}"
        try:
            http_response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
            if not http_response.ok:
                logger.warning(f"HTTP {http_response.status_code} when calling Giro ionosonde API.")
                return None, None, None
            return self._parse_all(http_response.text)
        except (ConnectTimeout, ReadTimeout):
            logger.warning("Timeout when accessing Giro ionosonde API.")
            return None, None, None
        except ConnectionError:
            logger.warning("Connection error when accessing Giro ionosonde API.")
            return None, None, None

    @staticmethod
    def _parse_all(text):
        """Parse web server response and return (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""

        fof2_data = {}
        muf_data = {}
        luf_data = {}
        for line in text.splitlines():
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            # Data rows have the following format: timestamp CS foF2 QD MUFD QD fmin QD
            parts = line.split()
            if len(parts) >= 5:
                try:
                    # Python 3.8 TZ parsing fudge
                    ts = datetime.fromisoformat(parts[0].replace("Z", "+00:00")).timestamp()
                except ValueError:
                    continue
                try:
                    fof2_data[ts] = float(parts[2])
                except ValueError:
                    pass
                try:
                    muf_data[ts] = float(parts[4])
                except ValueError:
                    pass
                if len(parts) >= 7:
                    try:
                        luf_data[ts] = float(parts[6])
                    except ValueError:
                        pass
        return fof2_data, muf_data, luf_data