w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-06-23 21:25:12 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Solar condition monitoring improvements, mostly polling GIRO at a steady continual rate rather than bursting every hour, bug fixes and commenting improvements

Browse Source
This commit is contained in:
Ian Renton
2026-06-21 08:53:06 +01:00
parent bed263fada
commit b3db6e695c
12 changed files with 75 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
solarconditionsproviders/giroionosonde.py
View File

@@ -10,7 +10,12 @@ from core.constants import HTTP_HEADERS
from solarconditionsproviders.ionosonde_utils import compute_band_states from solarconditionsproviders.ionosonde_utils import compute_band_states
from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider
POLL_INTERVAL = 3600 # 1 hour # Each station gets polled roughly once every hour (3600 seconds). Note that to avoid a burst of requests to the server
# every hour, the requests for data from each station are spaced out throughout the hour, leading to one request being
# sent every 1-2 minutes.
POLL_INTERVAL = 3600
# To avoid looking up all stations in the GIRO system and working out which ones are providing live data, this has been
# manually determined and a CSV provided of all the stations that we can query for live data.
STATIONS_INDEX = "datafiles/didbase-stations.csv" STATIONS_INDEX = "datafiles/didbase-stations.csv"
LGDC_URL = "https://lgdc.uml.edu/common/DIDBGetValues" LGDC_URL = "https://lgdc.uml.edu/common/DIDBGetValues"
HISTORY_HOURS = 24 HISTORY_HOURS = 24
@@ -19,8 +24,9 @@ HISTORY_HOURS = 24
class GIROIonosonde(SolarConditionsProvider): class GIROIonosonde(SolarConditionsProvider):
"""Solar conditions provider using ionosonde data from the GIRO Data Center. """Solar conditions provider using ionosonde data from the GIRO Data Center.
Queries foF2, MUF, and LUF measurements for all stations in datafiles/didbase-stations.csv. Queries foF2, MUF, and LUF measurements for all stations in datafiles/didbase-stations.csv.
Can run alongside KC2GProp: GIRO supplements KC2G's foF2/MUF data with LUF readings, and
stations from each source that the other does not cover are preserved.""" Designed to run alongside KC2GProp even though they produce similar data. GIRO has more stations and includes LUF
data, but is less reliable and often offline."""
def __init__(self, provider_config): def __init__(self, provider_config):
super().__init__(provider_config) super().__init__(provider_config)
@@ -61,64 +67,59 @@ class GIROIonosonde(SolarConditionsProvider):
self._stop_event.set() self._stop_event.set()
def _run(self): def _run(self):
# Real interval at which we poll is the "once per hour" divided by the number of stations, so each one gets
# polled once per hour, just not all at once
interval = POLL_INTERVAL / len(self._stations)
station_index = 0
while True: while True:
self._poll() self._poll_station(self._stations[station_index])
if self._stop_event.wait(timeout=POLL_INTERVAL): station_index = (station_index + 1) % len(self._stations)
if self._stop_event.wait(timeout=interval):
break break
def _poll(self): def _poll_station(self, station):
ursi = station["ursi"]
name = station["name"]
try: try:
logging.debug("Polling GIRO ionosonde data...") logging.debug(f"Polling GIRO ionosonde data for {ursi} ({name})...")
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
from_time = now - timedelta(hours=HISTORY_HOURS) from_time = now - timedelta(hours=HISTORY_HOURS)
cutoff_ts = from_time.timestamp() cutoff_ts = from_time.timestamp()
fof2, muf, luf = self._fetch_station_data(ursi, from_time, now)
if not fof2 or not muf:
return
# Start from the existing ionosonde_data so stations provided by other providers # Start from the existing ionosonde_data so stations provided by other providers
# (e.g. KC2GProp) are preserved for stations GIRO does not cover. # (e.g. KC2GProp) are preserved for stations GIRO does not cover.
ionosonde_data = dict(self._solar_conditions.ionosonde_data or {}) ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
updated_count = 0
for station in self._stations: # Merge GIRO's readings into any existing data for this station.
if self._stop_event.is_set(): existing = ionosonde_data.get(ursi, {})
break merged_fof2 = {**{float(t): v for t, v in (existing.get("fof2") or {}).items()}, **fof2}
ursi = station["ursi"] merged_muf = {**{float(t): v for t, v in (existing.get("muf") or {}).items()}, **muf}
name = station["name"] merged_luf = dict(luf) if luf else {}
try:
fof2, muf, luf = self._fetch_station_data(ursi, from_time, now)
if not fof2 or not muf:
continue
# Merge GIRO's readings into any existing data for this station. merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts}
existing = ionosonde_data.get(ursi, {}) merged_muf = {t: v for t, v in merged_muf.items() if t >= cutoff_ts}
merged_fof2 = {**{float(t): v for t, v in (existing.get("fof2") or {}).items()}, **fof2} merged_luf = {t: v for t, v in merged_luf.items() if t >= cutoff_ts}
merged_muf = {**{float(t): v for t, v in (existing.get("muf") or {}).items()}, **muf}
merged_luf = dict(luf) if luf else {}
merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts}
merged_muf = {t: v for t, v in merged_muf.items() if t >= cutoff_ts}
merged_luf = {t: v for t, v in merged_luf.items() if t >= cutoff_ts}
band_states = compute_band_states(merged_fof2, merged_muf, merged_luf)
ionosonde_data[ursi] = {
"ursi": ursi, "name": name,
"fof2": merged_fof2 or None,
"muf": merged_muf or None,
"luf": merged_luf or None,
"band_states": band_states,
}
updated_count += 1
except Exception:
logging.warning(f"Could not fetch ionosonde data for {ursi} ({name})")
band_states = compute_band_states(merged_fof2, merged_muf, merged_luf)
ionosonde_data[ursi] = {
"ursi": ursi, "name": name,
"fof2": merged_fof2 or None,
"muf": merged_muf or None,
"luf": merged_luf or None,
"band_states": band_states,
}
self.update_data({"ionosonde_data": ionosonde_data}) self.update_data({"ionosonde_data": ionosonde_data})
self.status = "OK" self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC) self.last_update_time = datetime.now(pytz.UTC)
logging.debug(f"Updated ionosonde data for {updated_count} stations.") logging.debug(f"Updated ionosonde data for {ursi} ({name}).")
except Exception: except Exception:
self.status = "Error" self.status = "Error"
logging.exception("Exception in GIRO Ionosonde data provider") logging.exception(f"Exception fetching GIRO ionosonde data for {ursi} ({name})")
self._stop_event.wait(timeout=1)
def _fetch_station_data(self, ursi, from_time, to_time): def _fetch_station_data(self, ursi, from_time, to_time):
"""Fetch foF2, MUF and LUF readings for a station. Returns (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp.""" """Fetch foF2, MUF and LUF readings for a station. Returns (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""

14
solarconditionsproviders/ionosonde_utils.py
View File

@@ -4,16 +4,20 @@ HF_BANDS = [b for b in BANDS if b.is_ham_hf]
def _latest(d) -> float | None: def _latest(d) -> float | None:
return float(d[max(d.keys())]) if d else None """Given a map where the key is a timestamp and the value is a number represented as a string, find the latest
timestamp and return the corresponding value as a float."""
val = str(d[max(d.keys())]) if d else None
return float(val) if (val is not None and val != "None") else None
def compute_band_states(fof2_dict, muf_dict, luf_dict): def compute_band_states(fof2_dict, muf_dict, luf_dict):
"""Compute HF band states from the latest foF2, MUF and LUF values. """Compute HF band states from the latest foF2, MUF and LUF values.
States: Returns a map where the keys are HF bands and the values are as follows:
Closed if band frequency is above MUF or below LUF (if known) "Closed" if band frequency is above MUF or below LUF (if known)
Short if band frequency is >= LUF and < foF2 (good for NVIS) "Short" if band frequency is >= LUF and < foF2 (good for NVIS)
Long if band frequency is >= foF2 and < MUF (good for DX) "Long" if band frequency is >= foF2 and < MUF (good for DX)
""" """
fof2 = _latest(fof2_dict) fof2 = _latest(fof2_dict)

8
solarconditionsproviders/kc2gprop.py
View File

@@ -16,11 +16,11 @@ HISTORY_HOURS = 24
class KC2GProp(SolarConditionsProvider): class KC2GProp(SolarConditionsProvider):
"""Solar conditions provider using ionosonde data from prop.kc2g.com. The API returns only the latest reading per """Solar conditions provider using ionosonde data from prop.kc2g.com. The API returns only the latest reading per
station, so this provider polls every 15 minutes and accumulates a 24-hour time series by merging each new reading station, so this provider polls every 15 minutes and accumulates a 24-hour time series by merging each new reading
into the persisted ionosonde_data, producing the same data structure as GIROIonosonde. into the persisted ionosonde_data, producing the same data structure as GIROIonosonde.
Can run alongside GIROIonosonde: KC2G provides foF2/MUF with good reliability, while GIRO supplements with LUF Designed to run alongside GIROIonosonde even though they produce similar data. KC2G is more reliable and is always
readings. Stations from each source that the other does not cover are preserved.""" online, but has fewer stations and does not provide LUF data."""
def __init__(self, provider_config): def __init__(self, provider_config):
super().__init__(provider_config) super().__init__(provider_config)

9
solarconditionsproviders/noaa3dayforecast.py
View File

@@ -19,6 +19,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
def _parse_percentage_table(lines, section_header, year): def _parse_percentage_table(lines, section_header, year):
"""Find and parse a forecast table using percentages, identified by section_header. This is common to the lookup """Find and parse a forecast table using percentages, identified by section_header. This is common to the lookup
of the solar storm and radio blackout forecast parsing.""" of the solar storm and radio blackout forecast parsing."""
start_idx = None start_idx = None
for i, line in enumerate(lines): for i, line in enumerate(lines):
if section_header in line: if section_header in line:
@@ -28,7 +29,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
logging.warning(f"NOAA 3-day forecast: could not find '{section_header}' section") logging.warning(f"NOAA 3-day forecast: could not find '{section_header}' section")
return None return None
# Find the date header line — the first line within the next few that contains month+day patterns # Find the date header line by scanning the next few lines for month & day patterns
date_header_idx = None date_header_idx = None
for j in range(start_idx + 1, min(start_idx + 6, len(lines))): for j in range(start_idx + 1, min(start_idx + 6, len(lines))):
if re.search(r'[A-Za-z]{3}\s+\d{2}', lines[j]): if re.search(r'[A-Za-z]{3}\s+\d{2}', lines[j]):
@@ -37,12 +38,12 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
if date_header_idx is None: if date_header_idx is None:
logging.warning(f"NOAA 3-day forecast: could not find date header after '{section_header}'") logging.warning(f"NOAA 3-day forecast: could not find date header after '{section_header}'")
return None return None
date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', lines[date_header_idx]) date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', lines[date_header_idx])
if not date_matches: if not date_matches:
logging.warning(f"NOAA 3-day forecast: no dates in header: {lines[date_header_idx]}") logging.warning(f"NOAA 3-day forecast: no dates in header: {lines[date_header_idx]}")
return None return None
# Figure out the date based on the line found
column_timestamps = [] column_timestamps = []
for month_str, day_str in date_matches: for month_str, day_str in date_matches:
try: try:
@@ -52,7 +53,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
logging.warning(f"NOAA 3-day forecast: could not parse date: {month_str} {day_str} {year}") logging.warning(f"NOAA 3-day forecast: could not parse date: {month_str} {day_str} {year}")
return None return None
# Parse data rows: each non-empty line should have a text label and percentage values # Parse data rows. Each non-empty line should have a text label followed by percentage values
result = {} result = {}
for line in lines[date_header_idx + 1:]: for line in lines[date_header_idx + 1:]:
line_stripped = line.strip() line_stripped = line.strip()
@@ -65,6 +66,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
if result: if result:
break break
continue continue
# Row label is everything before the first percentage value # Row label is everything before the first percentage value
row_label = line_stripped[:line_stripped.index(pct_matches[0].group())].strip() row_label = line_stripped[:line_stripped.index(pct_matches[0].group())].strip()
row_data = {} row_data = {}
@@ -90,7 +92,6 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
if "NOAA Kp index breakdown" in line: if "NOAA Kp index breakdown" in line:
start_idx = i start_idx = i
break break
if start_idx is None: if start_idx is None:
logging.warning("NOAA K-index forecast: could not find 'NOAA Kp index breakdown' section") logging.warning("NOAA K-index forecast: could not find 'NOAA Kp index breakdown' section")
return None return None

2
templates/add_spot.html
View File

@@ -76,7 +76,7 @@
</div> </div>
<script src="/js/add-spot.js?v=1782028335"></script> <script src="/js/add-spot.js?v=1782028386"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-add-spot").addClass("active"); $("#nav-link-add-spot").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

2
templates/alerts.html
View File

@@ -75,7 +75,7 @@
</div> </div>
<script src="/js/alerts.js?v=1782028335"></script> <script src="/js/alerts.js?v=1782028386"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-alerts").addClass("active"); $("#nav-link-alerts").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

4
templates/bands.html
View File

@@ -77,8 +77,8 @@
<script> <script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %}; let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script> </script>
<script src="/js/spotsbandsandmap.js?v=1782028335"></script> <script src="/js/spotsbandsandmap.js?v=1782028386"></script>
<script src="/js/bands.js?v=1782028335"></script> <script src="/js/bands.js?v=1782028386"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-bands").addClass("active"); $("#nav-link-bands").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

10
templates/base.html
View File

@@ -1,6 +1,6 @@
{% extends "skeleton.html" %} {% extends "skeleton.html" %}
{% block head_extra %} {% block head_extra %}
<link rel="stylesheet" href="/css/style.css?v=1782028335" type="text/css"> <link rel="stylesheet" href="/css/style.css?v=1782028386" type="text/css">
<link href="/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet"> <link href="/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
<link href="/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet"> <link href="/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
<link href="/vendor/css/solid-6.7.2.min.css" rel="stylesheet"> <link href="/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
<script src="/vendor/js/bootstrap-5.3.8.bundle.min.js"></script> <script src="/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
<script src="/vendor/js/tinycolor2-1.6.0.min.js"></script> <script src="/vendor/js/tinycolor2-1.6.0.min.js"></script>
<script src="/js/utils.js?v=1782028335"></script> <script src="/js/utils.js?v=1782028386"></script>
<script src="/js/ui-ham.js?v=1782028335"></script> <script src="/js/ui-ham.js?v=1782028386"></script>
<script src="/js/geo.js?v=1782028335"></script> <script src="/js/geo.js?v=1782028386"></script>
<script src="/js/common.js?v=1782028335"></script> <script src="/js/common.js?v=1782028386"></script>
{% end %} {% end %}
{% block body %} {% block body %}
<div class="container"> <div class="container">

2
templates/conditions.html
View File

@@ -284,7 +284,7 @@
</div> </div>
<script src="/vendor/js/chart-4.4.9.umd.min.js"></script> <script src="/vendor/js/chart-4.4.9.umd.min.js"></script>
<script src="/js/conditions.js?v=1782028335"></script> <script src="/js/conditions.js?v=1782028386"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-conditions").addClass("active"); $("#nav-link-conditions").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

4
templates/map.html
View File

@@ -95,8 +95,8 @@
<script> <script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %}; let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script> </script>
<script src="/js/spotsbandsandmap.js?v=1782028335"></script> <script src="/js/spotsbandsandmap.js?v=1782028386"></script>
<script src="/js/map.js?v=1782028335"></script> <script src="/js/map.js?v=1782028386"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-map").addClass("active"); $("#nav-link-map").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

4
templates/spots.html
View File

@@ -116,8 +116,8 @@
<script> <script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %}; let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script> </script>
<script src="/js/spotsbandsandmap.js?v=1782028335"></script> <script src="/js/spotsbandsandmap.js?v=1782028386"></script>
<script src="/js/spots.js?v=1782028335"></script> <script src="/js/spots.js?v=1782028386"></script>
<script>$(document).ready(function () { <script>$(document).ready(function () {
$("#nav-link-spots").addClass("active"); $("#nav-link-spots").addClass("active");
}); <!-- highlight active page in nav --></script> }); <!-- highlight active page in nav --></script>

2
templates/status.html
View File

@@ -59,7 +59,7 @@
</div> </div>
</div> </div>
<script src="/js/status.js?v=1782028335"></script> <script src="/js/status.js?v=1782028386"></script>
<script> <script>
$(document).ready(function () { $(document).ready(function () {
$("#nav-link-status").addClass("active"); $("#nav-link-status").addClass("active");