Refactor of caching & data storage part 9 #118

providers/alert/alert_provider.py

@@ -0,0 +1,45 @@
+from datetime import datetime
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+
+class AlertProvider:
+    """Generic alert provider class. Subclasses of this query the individual APIs for alerts."""
+
+    def __init__(self, name, provider_config):
+        """Constructor"""
+
+        self.name = name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+        self._alerts = DATA_STORE.alerts
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def _submit_batch(self, alerts):
+        """Submit a batch of alerts retrieved from the provider. There is no timestamp checking like there is for spots,
+        because alerts could be created at any point for any time in the future. Rely on hashcode-based id matching
+        to deal with duplicates."""
+
+        # Sort the batch so that earliest ones go in first. This helps keep the ordering correct when alerts are fired
+        # off to SSE listeners.
+        alerts = sorted(alerts, key=lambda a: (a.start_time if a and a.start_time else 0))
+        for alert in alerts:
+            # Fill in any blanks and add to the list
+            alert.infer_missing()
+            self._add_alert(alert)
+
+    def _add_alert(self, alert):
+        if not alert.expired():
+            self._alerts.set(alert.id, alert)
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/alert/bota.py

@@ -0,0 +1,62 @@
+from datetime import datetime, timedelta
+
+import pytz
+from bs4 import BeautifulSoup
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class BOTA(HTTPAlertProvider):
+    """Alert provider for Beaches on the Air"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://www.beachesontheair.com/"
+
+    def __init__(self, provider_config):
+        super().__init__("BOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Find the table of upcoming alerts
+        bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
+        if not bs.body:
+            return new_alerts
+        div = bs.body.find('div', attrs={'class': 'view-activations-public'})
+        if div:
+            table = div.find('table', attrs={'class': 'views-table'})
+            if table:
+                tbody = table.find('tbody')
+                if not tbody:
+                    return new_alerts
+                for row in tbody.find_all('tr'):
+                    cells = row.find_all('td')
+                    first_cell_anchor = cells[0].find('a') if len(cells) > 0 else None
+                    second_cell_anchor = cells[1].find('a') if len(cells) > 1 else None
+                    if not first_cell_anchor or not second_cell_anchor:
+                        continue
+                    first_cell_text = first_cell_anchor.get_text().strip()
+                    ref_name = first_cell_text.split(" by ")[0]
+                    dx_call = second_cell_anchor.get_text().strip().upper()
+
+                    # Get the date, dealing with the fact we get no year so have to figure out if it's last year or next year
+                    date_span = cells[2].find('span') if len(cells) > 2 else None
+                    if not date_span:
+                        continue
+                    date_text = date_span.get_text().strip()
+                    date_time = datetime.strptime(date_text, "%d %b - %H:%M UTC").replace(tzinfo=pytz.UTC)
+                    date_time = date_time.replace(year=datetime.now(pytz.UTC).year)
+                    # If this was more than a day ago, activation is actually next year
+                    if date_time < datetime.now(pytz.UTC) - timedelta(days=1):
+                        date_time = date_time.replace(year=datetime.now(pytz.UTC).year + 1)
+
+                    # Convert to our alert format
+                    alert = Alert(source=self.name,
+                                  dx_calls=[dx_call],
+                                  sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
+                                  start_time=date_time.timestamp(),
+                                  is_dxpedition=False)
+
+                    new_alerts.append(alert)
+        return new_alerts

providers/alert/http_alert_provider.py

@@ -0,0 +1,75 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+
+from providers.alert.alert_provider import AlertProvider
+from core.constants import HTTP_HEADERS
+
+
+class HTTPAlertProvider(AlertProvider):
+    """Generic alert provider class for providers that request data via HTTP(S). Just for convenience to avoid code
+    duplication. Subclasses of this query the individual APIs for data."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        super().__init__(name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info("Set up query of " + self.name + " alert API every " + str(self._poll_interval) + " seconds.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval):
+                break
+
+    def _poll(self):
+        try:
+            # Request data from API
+            logging.debug("Polling " + self.name + " alert API...")
+            http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                new_alerts = self._http_response_to_alerts(http_response)
+                # Submit the new alerts for processing. There might not be any alerts for the less popular programs.
+                if new_alerts:
+                    self._submit_batch(new_alerts)
+
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.debug("Received data from " + self.name + " alert API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} alerts API.")
+
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} alerts API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} alerts API.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP JSON Alert Provider (" + self.name + ")")
+            # Brief pause on error before the next poll, but still respond promptly to stop()
+            self._stop_event.wait(timeout=1)
+
+    def _http_response_to_alerts(self, http_response):
+        """Convert an HTTP response returned by the API into alert data. The whole response is provided here so the subclass
+        implementations can check for HTTP status codes if necessary, and handle the response as JSON, XML, text, whatever
+        the API actually provides."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/alert/ng3k.py

@@ -0,0 +1,89 @@
+import re
+from datetime import datetime
+from typing import cast
+
+import pytz
+from rss_parser import Parser
+from rss_parser.models.rss import RSS
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+
+
+class NG3K(HTTPAlertProvider):
+    """Alert provider NG3K DXpedition list"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://www.ng3k.com/adxo.xml"
+    AS_CALL_PATTERN = re.compile("as ([a-z0-9/]+)", re.IGNORECASE)
+
+    def __init__(self, provider_config):
+        super().__init__("NG3K", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
+        # Iterate through source data
+        for source_alert in rss.channel.items:
+            # Deal with "the format"...
+            parts = source_alert.description.split(" --\n")
+
+            start_string = parts[0].split("-")[0]
+            end_string = parts[0].split("-")[1]
+
+            end_year = end_string.split(", ")[1].strip()
+
+            if ", " in start_string:
+                start_year = start_string.split(", ")[1].strip()
+                start_mon = start_string.split(", ")[0][0:3].strip()
+                start_day = start_string.split(", ")[0][4:].strip()
+            else:
+                start_year = end_year
+                start_mon = start_string[0:3].strip()
+                start_day = start_string[4:].strip()
+
+            if " " in end_string.split(", ")[0]:
+                end_mon = end_string.split(", ")[0].split(" ")[0].strip()
+                end_day = end_string.split(", ")[0].split(" ")[1].strip()
+            else:
+                end_day = end_string.split(", ")[0].strip()
+                end_mon = start_mon
+
+            start_timestamp = datetime.strptime(start_year + " " + start_mon + " " + start_day, "%Y %b %d").replace(
+                tzinfo=pytz.UTC).timestamp()
+            end_timestamp = datetime.strptime(end_year + " " + end_mon + " " + end_day + " 23:59",
+                                              "%Y %b %d %H:%M").replace(
+                tzinfo=pytz.UTC).timestamp()
+
+            # Sometimes the DX callsign is "real", sometimes you just get a prefix with the real working callsigns being
+            # provided in the "by" field. e.g. call="JW", by="By LA7XK as JW7XK, LA6VM as JW6VM, LA9DL as JW9DL". So
+            # if there are "as" callsigns in the "by" field, we extract them and use them, otherwise we fall back to the
+            # "real" call field.
+            extra_parts = parts[5].split("; ")
+            by = extra_parts[0]
+            dx_calls = [x.group(1) for x in re.finditer(self.AS_CALL_PATTERN, by)]
+            if not dx_calls:
+                dx_calls = [parts[2].upper()]
+
+            # "Calls" of TBA, TBC or TBD are not real attempts at Turkish callsigns
+            dx_calls = list(filter(lambda a: a != "TBA" and a != "TBC" and a != "TBD", dx_calls))
+
+            dx_country = parts[1]
+            qsl_info = parts[3]
+            bands = extra_parts[1] if len(extra_parts) > 1 else ""
+            modes = extra_parts[2] if len(extra_parts) > 2 else ""
+            comment = extra_parts[3] if len(extra_parts) > 3 else ""
+
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          dx_calls=dx_calls,
+                          dx_country=dx_country,
+                          freqs_modes=bands + (("; " + modes) if modes != "" else ""),
+                          comment=by + "; " + comment + "; " + qsl_info,
+                          start_time=start_timestamp,
+                          end_time=end_timestamp,
+                          is_dxpedition=True)
+
+            # Add to our list.
+            new_alerts.append(alert)
+        return new_alerts

providers/alert/parksnpeaks.py

@@ -0,0 +1,61 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class ParksNPeaks(HTTPAlertProvider):
+    """Alert provider for Parks n Peaks"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://parksnpeaks.org/api/ALERTS/"
+
+    def __init__(self, provider_config):
+        super().__init__("ParksNPeaks", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json():
+            # Calculate some things
+            sig = source_alert["Class"].upper()
+            if " - " in source_alert["Location"]:
+                split = source_alert["Location"].split(" - ")
+                sig_ref = split[0]
+                sig_ref_name = split[1]
+            else:
+                sig_ref = source_alert["WWFFID"]
+                sig_ref_name = source_alert["Location"]
+            start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(
+                tzinfo=pytz.UTC).timestamp()
+
+            sigrefs = []
+            # PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list,
+            # so mask this out if we got it.
+            if sig != "QRP":
+                sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)]
+
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert["alID"],
+                          dx_calls=[source_alert["CallSign"].upper()],
+                          freqs_modes=source_alert["Freq"] + " " + source_alert["MODE"],
+                          comment=source_alert["Comments"],
+                          sig_refs=sigrefs,
+                          start_time=start_time,
+                          is_dxpedition=False)
+
+            # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
+            if sig and sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
+                logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!")
+
+            # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
+            # the alert list. Note that while ZLOTA has its own spots API, it doesn't have its own alerts API. So that
+            # means the PnP *spot* provider rejects ZLOTA spots here, but the PnP *alerts* provider here allows ZLOTA.
+            if sig not in ["POTA", "SOTA", "WWFF"]:
+                new_alerts.append(alert)
+        return new_alerts

providers/alert/pota.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+
+import pytz
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class POTA(HTTPAlertProvider):
+    """Alert provider for Parks on the Air"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://api.pota.app/activation"
+
+    def __init__(self, provider_config):
+        super().__init__("POTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json():
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert["scheduledActivitiesId"],
+                          dx_calls=[source_alert["activator"].upper()],
+                          freqs_modes=source_alert["frequencies"],
+                          comment=source_alert["comments"],
+                          sig_refs=[SIGRef(id=source_alert["reference"], sig="POTA", name=source_alert["name"],
+                                           url="https://pota.app/#/park/" + source_alert["reference"])],
+                          start_time=datetime.strptime(source_alert["startDate"] + source_alert["startTime"],
+                                                       "%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
+                          end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"],
+                                                     "%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
+                          is_dxpedition=False)
+
+            # Add to our list, but exclude any old spots that POTA can sometimes give us where even the end time is
+            # in the past. Don't worry about de-duping, removing old alerts etc. at this point; other code will do
+            # that for us.
+            if alert.end_time and alert.end_time > datetime.now(pytz.UTC).timestamp():
+                new_alerts.append(alert)
+        return new_alerts

providers/alert/sota.py

@@ -0,0 +1,44 @@
+from datetime import datetime
+
+import pytz
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class SOTA(HTTPAlertProvider):
+    """Alert provider for Summits on the Air"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://api-db2.sota.org.uk/api/alerts/365/all/all"
+
+    def __init__(self, provider_config):
+        super().__init__("SOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json():
+            # Convert to our alert format
+            details = source_alert["summitDetails"].split(", ")
+            summit_name = details[0]
+            summit_points = None
+            if len(details) > 2:
+                summit_points = int(details[-1].split(" ")[0])
+            alert = Alert(source=self.name,
+                          source_id=source_alert["id"],
+                          dx_calls=[source_alert["activatingCallsign"].upper()],
+                          dx_names=[source_alert["activatorName"].upper()],
+                          freqs_modes=source_alert["frequency"],
+                          comment=source_alert["comments"],
+                          sig_refs=[
+                              SIGRef(id=source_alert["associationCode"] + "/" + source_alert["summitCode"], sig="SOTA",
+                                     name=summit_name, activation_score=summit_points)],
+                          start_time=datetime.strptime(source_alert["dateActivated"],
+                                                       "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=pytz.UTC).timestamp(),
+                          is_dxpedition=False)
+
+            # Add to our list
+            new_alerts.append(alert)
+        return new_alerts

providers/alert/wota.py

@@ -0,0 +1,64 @@
+from datetime import datetime
+from typing import cast
+
+import pytz
+from rss_parser import Parser as RSSParser
+from rss_parser.models.rss import RSS
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class WOTA(HTTPAlertProvider):
+    """Alert provider for Wainwrights on the Air"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://www.wota.org.uk/alerts_rss.php"
+    RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
+
+    def __init__(self, provider_config):
+        super().__init__("WOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig")))
+        # Iterate through source data
+        for source_alert in rss.channel.items:
+
+            # Reject GUID missing or zero
+            if not source_alert.guid or not source_alert.guid.content or source_alert.guid.content == "http://www.wota.org.uk/alerts/0":
+                continue
+
+            # Pick apart the title
+            title_split = source_alert.title.split(" on ")
+            dx_call = title_split[0]
+            ref = None
+            ref_name = None
+            if len(title_split) > 1:
+                ref_split = title_split[1].split(" - ")
+                ref = str(ref_split[0])
+                if len(ref_split) > 1:
+                    ref_name = str(ref_split[1])
+
+            # Pick apart the description
+            desc_split = source_alert.description.split(". ")
+            freqs_modes = desc_split[0].replace("Frequencies/modes:", "").strip()
+            comment = None
+            if len(desc_split) > 1:
+                comment = desc_split[1].strip()
+
+            time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert.guid.content,
+                          dx_calls=[dx_call],
+                          freqs_modes=freqs_modes,
+                          comment=comment,
+                          sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
+                          start_time=time.timestamp())
+
+            # Add to our list.
+            new_alerts.append(alert)
+        return new_alerts

providers/alert/wwff.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+
+import pytz
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class WWFF(HTTPAlertProvider):
+    """Alert provider for Worldwide Flora and Fauna"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://spots.wwff.co/static/agendas.json"
+
+    def __init__(self, provider_config):
+        super().__init__("WWFF", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json():
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert["id"],
+                          dx_calls=[source_alert["activator_call"].upper()],
+                          freqs_modes=source_alert["band"] + " " + source_alert["mode"],
+                          comment=source_alert["remarks"],
+                          sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")],
+                          start_time=datetime.strptime(source_alert["utc_start"],
+                                                       "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
+                          end_time=datetime.strptime(source_alert["utc_end"],
+                                                     "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
+                          is_dxpedition=False)
+
+            # Add to our list
+            new_alerts.append(alert)
+        return new_alerts

providers/sigrefdata/arlhs.py

@@ -0,0 +1,35 @@
+import csv
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ARLHS(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Amateur Radio Light House Society"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "ARLHS"
+    DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            if "ARLHS" in row and row["ARLHS"] != "":
+                ref_id = row["ARLHS"]
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                              "Latitude"] != "" else None,
+                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                              "Longitude"] != "" else None,
+                                          grid=row["Maidenhead Locator"]))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/dme.py

@@ -0,0 +1,41 @@
+import csv
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
+
+
+class DME(LocalFileSIGRefDataProvider):
+    """SIG ref data provider for Diploma Municipios de Espana"""
+
+    SIG = "DME"
+    PATH = "datafiles/MUNICIPIOS.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.PATH)
+
+    def _file_to_data(self, path):
+        new_data = []
+        with open(path, encoding="latin-1") as _f:
+            for row in csv.DictReader(_f, delimiter=";"):
+                ref_id = row["COD_INE"][:5]
+                latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LATITUD_ETRS89_REGCAN95") else None
+                longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LONGITUD_ETRS89_REGCAN95") else None
+
+                ref = SIGRef(sig=self.SIG, id=ref_id,
+                                          name=row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"],
+                                          latitude=latitude,
+                                          longitude=longitude)
+                if latitude and longitude:
+                    ref.grid = latlong_to_locator(latitude, longitude, 6)
+                new_data.append(ref)
+
+                # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
+                # of the data in this case
+                if self._stop:
+                    break
+
+        return new_data

providers/sigrefdata/file_download_sig_ref_data_provider.py

@@ -0,0 +1,80 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from providers.sigrefdata.sig_ref_data_provider import SIGRefDataProvider
+
+
+class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
+    """Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
+
+    def __init__(self, sig_name, provider_config, url, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
+        super().__init__(sig_name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+        self._url_data_cache = URLDataCache("sigrefdata_" + sig_name)
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info(
+            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " days.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
+                break
+
+    def _poll(self):
+        try:
+            # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
+            logging.debug("Downloading " + self.sig_name + " SIG ref data...")
+            http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                new_data = self._http_response_to_data(http_response)
+                # Add the new data to the SIG Ref data store
+                if new_data:
+                    self._add_data(new_data)
+
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.debug("Received SIG ref data for " + self.sig_name)
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.")
+
+        except ConnectionError:
+            self.status = "Error"
+            logging.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.")
+        except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
+            logging.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP SIG Ref Data Provider (" + self.sig_name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _http_response_to_data(self, http_response):
+        """Convert an HTTP response returned by the server into SIG Ref data. The whole response is provided here so the
+        subclass implementations can check for HTTP status codes if necessary, and handle the response as JSON, CSV,
+        whatever the remote file actually is."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/sigrefdata/gma.py

@@ -0,0 +1,34 @@
+import csv
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class GMA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Global Mountain Activity"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "GMA"
+    DATA_URL = "https://www.gma.rocks/download/summits.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["Reference"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                          "Latitude"] != "" else None,
+                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                          "Longitude"] != "" else None,
+                                      grid=row["Maidenhead Locator"]))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/illw.py

@@ -0,0 +1,35 @@
+import csv
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ILLW(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for International Lighthouse & Lightship Weekend"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "ILLW"
+    DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            if "ILLW" in row and row["ILLW"] != "":
+                ref_id = row["ILLW"]
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                              "Latitude"] != "" else None,
+                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                              "Longitude"] != "" else None,
+                                          grid=row["Maidenhead Locator"]))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/iota.py

@@ -0,0 +1,41 @@
+import logging
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class IOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Islands on the Air"""
+
+    POLL_INTERVAL_DAYS = 365
+    SIG = "IOTA"
+    DATA_URL = "https://www.iota-world.org/islands-on-the-air/downloads/download-file.html?path=groups.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data:
+                ref_id = ref["refno"]
+                latitude = float(ref["latitude_min"]) + float(ref["latitude_max"]) / 2.0
+                longitude = float(ref["longitude_min"]) + float(ref["longitude_max"]) / 2.0
+                grid = None
+                try:
+                    grid = latlong_to_locator(latitude, longitude, 6)
+                except ValueError:
+                    logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
+
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
+                                          grid=grid, latitude=latitude, longitude=longitude))
+
+                # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
+                # of the data in this case
+                if self._stop_event.is_set():
+                    break
+
+        return new_data

providers/sigrefdata/llota.py

@@ -0,0 +1,37 @@
+from pyhamtools.locator import locator_to_latlong
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class LLOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Lagos y Lagunas on the Air"""
+
+    POLL_INTERVAL_DAYS = 7
+    SIG = "LLOTA"
+    DATA_URL = "https://llota.app/api/public/references"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data:
+                ref_id = ref["reference_code"]
+                grid = str(ref["grid_locator"])
+                ll = locator_to_latlong(grid)
+
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
+                                          url="https://llota.app/list/ref/" + ref_id,
+                                          grid=grid,
+                                          latitude=ll[0],
+                                          longitude=ll[1]))
+
+                # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
+                # of the data in this case
+                if self._stop_event.is_set():
+                    break
+
+        return new_data

providers/sigrefdata/local_file_sig_ref_data_provider.py

@@ -0,0 +1,38 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from providers.sigrefdata.sig_ref_data_provider import SIGRefDataProvider
+
+
+class LocalFileSIGRefDataProvider(SIGRefDataProvider):
+    """Generic SIG ref data provider class for providers that fetch their data from a local file on startup."""
+
+    def __init__(self, sig, provider_config, path):
+        super().__init__(sig, provider_config)
+        self._path = path
+        self._stop = False
+
+    def start(self):
+        logging.debug("Loading " + self.sig_name + " SIG ref data from file.")
+        try:
+            new_data = self._file_to_data(self._path)
+            if new_data:
+                self._add_data(new_data)
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+            else:
+                self.status = "Error"
+                logging.info("Failed to load SIG ref data for " + self.sig_name)
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")", e, exc_info=True)
+
+    def stop(self):
+        self._stop = True
+
+    def _file_to_data(self, path):
+        """Load a file on the given path and turn it into SIG Ref data."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/sigrefdata/mota.py

@@ -0,0 +1,34 @@
+import csv
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class MOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Mills on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "MOTA"
+    DATA_URL = "https://www.gma.rocks/download/mills.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["Reference"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                          "Latitude"] != "" else None,
+                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                          "Longitude"] != "" else None,
+                                      grid=row["Maidenhead Locator"]))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/pota.py

@@ -0,0 +1,34 @@
+import csv
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class POTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Parks on the Air"""
+
+    POLL_INTERVAL_DAYS = 7
+    SIG = "POTA"
+    DATA_URL = "https://pota.app/all_parks_ext.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["reference"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
+                                      url="https://pota.app/#/park/" + ref_id,
+                                      grid=row["grid"] if "grid" in row else None,
+                                      latitude=float(row["latitude"]) if "latitude" in row and row[
+                                          "latitude"] != "" else None,
+                                      longitude=float(row["longitude"]) if "longitude" in row and row[
+                                          "longitude"] != "" else None))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/sig_ref_data_provider.py

@@ -0,0 +1,39 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+
+class SIGRefDataProvider:
+    """Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data."""
+
+    def __init__(self, sig_name, provider_config):
+        """Constructor"""
+
+        self.sig_name = sig_name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+        self.reference_count = 0
+
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def _add_data(self, new_data):
+        """Add all the provided reference data objects to the data store."""
+
+        for d in new_data:
+            DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d
+        self.reference_count = len(new_data)
+        logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)

providers/sigrefdata/siota.py

@@ -0,0 +1,31 @@
+import csv
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class SIOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Silos on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "SIOTA"
+    DATA_URL = "https://www.silosontheair.com/data/silos.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["SILO_CODE"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
+                                      grid=row["LOCATOR"] if "LOCATOR" in row else None,
+                                      latitude=float(row["LAT"]) if "LAT" in row else None,
+                                      longitude=float(row["LNG"]) if "LNG" in row else None))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/sota.py

@@ -0,0 +1,39 @@
+import csv
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class SOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Summits on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "SOTA"
+    DATA_URL = "https://storage.sota.org.uk/summitslist.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["SummitCode"]
+            latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
+            longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
+            ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
+                                      url="https://www.sotadata.org.uk/en/summit/" + ref_id,
+                                      latitude=latitude,
+                                      longitude=longitude,
+                                      activation_score=int(row["Points"]) if "Points" in row else None)
+            if latitude and longitude:
+                ref.grid = latlong_to_locator(latitude, longitude, 6)
+            new_data.append(ref)
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/toilets.py

@@ -0,0 +1,30 @@
+import csv
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
+
+
+class Toilets(LocalFileSIGRefDataProvider):
+    """SIG ref data provider for Toilets on the Air"""
+
+    SIG = "Toilets"
+    PATH = "datafiles/toilets.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.PATH)
+
+    def _file_to_data(self, path):
+        new_data = []
+        with open(path) as _f:
+            csv_data = _f.read()
+            dr = csv.DictReader(csv_data.splitlines())
+            for row in dr:
+                new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
+                                              longitude=float(row["lon"])))
+
+                # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
+                # of the data in this case
+                if self._stop:
+                    break
+
+        return new_data

providers/sigrefdata/towers.py

@@ -0,0 +1,32 @@
+import csv
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class Towers(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Towers on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "Towers"
+    DATA_URL = "https://wwtota.com/servis/generate_csv.php?ref=&filter=all"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
+            ref_id = row["Ref"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
+                                      url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id,
+                                      grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
+                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
+                                      longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/wca.py

@@ -0,0 +1,49 @@
+import csv
+import logging
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WCA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for World Castles Award"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "WCA"
+    DATA_URL = "https://polo.ham2k.com/data/activities/wca/all-castles.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["REF"]
+
+            coords_str = row["COORDINATES"]
+            latitude = None
+            longitude = None
+            grid = None
+            try:
+                if coords_str:
+                    split = coords_str.split(", ")
+                    latitude = float(split[0])
+                    longitude = float(split[1])
+                    grid = latlong_to_locator(latitude, longitude)
+            except ValueError:
+                logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
+
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=latitude,
+                                      longitude=longitude,
+                                      grid=grid))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/wota.py

@@ -0,0 +1,36 @@
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Wainwrights on the Air"""
+
+    POLL_INTERVAL_DAYS = 365
+    SIG = "WOTA"
+    DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for feature in http_response.json().get("features", []):
+            ref_id = feature["properties"]["wotaId"]
+            # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
+            # added to them
+            url = "https://www.wota.org.uk/MM_" + ref_id
+            if ref_id.upper().startswith("LDO-"):
+                number = int(ref_id.upper().replace("LDO-", ""))
+                url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
+
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
+                                      grid=feature["properties"]["qthLocator"],
+                                      latitude=feature["geometry"]["coordinates"][1],
+                                      longitude=feature["geometry"]["coordinates"][0]))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/wwbota.py

@@ -0,0 +1,32 @@
+import csv
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WWBOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Worldwide Bunkers on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "WWBOTA"
+    DATA_URL = "https://api.wwbota.org/bunkers/?format=CSV"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["Reference"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None,
+                                      grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
+                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
+                                      longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/wwff.py

@@ -0,0 +1,35 @@
+import csv
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WWFF(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Worldwide Flora & Fauna"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "WWFF"
+    DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["reference"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
+                                      url="https://wwff.co/directory/?showRef=" + ref_id,
+                                      grid=row["iaruLocator"] if "iaruLocator" in row and row[
+                                          "iaruLocator"] != "-" else None,
+                                      latitude=float(row["latitude"]) if "latitude" in row and row[
+                                          "latitude"] != "" and row["latitude"] != "-" else None,
+                                      longitude=float(row["longitude"]) if "longitude" in row and row[
+                                          "longitude"] != "" and row["longitude"] != "-" else None))
+
+            # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
+            # the data in this case
+            if self._stop_event.is_set():
+                break
+
+        return new_data

providers/sigrefdata/zlota.py

@@ -0,0 +1,39 @@
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ZLOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for New Zealand on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "ZLOTA"
+    DATA_URL = "https://ontheair.nz/assets/assets.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data:
+                ref_id = ref["code"]
+                latitude = ref["y"]
+                longitude = ref["x"]
+
+                ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
+                                          url="https://ontheair.nz/assets/" + ref_id.replace("/", "_"),
+                                          latitude=latitude,
+                                          longitude=longitude)
+                if latitude and longitude:
+                    ref.grid = latlong_to_locator(latitude, longitude, 6)
+                new_data.append(ref)
+
+                # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
+                # of the data in this case
+                if self._stop_event.is_set():
+                    break
+
+        return new_data

providers/solarconditions/giroionosonde.py

@@ -0,0 +1,173 @@
+import csv
+import logging
+from datetime import datetime, timezone, timedelta
+from threading import Thread, Event
+
+import pytz
+import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from providers.solarconditions.ionosonde_utils import compute_band_states
+from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider
+
+# Each station gets polled roughly once every hour (3600 seconds). Note that to avoid a burst of requests to the server
+# every hour, the requests for data from each station are spaced out throughout the hour, leading to one request being
+# sent every 1-2 minutes.
+POLL_INTERVAL = 3600
+# To avoid looking up all stations in the GIRO system and working out which ones are providing live data, this has been
+# manually determined and a CSV provided of all the stations that we can query for live data.
+STATIONS_INDEX = "datafiles/didbase-stations.csv"
+LGDC_URL = "https://lgdc.uml.edu/common/DIDBGetValues"
+HISTORY_HOURS = 24
+
+
+class GIROIonosonde(SolarConditionsProvider):
+    """Solar conditions provider using ionosonde data from the GIRO Data Center.
+    Queries foF2, MUF, and LUF measurements for all stations in datafiles/didbase-stations.csv.
+
+    Designed to run alongside KC2GProp even though they produce similar data. GIRO has more stations and includes LUF
+    data, but is less reliable and often offline."""
+
+    def __init__(self, provider_config):
+        super().__init__("GIRO Ionosonde Data", provider_config)
+        self._stations = self._load_stations()
+        self._thread = None
+        self._stop_event = Event()
+
+        # Pre-populate ionosonde_data with known station names for stations not already present,
+        # so the station dropdown is available before the first poll. Does not overwrite existing
+        # entries so KC2G cache data is preserved.
+        existing = self._solar_conditions.ionosonde_data or {}
+        new_entries = {
+            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
+                        "luf": None, "band_states": None}
+            for s in self._stations if s["ursi"] not in existing
+        }
+        if new_entries:
+            self.update_data({"ionosonde_data": {**existing, **new_entries}})
+
+    @staticmethod
+    def _load_stations():
+        stations = []
+        with open(STATIONS_INDEX, newline='') as f:
+            for row in csv.reader(f):
+                if len(row) >= 2:
+                    stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
+        return stations
+
+    def start(self):
+        logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        # Real interval at which we poll is the "once per hour" divided by the number of stations, so each one gets
+        # polled once per hour, just not all at once
+        interval = POLL_INTERVAL / len(self._stations)
+        station_index = 0
+        while True:
+            self._poll_station(self._stations[station_index])
+            station_index = (station_index + 1) % len(self._stations)
+            if self._stop_event.wait(timeout=interval):
+                break
+
+    def _poll_station(self, station):
+        ursi = station["ursi"]
+        name = station["name"]
+        try:
+            logging.debug(f"Polling GIRO ionosonde data for {ursi} ({name})...")
+            now = datetime.now(timezone.utc)
+            from_time = now - timedelta(hours=HISTORY_HOURS)
+            cutoff_ts = from_time.timestamp()
+
+            fof2, muf, luf = self._fetch_station_data(ursi, from_time, now)
+            if not fof2 or not muf:
+                return
+
+            # Start from the existing ionosonde_data so stations provided by other providers
+            # (e.g. KC2GProp) are preserved for stations GIRO does not cover.
+            ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
+
+            # Merge GIRO's readings into any existing data for this station.
+            existing = ionosonde_data.get(ursi, {})
+            merged_fof2 = {**{float(t): v for t, v in (existing.get("fof2") or {}).items()}, **fof2}
+            merged_muf = {**{float(t): v for t, v in (existing.get("muf") or {}).items()}, **muf}
+            merged_luf = dict(luf) if luf else {}
+
+            merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts}
+            merged_muf = {t: v for t, v in merged_muf.items() if t >= cutoff_ts}
+            merged_luf = {t: v for t, v in merged_luf.items() if t >= cutoff_ts}
+
+            band_states = compute_band_states(merged_fof2, merged_muf, merged_luf)
+            ionosonde_data[ursi] = {
+                "ursi": ursi, "name": name,
+                "fof2": merged_fof2 or None,
+                "muf": merged_muf or None,
+                "luf": merged_luf or None,
+                "band_states": band_states,
+            }
+            self.update_data({"ionosonde_data": ionosonde_data})
+            self.status = "OK"
+            self.last_update_time = datetime.now(pytz.UTC)
+            logging.debug(f"Updated ionosonde data for {ursi} ({name}).")
+
+        except Exception:
+            self.status = "Error"
+            logging.exception(f"Exception fetching GIRO ionosonde data for {ursi} ({name})")
+
+    def _fetch_station_data(self, ursi, from_time, to_time):
+        """Fetch foF2, MUF and LUF readings for a station. Returns (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""
+
+        from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S")
+        to_str = to_time.strftime("%Y.%m.%d+%H:%M:%S")
+        url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUFD,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}"
+        try:
+            http_response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
+            if not http_response.ok:
+                logging.warning(f"HTTP {http_response.status_code} when calling Giro ionosonde API.")
+                return None, None, None
+            return self._parse_all(http_response.text)
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing Giro ionosonde API.")
+            return None, None, None
+        except ConnectionError:
+            logging.warning("Connection error when accessing Giro ionosonde API.")
+            return None, None, None
+
+    @staticmethod
+    def _parse_all(text):
+        """Parse web server response and return (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""
+
+        fof2_data = {}
+        muf_data = {}
+        luf_data = {}
+        for line in text.splitlines():
+            line = line.strip()
+            if not line or line.startswith('#'):
+                continue
+            # Data rows have the following format: timestamp CS foF2 QD MUFD QD fmin QD
+            parts = line.split()
+            if len(parts) >= 5:
+                try:
+                    # Python 3.8 TZ parsing fudge
+                    ts = datetime.fromisoformat(parts[0].replace('Z', '+00:00')).timestamp()
+                except ValueError:
+                    continue
+                try:
+                    fof2_data[ts] = float(parts[2])
+                except ValueError:
+                    pass
+                try:
+                    muf_data[ts] = float(parts[4])
+                except ValueError:
+                    pass
+                if len(parts) >= 7:
+                    try:
+                        luf_data[ts] = float(parts[6])
+                    except ValueError:
+                        pass
+        return fof2_data, muf_data, luf_data

providers/solarconditions/hamqsl.py

@@ -0,0 +1,112 @@
+import logging
+from xml.etree import ElementTree
+
+import pytz
+from dateutil import parser as dateutil_parser, tz as dateutil_tz
+
+from providers.solarconditions.http_solar_conditions_provider import HTTPSolarConditionsProvider
+
+POLL_INTERVAL = 3600  # 1 hour
+URL = "https://www.hamqsl.com/solarxml.php"
+
+
+class HamQSL(HTTPSolarConditionsProvider):
+    """Solar conditions provider using the HamQSL.com XML API (https://www.hamqsl.com/solarxml.php).
+    Provides solar flux index, geomagnetic indices, and HF/VHF propagation condition summaries."""
+
+    def __init__(self, provider_config):
+        super().__init__("HamQSL", provider_config, URL, POLL_INTERVAL)
+
+    def _http_response_to_solar_conditions(self, http_response):
+        root = ElementTree.fromstring(http_response.text)
+        sd = root.find("solardata")
+        if sd is None:
+            logging.warning("HamQSL solar conditions API returned unexpected XML structure")
+            return None
+
+        # Some error checking functions in case the data is janky.
+
+        def text(tag, default=None):
+            if sd is None:
+                logging.warning("HamQSL solar conditions API returned unexpected XML structure")
+                return default
+            el = sd.find(tag)
+            return el.text.strip() if el is not None and el.text else default
+
+        def float_val(tag, default=None):
+            try:
+                return float(text(tag))
+            except (ValueError, TypeError):
+                return default
+
+        def int_val(tag, default=None):
+            try:
+                return int(text(tag))
+            except (ValueError, TypeError):
+                return default
+
+        # Process HF band conditions
+        hf_conditions = {}
+        calc = sd.find("calculatedconditions")
+        if calc is not None:
+            for band_el in calc.findall("band"):
+                name = band_el.get("name")
+                time = band_el.get("time")
+                condition = band_el.text.strip() if band_el.text else None
+                if name and time and condition:
+                    hf_conditions[f"{name}-{time}"] = condition
+
+        # Process VHF propagation conditions
+        vhf_map = {}
+        vhf = sd.find("calculatedvhfconditions")
+        if vhf is not None:
+            for ph_el in vhf.findall("phenomenon"):
+                key = (ph_el.get("name"), ph_el.get("location"))
+                vhf_map[key] = ph_el.text.strip() if ph_el.text else None
+
+        # Parse the "updated" timestamp string (format: "28 Mar 2026 0949 GMT") to UTC epoch seconds.
+        updated = None
+        updated_str = text("updated")
+        if updated_str:
+            try:
+                tz_abbr = updated_str.split()[-1]
+                timezone = dateutil_tz.gettz(tz_abbr)
+                if timezone is None:
+                    raise ValueError("Unknown timezone abbreviation: " + tz_abbr)
+                dt = dateutil_parser.parse(updated_str, tzinfos={tz_abbr: timezone})
+                updated = dt.astimezone(pytz.UTC).timestamp()
+            except (ValueError, IndexError):
+                logging.warning("HamQSL solar conditions API returned unrecognised timestamp format: " + updated_str)
+
+        # Return the data ready to be put into the solar conditions object.
+        return {
+            "updated": updated,
+            "sfi": int_val("solarflux"),
+            "a_index": int_val("aindex"),
+            "k_index": int_val("kindex"),
+            "xray": text("xray"),
+            "sunspots": int_val("sunspots"),
+            "proton_flux": int_val("protonflux"),
+            "electron_flux": int_val("electonflux"),
+            "aurora": int_val("aurora"),
+            "aurora_latitude": float_val("latdegree"),
+            "solar_wind": float_val("solarwind"),
+            "magnetic_field": float_val("magneticfield"),
+            "geomag_field": text("geomagfield").title()
+            .replace("Vr Quiet", "Very Quiet")
+            .replace("Unsettld", "Unsettled")
+            .replace("Min Strm", "Minor Storm")
+            .replace("Maj Strm", "Major Storm")
+            .replace("Sev Strm", "Severe Storm")
+            .replace("Ext Strm", "Extreme Storm"),
+            "geomag_noise": text("signalnoise"),
+            "hf_conditions": hf_conditions,
+            "vhf_conditions": {
+                "vhf_aurora_northern_hemi": (vhf_map.get(("vhf-aurora", "northern_hemi")) or "").title().replace(
+                    "Lat Aur", "Latitude") or None,
+                "es_2m_europe": vhf_map.get(("E-Skip", "europe")),
+                "es_4m_europe": vhf_map.get(("E-Skip", "europe_4m")),
+                "es_6m_europe": vhf_map.get(("E-Skip", "europe_6m")),
+                "es_2m_na": vhf_map.get(("E-Skip", "north_america")),
+            },
+        }

providers/solarconditions/http_solar_conditions_provider.py

@@ -0,0 +1,69 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider
+
+
+class HTTPSolarConditionsProvider(SolarConditionsProvider):
+    """Generic solar conditions provider for providers that request data via HTTP(S). Subclasses implement
+    _http_response_to_solar_conditions() to parse the specific API response format."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        super().__init__(name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+
+    def start(self):
+        logging.info(
+            "Set up query of " + self.name + " solar conditions API every " + str(self._poll_interval) + " seconds.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval):
+                break
+
+    def _poll(self):
+        try:
+            logging.debug("Polling " + self.name + " solar conditions API...")
+            http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
+                new_data = self._http_response_to_solar_conditions(http_response)
+                self.update_data(new_data)
+
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.debug("Received data from " + self.name + " solar conditions API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} solar conditions API.")
+
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} solar conditions API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} solar conditions API.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP Solar Conditions Provider (" + self.name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _http_response_to_solar_conditions(self, http_response):
+        """Convert an HTTP response into solar conditions data. Returns a dict mapping SolarConditions field
+        names to their new values, or None if the response could not be parsed. Only the fields returned will
+        be updated on the shared SolarConditions object; any fields not included will be left unchanged."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/solarconditions/ionosonde_utils.py

@@ -0,0 +1,37 @@
+from core.constants import BANDS
+
+HF_BANDS = [b for b in BANDS if b.is_ham_hf]
+
+
+def _latest(d) -> float | None:
+    """Given a map where the key is a timestamp and the value is a number represented as a string, find the latest
+    timestamp and return the corresponding value as a float."""
+
+    val = str(d[max(d.keys())]) if d else None
+    return float(val) if (val is not None and val != "None") else None
+
+
+def compute_band_states(fof2_dict, muf_dict, luf_dict):
+    """Compute HF band states from the latest foF2, MUF and LUF values.
+
+    Returns a map where the keys are HF bands and the values are as follows:
+      "Closed" if band frequency is above MUF or below LUF (if known)
+      "Short" if band frequency is >= LUF and < foF2 (good for NVIS)
+      "Long" if band frequency is >= foF2 and < MUF (good for DX)
+    """
+
+    fof2 = _latest(fof2_dict)
+    muf = _latest(muf_dict)
+    luf = _latest(luf_dict) if luf_dict else None
+    if fof2 is None or muf is None:
+        return {}
+    band_states = {}
+    for band in HF_BANDS:
+        freq = band.start_freq / 1_000_000
+        if freq > muf or (luf is not None and freq < luf):
+            band_states[band.name] = "Closed"
+        elif freq < fof2:
+            band_states[band.name] = "Short"
+        else:
+            band_states[band.name] = "Long"
+    return band_states

providers/solarconditions/kc2gprop.py

@@ -0,0 +1,126 @@
+import logging
+from datetime import datetime, timezone, timedelta
+from threading import Thread, Event
+
+import pytz
+import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from providers.solarconditions.ionosonde_utils import compute_band_states
+from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider
+
+POLL_INTERVAL = 900  # 15 minutes
+KC2G_URL = "https://prop.kc2g.com/api/stations.json"
+HISTORY_HOURS = 24
+
+
+class KC2GProp(SolarConditionsProvider):
+    """Solar conditions provider using ionosonde data from prop.kc2g.com. The API returns only the latest reading per
+    station, so this provider polls every 15 minutes and accumulates a 24-hour time series by merging each new reading
+    into the persisted ionosonde_data, producing the same data structure as GIROIonosonde.
+
+    Designed to run alongside GIROIonosonde even though they produce similar data. KC2G is more reliable and is always
+    online, but has fewer stations and does not provide LUF data."""
+
+    def __init__(self, provider_config):
+        super().__init__("KC2G Propagation Data", provider_config)
+        self._thread = None
+        self._stop_event = Event()
+
+    def start(self):
+        logging.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=POLL_INTERVAL):
+                break
+
+    def _poll(self):
+        try:
+            logging.debug("Polling KC2G ionosonde data...")
+            http_response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
+            if not http_response.ok:
+                logging.warning(f"HTTP {http_response.status_code} when calling KG2G ionosonde API.")
+                return
+
+            now = datetime.now(timezone.utc)
+            cutoff_ts = (now - timedelta(hours=HISTORY_HOURS)).timestamp()
+
+            # Start from existing ionosonde_data so the accumulated time series survives across polls and restarts and
+            # stations provided only by GIROIonosonde are not discarded
+            ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
+            updated_count = 0
+
+            for reading in http_response.json():
+                station = reading.get("station", {})
+                ursi = station.get("code")
+                name = station.get("name")
+                if not ursi or not name:
+                    continue
+
+                time_str = reading.get("time")
+                if not time_str:
+                    continue
+                try:
+                    ts = datetime.fromisoformat(time_str)
+                    if ts.tzinfo is None:
+                        ts = ts.replace(tzinfo=timezone.utc)
+                    ts_float = ts.timestamp()
+                except ValueError:
+                    continue
+
+                # Skip readings outside our history window (some stations have months-old data)
+                if ts_float < cutoff_ts:
+                    continue
+
+                fof2_val = reading.get("fof2")
+                muf_val = reading.get("mufd")
+                if fof2_val is None and muf_val is None:
+                    continue
+
+                # Merge this reading into the existing time series for the station.
+                existing = ionosonde_data.get(ursi, {})
+                fof2_dict = dict(existing.get("fof2") or {})
+                muf_dict = dict(existing.get("muf") or {})
+                # LUF is not available from KC2G; carry forward whatever GIRO has written.
+                luf_dict = {float(t): v for t, v in (existing.get("luf") or {}).items()}
+
+                fof2_dict[ts_float] = fof2_val
+                muf_dict[ts_float] = muf_val
+
+                # Trim all series to the 24-hour window.
+                fof2_dict = {t: v for t, v in fof2_dict.items() if t >= cutoff_ts}
+                muf_dict = {t: v for t, v in muf_dict.items() if t >= cutoff_ts}
+                luf_dict = {t: v for t, v in luf_dict.items() if t >= cutoff_ts}
+
+                band_states = compute_band_states(fof2_dict, muf_dict, luf_dict)
+                ionosonde_data[ursi] = {
+                    "ursi": ursi,
+                    "name": name,
+                    "fof2": fof2_dict or None,
+                    "muf": muf_dict or None,
+                    "luf": luf_dict or None,
+                    "band_states": band_states,
+                }
+                updated_count += 1
+
+            self.update_data({"ionosonde_data": ionosonde_data})
+            self.status = "OK"
+            self.last_update_time = datetime.now(pytz.UTC)
+            logging.debug(f"Updated KC2G ionosonde data for {updated_count} stations.")
+
+        except ConnectionError:
+            logging.warning("Connection error when accessing KC2G ionosonde API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing KC2G ionosonde API.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in KC2G ionosonde data provider")
+            self._stop_event.wait(timeout=1)

providers/solarconditions/noaa3dayforecast.py

@@ -0,0 +1,174 @@
+import logging
+import re
+from datetime import datetime, timezone
+
+from providers.solarconditions.http_solar_conditions_provider import HTTPSolarConditionsProvider
+
+POLL_INTERVAL = 10800  # Every 3 hours
+URL = "https://services.swpc.noaa.gov/text/3-day-forecast.txt"
+
+
+class NOAA3dayForecast(HTTPSolarConditionsProvider):
+    """Solar conditions provider using the NOAA 3-day forecast text file. Parses the NOAA forecast and populates
+    corresponding fields in the solar conditions object.."""
+
+    def __init__(self, provider_config):
+        super().__init__("NOAA 3-day Forecast", provider_config, URL, POLL_INTERVAL)
+
+    @staticmethod
+    def _parse_percentage_table(lines, section_header, year):
+        """Find and parse a forecast table using percentages, identified by section_header. This is common to the lookup
+        of the solar storm and radio blackout forecast parsing."""
+
+        start_idx = None
+        for i, line in enumerate(lines):
+            if section_header in line:
+                start_idx = i
+                break
+        if start_idx is None:
+            logging.warning(f"NOAA 3-day forecast: could not find '{section_header}' section")
+            return None
+
+        # Find the date header line by scanning the next few lines for month & day patterns
+        date_header_idx = None
+        for j in range(start_idx + 1, min(start_idx + 6, len(lines))):
+            if re.search(r'[A-Za-z]{3}\s+\d{2}', lines[j]):
+                date_header_idx = j
+                break
+        if date_header_idx is None:
+            logging.warning(f"NOAA 3-day forecast: could not find date header after '{section_header}'")
+            return None
+        date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', lines[date_header_idx])
+        if not date_matches:
+            logging.warning(f"NOAA 3-day forecast: no dates in header: {lines[date_header_idx]}")
+            return None
+
+        # Figure out the date based on the line found
+        column_timestamps = []
+        for month_str, day_str in date_matches:
+            try:
+                dt = datetime.strptime(f"{day_str} {month_str} {year}", "%d %b %Y").replace(tzinfo=timezone.utc)
+                column_timestamps.append(dt.timestamp())
+            except ValueError:
+                logging.warning(f"NOAA 3-day forecast: could not parse date: {month_str} {day_str} {year}")
+                return None
+
+        # Parse data rows. Each non-empty line should have a text label followed by percentage values
+        result = {}
+        for line in lines[date_header_idx + 1:]:
+            line_stripped = line.strip()
+            if not line_stripped:
+                if result:
+                    break
+                continue
+            pct_matches = list(re.finditer(r'\b(\d+)%', line_stripped))
+            if not pct_matches:
+                if result:
+                    break
+                continue
+
+            # Row label is everything before the first percentage value
+            row_label = line_stripped[:line_stripped.index(pct_matches[0].group())].strip()
+            row_data = {}
+            for j, match in enumerate(pct_matches):
+                if j >= len(column_timestamps):
+                    break
+                row_data[column_timestamps[j]] = int(match.group(1))
+            if row_data:
+                result[row_label] = row_data
+
+        return result if result else None
+
+    def _http_response_to_solar_conditions(self, http_response):
+        lines = http_response.text.splitlines()
+
+        # Find the "NOAA Kp index breakdown" section header
+        start_idx = None
+        for i, line in enumerate(lines):
+            if "NOAA Kp index breakdown" in line:
+                start_idx = i
+                break
+        if start_idx is None:
+            logging.warning("NOAA K-index forecast: could not find 'NOAA Kp index breakdown' section")
+            return None
+
+        # Extract the year from the header line, e.g. "NOAA Kp index breakdown Apr 2-Apr 4, 2026"
+        header_line = lines[start_idx]
+        year_match = re.search(r'\b(\d{4})\b', header_line)
+        if not year_match:
+            logging.warning("NOAA K-index forecast: could not extract year from: " + header_line)
+            return None
+        year = int(year_match.group(1))
+
+        # Parse the column date headers on the next line, e.g. "              Apr 02     Apr 03     Apr 04"
+        if start_idx + 1 >= len(lines):
+            logging.warning("NOAA K-index forecast: missing date header line")
+            return None
+
+        date_header_line = lines[start_idx + 2]
+        date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', date_header_line)
+        if not date_matches:
+            logging.warning("NOAA K-index forecast: could not parse date headers from: " + date_header_line)
+            return None
+
+        column_dates = []
+        for month_str, day_str in date_matches:
+            try:
+                column_dates.append(datetime.strptime(f"{day_str} {month_str} {year}", "%d %b %Y").date())
+            except ValueError:
+                logging.warning(f"NOAA K-index forecast: could not parse date: {month_str} {day_str} {year}")
+                return None
+
+        # Parse each data row, e.g. "00-03UT          2.00          3.00          2.00"
+        k_index_forecast = {}
+        for line in lines[start_idx + 3:]:
+            time_match = re.match(r'^(\d{2})-(\d{2})UT\s+(.*)', line.strip())
+            if not time_match:
+                if k_index_forecast:
+                    break
+                continue
+
+            start_hour = int(time_match.group(1))
+            # Split on 2 or more spaces so that e.g. "5.67 (G2)" stays as one token per column
+            raw_values = re.split(r' {2,}', time_match.group(3).strip())
+
+            for i, val in enumerate(raw_values):
+                if i >= len(column_dates):
+                    break
+                # Take only the leading numeric part, discarding any bracketed section
+                try:
+                    kp = float(val.split()[0])
+                except (ValueError, IndexError):
+                    continue
+
+                date = column_dates[i]
+                start_dt = datetime(date.year, date.month, date.day, start_hour, 0, 0, tzinfo=timezone.utc)
+
+                # Key the data dict by start time
+                key = start_dt.timestamp()
+                k_index_forecast[key] = kp
+
+        if not k_index_forecast:
+            logging.warning("NOAA K-index forecast: no data rows parsed")
+            return None
+
+        # Parse Solar Radiation Storm Forecast (single row: "S1 or greater")
+        solar_storm_forecast = None
+        radiation_table = self._parse_percentage_table(lines, "Solar Radiation Storm Forecast", year)
+        if radiation_table:
+            solar_storm_forecast = radiation_table.get("S1 or greater")
+
+        # Parse Radio Blackout Forecast (two rows: "R1-R2" and "R3 or greater")
+        blackout_forecast_r1r2 = None
+        blackout_forecast_r3_or_greater = None
+        blackout_table = self._parse_percentage_table(lines, "Radio Blackout Forecast", year)
+        if blackout_table:
+            blackout_forecast_r1r2 = blackout_table.get("R1-R2")
+            blackout_forecast_r3_or_greater = blackout_table.get("R3 or greater")
+
+        return {
+            "k_index_forecast": k_index_forecast,
+            "solar_storm_forecast": solar_storm_forecast,
+            "blackout_forecast_r1r2": blackout_forecast_r1r2,
+            "blackout_forecast_r3_or_greater": blackout_forecast_r3_or_greater,
+        }

providers/solarconditions/solar_conditions_provider.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+
+class SolarConditionsProvider:
+    """Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
+     propagation data."""
+
+    def __init__(self, name, provider_config):
+        """Constructor"""
+
+        self.name = name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+        self._solar_conditions = DATA_STORE.solar_conditions
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def update_data(self, new_data):
+        """Update the solar conditions object with new data"""
+
+        if new_data:
+            for key, value in new_data.items():
+                if hasattr(self._solar_conditions, key):
+                    setattr(self._solar_conditions, key, value)
+            self._solar_conditions.infer_descriptions()

providers/spot/aprsis.py

@@ -0,0 +1,62 @@
+import logging
+from datetime import datetime
+from threading import Thread
+
+import aprslib
+import pytz
+
+from core.config import SERVER_OWNER_CALLSIGN
+from data.spot import Spot
+from providers.spot.spot_provider import SpotProvider
+
+
+class APRSIS(SpotProvider):
+    """Spot provider for the APRS-IS."""
+
+    def __init__(self, provider_config):
+        super().__init__("APRS-IS", provider_config)
+        self._thread = Thread(target=self._connect)
+        self._thread.daemon = True
+        self._aprsis = None
+
+    def start(self):
+        self._thread.start()
+
+    def _connect(self):
+        self._aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN)
+        self.status = "Connecting"
+        logging.info("APRS-IS connecting...")
+        self._aprsis.connect()
+        self._aprsis.consumer(self._handle)
+        logging.info("APRS-IS connected.")
+
+    def stop(self):
+        self.status = "Shutting down"
+        self._aprsis.close()
+        self._thread.join()
+
+    def _handle(self, data):
+        # Split SSID in "from" call and store separately
+        from_parts = str(data["from"]).split("-")
+        dx_call = from_parts[0].upper()
+        dx_ssid = from_parts[1].upper() if len(from_parts) > 1 else None
+        via_parts = str(data["via"]).split("-")
+        de_call = via_parts[0].upper()
+        de_ssid = via_parts[1].upper() if len(via_parts) > 1 else None
+        spot = Spot(source="APRS-IS",
+                    dx_call=dx_call,
+                    dx_ssid=dx_ssid,
+                    de_call=de_call,
+                    de_ssid=de_ssid,
+                    comment=str(data["comment"]) if "comment" in data else None,
+                    dx_latitude=float(data["latitude"]) if "latitude" in data else None,
+                    dx_longitude=float(data["longitude"]) if "longitude" in data else None,
+                    time=datetime.now(
+                        pytz.UTC).timestamp())  # APRS-IS spots are live so we can assume spot time is "now"
+
+        # Add to our list
+        self._submit(spot)
+
+        self.status = "OK"
+        self.last_update_time = datetime.now(pytz.UTC)
+        logging.debug("Data received from APRS-IS.")

providers/spot/dxcluster.py

@@ -0,0 +1,106 @@
+import logging
+import re
+from datetime import datetime
+from threading import Thread
+from time import sleep
+
+import pytz
+import telnetlib3
+
+from core.config import SERVER_OWNER_CALLSIGN
+from data.spot import Spot
+from providers.spot.spot_provider import SpotProvider
+
+
+class DXCluster(SpotProvider):
+    """Spot provider for a DX Cluster. Hostname, port, login_prompt, login_callsign and allow_rbn_spots are provided in config.
+    See config-example.yml for examples."""
+
+    _LINE_PATTERN_EXCLUDE_RBN = re.compile(
+        r"^DX de ([a-z0-9/]+):\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
+        re.IGNORECASE)
+    _LINE_PATTERN_ALLOW_RBN = re.compile(
+        r"^DX de ([a-z0-9/]+)-?#?:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
+        re.IGNORECASE)
+
+    def __init__(self, provider_config):
+        """Constructor requires hostname and port"""
+
+        name = provider_config["name"] if "name" in provider_config else "Cluster"
+        super().__init__(name, provider_config)
+        self._hostname = provider_config["host"]
+        self._port = provider_config["port"]
+        self._login_prompt = provider_config["login_prompt"] if "login_prompt" in provider_config else "login:"
+        self._login_callsign = provider_config[
+            "login_callsign"] if "login_callsign" in provider_config else SERVER_OWNER_CALLSIGN
+        self._allow_rbn_spots = provider_config["allow_rbn_spots"] if "allow_rbn_spots" in provider_config else False
+        self._spot_line_pattern = self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
+        self._telnet = None
+        self._thread = Thread(target=self._handle)
+        self._thread.daemon = True
+        self._running = True
+
+    def start(self):
+        self._thread.start()
+
+    def stop(self):
+        self._running = False
+        self._telnet.close()
+        self._thread.join()
+
+    def _handle(self):
+        while self._running:
+            connected = False
+            while not connected and self._running:
+                try:
+                    self.status = "Connecting"
+                    logging.info("DX Cluster " + self._hostname + " connecting...")
+                    self._telnet = telnetlib3.Telnet(self._hostname, self._port)
+                    self._telnet.read_until(self._login_prompt.encode("latin-1"))
+                    self._telnet.write((self._login_callsign + "\n").encode("latin-1"))
+                    connected = True
+                    logging.info("DX Cluster " + self._hostname + " connected.")
+                except ConnectionRefusedError:
+                    self.status = "Error"
+                    logging.warning("Connection refused to DX cluster " + self._hostname)
+                    sleep(300)
+                except Exception:
+                    self.status = "Error"
+                    logging.exception("Exception while connecting to DX Cluster Provider (" + self._hostname + ").")
+                    sleep(5)
+
+            self.status = "Waiting for Data"
+            while connected and self._running:
+                try:
+                    # Check new telnet info against regular expression
+                    telnet_output = self._telnet.read_until("\n".encode("latin-1"))
+                    match = self._spot_line_pattern.match(telnet_output.decode("latin-1"))
+                    if match:
+                        spot_time = datetime.strptime(match.group(5), "%H%MZ")
+                        spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(),
+                                                         tzinfo=pytz.UTC)
+                        spot = Spot(source=self.name,
+                                    dx_call=match.group(3),
+                                    de_call=match.group(1),
+                                    freq=float(match.group(2)) * 1000,
+                                    comment=match.group(4).strip(),
+                                    time=spot_datetime.timestamp())
+
+                        # Add to our list
+                        self._submit(spot)
+
+                        self.status = "OK"
+                        self.last_update_time = datetime.now(pytz.UTC)
+                        logging.debug("Data received from DX Cluster " + self._hostname + ".")
+
+                except Exception:
+                    connected = False
+                    if self._running:
+                        self.status = "Error"
+                        logging.exception("Exception in DX Cluster Provider (" + self._hostname + ")")
+                        sleep(5)
+                    else:
+                        logging.info("DX Cluster " + self._hostname + " shutting down...")
+                        self.status = "Shutting down"
+
+            self.status = "Disconnected"

providers/spot/gma.py

@@ -0,0 +1,115 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class GMA(HTTPSpotProvider):
+    """Spot provider for General Mountain Activity"""
+
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://www.gma.rocks/api/spots/25/"
+    # GMA spots don't contain the details of the programme they are for, we need a separate lookup for that
+    REF_INFO_URL_ROOT = "https://www.gma.rocks/api/ref/?"
+
+    def __init__(self, provider_config):
+        # Ensure there is an API key in our config, and set up the query URL using it. If no key is provided,
+        # disable this spot provider.
+        self._api_key = provider_config.get("api-key", "")
+        if self._api_key == "":
+            provider_config["enabled"] = False
+            logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
+        self._url_data_cache = URLDataCache("GMA")
+
+        super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self._api_key, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_spots(self, http_response):
+        new_spots = []
+        # Iterate through source data
+        if "RCD" in http_response.json():
+            for source_spot in http_response.json()["RCD"]:
+                # Convert to our spot format
+                spot = Spot(source=self.name,
+                            dx_call=source_spot["ACTIVATOR"].upper(),
+                            de_call=source_spot["SPOTTER"].upper(),
+                            # Seen GMA spots with no frequency or with "QRT" in this field
+                            freq=float(source_spot["QRG"]) * 1000 if (
+                                    source_spot["QRG"] != "" and source_spot["QRG"] != "QRT") else None,
+                            # Filter out some weird mode strings
+                            mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
+                            comment=source_spot["TEXT"],
+                            sig_refs=[SIGRef(id=source_spot["REF"], sig="", name=source_spot["NAME"])],
+                            time=datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
+                                tzinfo=pytz.UTC).timestamp(),
+                            # Seen GMA spots with no (or empty) lat/lon
+                            dx_latitude=float(source_spot["LAT"]) if (
+                                    source_spot["LAT"] and source_spot["LAT"] != "") else None,
+                            dx_longitude=float(source_spot["LON"]) if (
+                                    source_spot["LON"] and source_spot["LON"] != "") else None,
+                            qrt=source_spot["QRG"] == "QRT")
+
+                # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
+                if "REF" in source_spot:
+                    try:
+                        ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
+                                                          headers=HTTP_HEADERS)
+                        # Sometimes this is blank even if it's a 200 response, so handle that
+                        if ref_response.ok and ref_response.text is not None and ref_response.text != "":
+                            ref_info = ref_response.json()
+                            # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
+                            # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
+                            # and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry
+                            # to determine if it's a SOTA summit.
+                            if spot.sig_refs and "reftype" in ref_info and ref_info["reftype"] not in ["POTA",
+                                                                                                       "WWFF"] and (
+                                    ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info[
+                                "sota"] == ""):
+                                match ref_info["reftype"]:
+                                    case "Summit":
+                                        spot.sig_refs[0].sig = "GMA"
+                                        spot.sig = "GMA"
+                                    case "IOTA Island":
+                                        spot.sig_refs[0].sig = "IOTA"
+                                        spot.sig = "IOTA"
+                                    case "Lighthouse (ILLW)":
+                                        spot.sig_refs[0].sig = "ILLW"
+                                        spot.sig = "ILLW"
+                                    case "Lighthouse (ARLHS)":
+                                        spot.sig_refs[0].sig = "ARLHS"
+                                        spot.sig = "ARLHS"
+                                    case "Castle":
+                                        spot.sig_refs[0].sig = "WCA"
+                                        spot.sig = "WCA"
+                                    case "Mill":
+                                        spot.sig_refs[0].sig = "MOTA"
+                                        spot.sig = "MOTA"
+                                    case _:
+                                        logging.warning("GMA spot found with ref type " + ref_info[
+                                            "reftype"] + ", developer needs to add support for this!")
+                                        spot.sig_refs[0].sig = ref_info["reftype"]
+                                        spot.sig = ref_info["reftype"]
+
+                                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point;
+                                # other code will do that for us.
+                                new_spots.append(spot)
+
+                        elif not ref_response.from_cache:
+                            if not ref_response.ok:
+                                logging.warning(
+                                    f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot["REF"]}")
+                            else:
+                                logging.warning(
+                                    f"GMA API returned a malformed response when looking up ref {source_spot["REF"]}")
+                    except:
+                        logging.warning("Exception when looking up " + self.REF_INFO_URL_ROOT + source_spot[
+                            "REF"] + ", ignoring this spot for now")
+        else:
+            logging.warning(f"The GMA API returned an unexpected response (HTTP {http_response.status_code}).")
+
+        return new_spots

providers/spot/hema.py

@@ -0,0 +1,75 @@
+import logging
+import re
+from datetime import datetime
+
+import pytz
+import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class HEMA(HTTPSpotProvider):
+    """Spot provider for HuMPs Excluding Marilyns Award"""
+
+    POLL_INTERVAL_SEC = 300
+    # HEMA wants us to check for a "spot seed" from the API and see if it's actually changed before querying the main
+    # data API. So it's actually the SPOT_SEED_URL that we pass into the constructor and get the superclass to call on a
+    # timer. The actual data lookup all happens after parsing and checking the seed.
+    SPOT_SEED_URL = "http://www.hema.org.uk/spotSeed.jsp"
+    SPOTS_URL = "http://www.hema.org.uk/spotsMobile.jsp"
+    FREQ_MODE_PATTERN = re.compile("^([\\d.]*) \\((.*)\\)$")
+    SPOTTER_COMMENT_PATTERN = re.compile("^\\((.*)\\) (.*)$")
+
+    def __init__(self, provider_config):
+        super().__init__("HEMA", provider_config, self.SPOT_SEED_URL, self.POLL_INTERVAL_SEC)
+        self._spot_seed = ""
+
+    def _http_response_to_spots(self, http_response):
+        # OK, source data is actually just the spot seed at this point. We'll then go on to fetch real data if we know
+        # this has changed.
+        spot_seed_changed = http_response.text != self._spot_seed
+        self._spot_seed = http_response.text
+
+        new_spots = []
+        # OK, if the spot seed actually changed, now we make the real request for data.
+        if spot_seed_changed:
+            try:
+                source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30))
+                source_data_items = source_data.text.split("=")
+                # Iterate through source data items.
+                for source_spot in source_data_items:
+                    spot_items = source_spot.split(";")
+                    # Any line with less than 9 items is not a proper spot line
+                    if len(spot_items) >= 9:
+                        # Fiddle with some data to extract bits we need. Freq/mode and spotter/comment come in combined fields.
+                        freq_mode_match = re.search(self.FREQ_MODE_PATTERN, spot_items[5])
+                        spotter_comment_match = re.search(self.SPOTTER_COMMENT_PATTERN, spot_items[6])
+                        if not freq_mode_match or not spotter_comment_match:
+                            continue
+
+                        # Convert to our spot format
+                        spot = Spot(source=self.name,
+                                    dx_call=spot_items[2].upper(),
+                                    de_call=spotter_comment_match.group(1).upper(),
+                                    freq=float(freq_mode_match.group(1)) * 1000000,
+                                    mode=freq_mode_match.group(2).upper(),
+                                    comment=spotter_comment_match.group(2),
+                                    sig="HEMA",
+                                    sig_refs=[SIGRef(id=spot_items[3].upper(), sig="HEMA", name=spot_items[4])],
+                                    time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(
+                                        tzinfo=pytz.UTC).timestamp(),
+                                    dx_latitude=float(spot_items[7]),
+                                    dx_longitude=float(spot_items[8]))
+
+                        # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other
+                        # code will do that for us.
+                        new_spots.append(spot)
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when accessing HEMA spots API.")
+            except ConnectionError:
+                logging.warning("Connection error when accessing HEMA spots API.")
+        return new_spots

providers/spot/http_spot_provider.py

@@ -0,0 +1,75 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+import requests
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from providers.spot.spot_provider import SpotProvider
+
+
+class HTTPSpotProvider(SpotProvider):
+    """Generic spot provider class for providers that request data via HTTP(S). Just for convenience to avoid code
+    duplication. Subclasses of this query the individual APIs for data."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        super().__init__(name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info("Set up query of " + self.name + " spot API every " + str(self._poll_interval) + " seconds.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval):
+                break
+
+    def _poll(self):
+        try:
+            # Request data from API
+            logging.debug("Polling " + self.name + " spot API...")
+            http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                new_spots = self._http_response_to_spots(http_response)
+                # Submit the new spots for processing. There might not be any spots for the less popular programs.
+                if new_spots:
+                    self._submit_batch(new_spots)
+
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.debug("Received data from " + self.name + " spot API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} spot API.")
+
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} spots API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} spots API.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP Spot Provider (" + self.name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _http_response_to_spots(self, http_response):
+        """Convert an HTTP response returned by the API into spot data. The whole response is provided here so the subclass
+        implementations can check for HTTP status codes if necessary, and handle the response as JSON, XML, text, whatever
+        the API actually provides."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/spot/llota.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class LLOTA(HTTPSpotProvider):
+    """Spot provider for Lagos y Lagunas On the Air"""
+
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://llota.app/api/public/spots"
+
+    def __init__(self, provider_config):
+        super().__init__("LLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_spots(self, http_response):
+        new_spots = []
+        # Iterate through source data
+        for source_spot in http_response.json():
+            # Find the most recent spotter and comment from the history array
+            comment = None
+            spotter = None
+            if "history" in source_spot and len(source_spot["history"]) > 0:
+                comment = str(source_spot["history"][-1]["comment"])
+                spotter = str(source_spot["history"][-1]["spotter_callsign"])
+            # Convert to our spot format
+            spot = Spot(source=self.name,
+                        source_id=source_spot["id"],
+                        dx_call=source_spot["callsign"].upper(),
+                        de_call=spotter.upper() if spotter else None,
+                        freq=float(source_spot["frequency"]) * 1000000,
+                        mode=source_spot["mode"].upper(),
+                        comment=comment,
+                        sig="LLOTA",
+                        sig_refs=[SIGRef(id=source_spot["reference"], sig="LLOTA", name=source_spot["reference_name"])],
+                        time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp())
+
+            # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
+            # that for us.
+            new_spots.append(spot)
+        return new_spots

providers/spot/parksnpeaks.py

@@ -0,0 +1,65 @@
+import logging
+import re
+from datetime import datetime
+
+import pytz
+
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class ParksNPeaks(HTTPSpotProvider):
+    """Spot provider for Parks n Peaks"""
+
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
+    SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
+
+    def __init__(self, provider_config):
+        super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_spots(self, http_response):
+        new_spots = []
+        # Iterate through source data
+        if http_response and http_response != "":
+            for source_spot in http_response.json():
+                # Convert to our spot format
+                spot = Spot(source=self.name,
+                            source_id=source_spot["actID"],
+                            dx_call=source_spot["actCallsign"].upper(),
+                            de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
+                            # typo exists in API
+                            freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "")
+                                       .replace("+/-", "").strip()) * 1000000 if (source_spot["actFreq"] != "") else None,
+                            # Seen PNP spots with empty frequency, and with comma-separated thousands digits
+                            mode=source_spot["actMode"].upper(),
+                            comment=source_spot["actComments"],
+                            time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S").replace(
+                                tzinfo=pytz.UTC).timestamp())
+
+                # Extract a de_call if it's in the comment but not in the "actSpoter" field
+                m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment or "")
+                if not spot.de_call and m:
+                    spot.de_call = str(m.group(1))
+
+                # Record SIG information. Sometimes we get a "SIG" of "QRP", which we ignore as it's not a programme with a
+                # defined set of references
+                sig = source_spot["actClass"].upper()
+                sig_ref = source_spot["actSiteID"]
+                if sig and sig != "" and sig != "QRP" and sig_ref and sig_ref != "":
+                    spot.sig = sig
+                    sig_refs = [SIGRef(id=source_spot["actSiteID"], sig=source_spot["actClass"].upper())]
+                    spot.sig_refs = sig_refs
+
+                    # Free text location is not present in all spots, so only add it if it's set
+                    if "actLocation" in source_spot and source_spot["actLocation"] != "":
+                        sig_refs[0].name = source_spot["actLocation"]
+
+                    # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
+                    if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA"]:
+                        logging.warning("PNP spot found with sig " + sig + ", developer needs to add support for this!")
+
+                # Add new spot to the list
+                new_spots.append(spot)
+        return new_spots

providers/spot/pota.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+
+import pytz
+
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class POTA(HTTPSpotProvider):
+    """Spot provider for Parks on the Air"""
+
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://api.pota.app/spot/activator"
+
+    def __init__(self, provider_config):
+        super().__init__("POTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_spots(self, http_response):
+        new_spots = []
+        # Iterate through source data
+        for source_spot in http_response.json():
+            # Convert to our spot format
+            spot = Spot(source=self.name,
+                        source_id=source_spot["spotId"],
+                        dx_call=source_spot["activator"].upper(),
+                        de_call=source_spot["spotter"].upper(),
+                        freq=float(source_spot["frequency"]) * 1000,
+                        mode=source_spot["mode"].upper(),
+                        comment=source_spot["comments"],
+                        sig="POTA",
+                        sig_refs=[SIGRef(id=source_spot["reference"], sig="POTA", name=source_spot["name"])],
+                        time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(
+                            tzinfo=pytz.UTC).timestamp(),
+                        dx_grid=source_spot["grid6"],
+                        dx_latitude=source_spot["latitude"],
+                        dx_longitude=source_spot["longitude"])
+
+            # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
+            # that for us.
+            new_spots.append(spot)
+        return new_spots

providers/spot/rbn.py

@@ -0,0 +1,93 @@
+import logging
+import re
+from datetime import datetime
+from threading import Thread
+from time import sleep
+
+import pytz
+import telnetlib3
+
+from core.config import SERVER_OWNER_CALLSIGN
+from data.spot import Spot
+from providers.spot.spot_provider import SpotProvider
+
+
+class RBN(SpotProvider):
+    """Spot provider for the Reverse Beacon Network. Connects to a single port, if you want both CW/RTTY (port 7000) and FT8
+    (port 7001) you need to instantiate two copies of this. The port is provided as an argument to the constructor."""
+
+    _LINE_PATTERN = re.compile(
+        r"^DX de ([a-z0-9/]+)-.*:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
+        re.IGNORECASE)
+
+    def __init__(self, provider_config):
+        """Constructor requires port number."""
+
+        name = provider_config["name"] if "name" in provider_config else "RBN"
+        super().__init__(name, provider_config)
+        self._port = provider_config["port"]
+        self._telnet = None
+        self._thread = Thread(target=self._handle)
+        self._thread.daemon = True
+        self._running = True
+
+    def start(self):
+        self._thread.start()
+
+    def stop(self):
+        self._running = False
+        self._telnet.close()
+        self._thread.join()
+
+    def _handle(self):
+        while self._running:
+            connected = False
+            while not connected and self._running:
+                try:
+                    self.status = "Connecting"
+                    logging.info("RBN port " + str(self._port) + " connecting...")
+                    self._telnet = telnetlib3.Telnet("telnet.reversebeacon.net", self._port)
+                    self._telnet.read_until("Please enter your call: ".encode("latin-1"))
+                    self._telnet.write((SERVER_OWNER_CALLSIGN + "\n").encode("latin-1"))
+                    connected = True
+                    logging.info("RBN port " + str(self._port) + " connected.")
+                except Exception:
+                    self.status = "Error"
+                    logging.exception("Exception while connecting to RBN (port " + str(self._port) + ").")
+                    sleep(5)
+
+            self.status = "Waiting for Data"
+            while connected and self._running:
+                try:
+                    # Check new telnet info against regular expression
+                    telnet_output = self._telnet.read_until("\n".encode("latin-1"))
+                    match = self._LINE_PATTERN.match(telnet_output.decode("latin-1"))
+                    if match:
+                        spot_time = datetime.strptime(match.group(5), "%H%MZ")
+                        spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(),
+                                                         tzinfo=pytz.UTC)
+                        spot = Spot(source=self.name,
+                                    dx_call=match.group(3),
+                                    de_call=match.group(1),
+                                    freq=float(match.group(2)) * 1000,
+                                    comment=match.group(4).strip(),
+                                    time=spot_datetime.timestamp())
+
+                        # Add to our list
+                        self._submit(spot)
+
+                        self.status = "OK"
+                        self.last_update_time = datetime.now(pytz.UTC)
+                        logging.debug("Data received from RBN on port " + str(self._port) + ".")
+
+                except Exception:
+                    connected = False
+                    if self._running:
+                        self.status = "Error"
+                        logging.exception("Exception in RBN provider (port " + str(self._port) + ")")
+                        sleep(5)
+                    else:
+                        logging.info("RBN provider (port " + str(self._port) + ") shutting down...")
+                        self.status = "Shutting down"
+
+            self.status = "Disconnected"

providers/spot/sota.py

@@ -0,0 +1,63 @@
+import logging
+from datetime import datetime
+
+import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class SOTA(HTTPSpotProvider):
+    """Spot provider for Summits on the Air"""
+
+    POLL_INTERVAL_SEC = 120
+    # SOTA wants us to check for an "epoch" from the API and see if it's actually changed before querying the main data
+    # APIs. So it's actually the EPOCH_URL that we pass into the constructor and get the superclass to call on a timer.
+    # The actual data lookup all happens after parsing and checking the epoch.
+    EPOCH_URL = "https://api-db2.sota.org.uk/api/spots/epoch"
+    SPOTS_URL = "https://api-db2.sota.org.uk/api/spots/60/all/all"
+
+    def __init__(self, provider_config):
+        super().__init__("SOTA", provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)
+        self._api_epoch = ""
+
+    def _http_response_to_spots(self, http_response):
+        # OK, source data is actually just the epoch at this point. We'll then go on to fetch real data if we know this
+        # has changed.
+        epoch_changed = http_response.text != self._api_epoch
+        self._api_epoch = http_response.text
+
+        new_spots = []
+        # OK, if the epoch actually changed, now we make the real request for data.
+        if epoch_changed:
+            try:
+                source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30)).json()
+                # Iterate through source data
+                for source_spot in source_data:
+                    # Convert to our spot format
+                    spot = Spot(source=self.name,
+                                source_id=source_spot["id"],
+                                dx_call=source_spot["activatorCallsign"].upper(),
+                                dx_name=source_spot["activatorName"],
+                                de_call=source_spot["callsign"].upper(),
+                                freq=(float(source_spot["frequency"]) * 1000000) if (
+                                        source_spot["frequency"] is not None) else None,
+                                # Seen SOTA spots with no frequency!
+                                mode=source_spot["mode"].upper(),
+                                comment=source_spot["comments"],
+                                sig="SOTA",
+                                sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA", name=source_spot["summitName"],
+                                                 activation_score=source_spot["points"])],
+                                time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp())
+
+                    # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
+                    # that for us.
+                    new_spots.append(spot)
+            except ConnectionError:
+                logging.warning("Connection error when accessing SOTA spots API")
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when accessing SOTA spots API.")
+        return new_spots

providers/spot/spot_provider.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+
+class SpotProvider:
+    """Generic spot provider class. Subclasses of this query the individual APIs for data."""
+
+    def __init__(self, name, provider_config):
+        """Constructor"""
+
+        self.name = name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+        self._spots = DATA_STORE.spots
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def _submit_batch(self, spots):
+        """Submit a batch of spots retrieved from the provider. Only spots that are newer than the last spot retrieved
+        by this provider will be added to the spot list, to prevent duplications. Spots passing the check will also have
+        their infer_missing() method called to complete their data set. This is called by the API-querying
+        subclasses on receiving spots."""
+
+        # Sort the batch so that earliest ones go in first. This helps keep the ordering correct when spots are fired
+        # off to SSE listeners.
+        spots = sorted(spots, key=lambda s: (s.time if s and s.time else 0))
+        for spot in spots:
+            if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time:
+                # Fill in any blanks and add to the list
+                spot.infer_missing()
+                self._add_spot(spot)
+        if spots:
+            self.last_spot_time = datetime.fromtimestamp(max(map(lambda s: s.time, spots)), pytz.UTC)
+
+    def _submit(self, spot):
+        """Submit a single spot retrieved from the provider. This will be added to the list regardless of its age. Spots
+        passing the check will also have their infer_missing() method called to complete their data set. This is called by
+        the data streaming subclasses, which can be relied upon not to re-provide old spots."""
+
+        # Fill in any blanks and add to the list
+        spot.infer_missing()
+        self._add_spot(spot)
+        self.last_spot_time = datetime.fromtimestamp(spot.time, pytz.UTC)
+
+    def _add_spot(self, spot):
+        if not spot.expired():
+            self._spots.set(spot.id, spot)
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/spot/sse_spot_provider.py

@@ -0,0 +1,79 @@
+import logging
+from datetime import datetime
+from threading import Thread
+from time import sleep
+
+import pytz
+from requests_sse import EventSource
+
+from core.constants import HTTP_HEADERS
+from providers.spot.spot_provider import SpotProvider
+
+
+class SSESpotProvider(SpotProvider):
+    """Spot provider using Server-Sent Events."""
+
+    def __init__(self, name, provider_config, url):
+        super().__init__(name, provider_config)
+        self._url = url
+        self._event_source = None
+        self._thread = None
+        self._stopped = False
+        self._last_event_id = None
+
+    def start(self):
+        logging.info("Set up SSE connection to " + self.name + " spot API.")
+        self._stopped = False
+        self._thread = Thread(target=self._run)
+        self._thread.daemon = True
+        self._thread.start()
+
+    def stop(self):
+        self._stopped = True
+        if self._event_source:
+            self._event_source.close()
+        if self._thread:
+            self._thread.join()
+
+    def _on_open(self):
+        self.status = "Waiting for Data"
+
+    def _on_error(self):
+        self.status = "Connecting"
+
+    def _run(self):
+        while not self._stopped:
+            try:
+                logging.debug("Connecting to " + self.name + " spot API...")
+                self.status = "Connecting"
+                with EventSource(self._url, headers=HTTP_HEADERS, latest_event_id=self._last_event_id, timeout=30,
+                                 on_open=self._on_open, on_error=self._on_error) as event_source:
+                    self._event_source = event_source
+                    for event in self._event_source:
+                        if event.type == 'message':
+                            try:
+                                self._last_event_id = event.last_event_id
+                                new_spot = self._sse_message_to_spot(event.data)
+                                if new_spot:
+                                    self._submit(new_spot)
+
+                                self.status = "OK"
+                                self.last_update_time = datetime.now(pytz.UTC)
+                                logging.debug("Received data from " + self.name + " spot API.")
+
+                            except Exception:
+                                logging.exception(
+                                    "Exception processing message from SSE Spot Provider (" + self.name + ")")
+
+            except Exception:
+                self.status = "Error"
+                logging.exception("Exception in SSE Spot Provider (" + self.name + ")")
+            else:
+                self.status = "Disconnected"
+            sleep(5)  # Wait before trying to reconnect
+
+    def _sse_message_to_spot(self, message_data):
+        """Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass
+        implementations can handle the message as JSON, XML, text, whatever the API actually provides."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/spot/tiles.py

@@ -0,0 +1,52 @@
+from datetime import datetime
+
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class Tiles(HTTPSpotProvider):
+    """Spot provider for Tiles on the Air"""
+
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/spots?active_hours=24"
+
+    def __init__(self, provider_config):
+        super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_spots(self, http_response):
+        new_spots = []
+        # Iterate through source data
+        for source_spot in http_response.json()["spots"]:
+            # Convert to our spot format
+            spot = Spot(source=self.name,
+                        source_id=source_spot["id"],
+                        dx_call=source_spot["call_sign"].upper(),
+                        # No separate spotter callsign, assume all spots are self-spots
+                        de_call=source_spot["call_sign"].upper(),
+                        freq=float(strip_extra_decimal_points(source_spot["frequency"])) * 1000000,
+                        mode=source_spot["mode"].upper(),
+                        comment=source_spot["notes"],
+                        sig="Tiles",
+                        # Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
+                        # Just take the grid reference itself as the single Tiles SIG reference.
+                        sig_refs=[SIGRef(id=source_spot["maidenhead_grid"], sig="Tiles",
+                                         name=source_spot["maidenhead_grid"])],
+                        time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
+                        dx_grid=source_spot["maidenhead_grid"],
+                        dx_latitude=source_spot["latitude"],
+                        dx_longitude=source_spot["longitude"])
+
+            # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
+            # that for us.
+            new_spots.append(spot)
+        return new_spots
+
+
+# Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles'
+# strange frequency format where we can sometimes have e.g. "14.123.5".
+def strip_extra_decimal_points(s):
+    parts = s.split('.', 1)
+    if len(parts) == 1:
+        return s
+    return parts[0] + '.' + parts[1].replace('.', '')

providers/spot/towers.py

@@ -0,0 +1,41 @@
+import json
+from datetime import datetime
+
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class Towers(HTTPSpotProvider):
+    """Spot provider for Towers on the Air"""
+
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://wwtota.com/api/cluster_live.php"
+
+    def __init__(self, provider_config):
+        super().__init__("Towers", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_spots(self, http_response):
+        new_spots = []
+        response_fixed = http_response.text.replace("\\/", "/")
+        response_json = json.loads(response_fixed)
+
+        # Iterate through source data
+        for source_spot in response_json["spots"]:
+            # Convert to our spot format
+            likely_freq = float(source_spot["freq"]) * 1000
+            if likely_freq < 1000000:
+                likely_freq = likely_freq * 1000
+            spot = Spot(source=self.name,
+                        dx_call=source_spot["call"].upper(),
+                        freq=likely_freq,
+                        comment=source_spot["comment"],
+                        sig="Towers",
+                        sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
+                        time=datetime.strptime(response_json["updated"][:10] + source_spot["time"],
+                                               "%Y-%m-%d%H:%M").timestamp())
+
+            # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
+            # that for us.
+            new_spots.append(spot)
+        return new_spots

providers/spot/ukpacketnet.py

@@ -0,0 +1,94 @@
+import re
+from datetime import datetime
+
+import pytz
+
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class UKPacketNet(HTTPSpotProvider):
+    """Spot provider for UK Packet Radio network API"""
+
+    POLL_INTERVAL_SEC = 600
+    SPOTS_URL = "https://nodes.ukpacketradio.network/api/nodedata"
+
+    def __init__(self, provider_config):
+        super().__init__("UK Packet Net", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_spots(self, http_response):
+        new_spots = []
+        # Iterate through source data
+        nodes = http_response.json()["nodes"]
+        for callsign, node in nodes.items():
+            # The node corresponse to the spotter here. It has an "mheard" section which indicates which nodes it has
+            # recently heard, which will be our "DX". But "mheard" stations are not necessarily over RF, they could be
+            # via the internet, so we also need to look up the "port" on which the station was heard, and check that it
+            # is RF.
+            if "mheard" in node:
+                for heard in node["mheard"]:
+                    heard_port_id = heard["port"]
+                    if "ports" in node:
+                        for listed_port in node["ports"]:
+                            if listed_port["id"] == heard_port_id and listed_port["linkType"] == "RF":
+                                # This is another packet station heard over RF, so we are good to create a Spot object.
+
+                                # First build a "full" comment combining some of the extra info
+                                comment = listed_port["comment"] if "comment" in listed_port else ""
+                                comment = (comment + " " + listed_port["mode"]) if "mode" in listed_port else comment
+                                comment = (comment + " " + listed_port[
+                                    "modulation"]) if "modulation" in listed_port else comment
+                                comment = (comment + " " + str(
+                                    listed_port["baud"]) + " baud") if "baud" in listed_port and listed_port[
+                                    "baud"] > 0 else comment
+
+                                # Get frequency from the comment if it's not set properly in the data structure. This is
+                                # very hacky but a lot of node comments contain their frequency as the first or second
+                                # word of their comment, but not in the proper data structure field.
+                                freq = listed_port["freq"] if "freq" in listed_port and listed_port[
+                                    "freq"] > 0 else None
+                                if not freq and comment:
+                                    possible_freq = comment.split(" ")[0].upper().replace("MHZ", "")
+                                    if re.match(r"^[0-9.]+$",
+                                                possible_freq) and possible_freq != "1200" and possible_freq != "9600":
+                                        freq = float(possible_freq) * 1000000
+                                    if not freq and len(comment.split(" ")) > 1:
+                                        possible_freq = comment.split(" ")[1].upper().replace("MHZ", "")
+                                        if re.match(r"^[0-9.]+$",
+                                                    possible_freq) and possible_freq != "1200" and possible_freq != "9600":
+                                            freq = float(possible_freq) * 1000000
+                                    # Check for a found frequency likely having been in kHz, sorry to all GHz packet folks
+                                    if freq and freq > 1000000000:
+                                        freq = freq / 1000
+
+                                # Now build the spot object
+                                spot = Spot(source=self.name,
+                                            dx_call=heard["callsign"].upper(),
+                                            de_call=node["callsign"].upper(),
+                                            freq=freq,
+                                            mode="PKT",
+                                            comment=comment,
+                                            time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S").replace(
+                                                tzinfo=pytz.UTC).timestamp(),
+                                            de_grid=node["location"]["locator"] if "locator" in node[
+                                                "location"] else None,
+                                            de_latitude=node["location"]["coords"]["lat"],
+                                            de_longitude=node["location"]["coords"]["lon"])
+
+                                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
+                                # that for us.
+                                new_spots.append(spot)
+                                break
+
+        # Now we have a list of every node that heard every other node via RF, as Spothole spots. What each spot doesn't
+        # yet have is a DX lat/lon/grid, because the data doesn't provide the location of the "mheard" stations within
+        # the structure. However, each "heard" station should also be represented in the list somewhere with its own
+        # data, and we can use that to look these up.
+        for spot in new_spots:
+            if spot.dx_call in nodes:
+                spot.dx_grid = nodes[spot.dx_call]["location"]["locator"] if "locator" in nodes[spot.dx_call][
+                    "location"] else None
+                spot.dx_latitude = nodes[spot.dx_call]["location"]["coords"]["lat"]
+                spot.dx_longitude = nodes[spot.dx_call]["location"]["coords"]["lon"]
+
+        return new_spots

providers/spot/websocket_spot_provider.py

@@ -0,0 +1,77 @@
+import logging
+from datetime import datetime
+from threading import Thread
+from time import sleep
+
+import pytz
+from websocket import create_connection
+
+from core.constants import HTTP_HEADERS
+from providers.spot.spot_provider import SpotProvider
+
+
+class WebsocketSpotProvider(SpotProvider):
+    """Spot provider using websockets."""
+
+    def __init__(self, name, provider_config, url):
+        super().__init__(name, provider_config)
+        self._url = url
+        self._ws = None
+        self._thread = None
+        self._stopped = False
+        self._last_event_id = None
+
+    def start(self):
+        logging.info("Set up websocket connection to " + self.name + " spot API.")
+        self._stopped = False
+        self._thread = Thread(target=self._run)
+        self._thread.daemon = True
+        self._thread.start()
+
+    def stop(self):
+        self._stopped = True
+        if self._ws:
+            self._ws.close()
+        if self._thread:
+            self._thread.join()
+
+    def _on_open(self):
+        self.status = "Waiting for Data"
+
+    def _on_error(self):
+        self.status = "Connecting"
+
+    def _run(self):
+        while not self._stopped:
+            try:
+                logging.debug("Connecting to " + self.name + " spot API...")
+                self.status = "Connecting"
+                self._ws = create_connection(self._url, header=HTTP_HEADERS)
+                self.status = "Connected"
+                data = self._ws.recv()
+                if data:
+                    try:
+                        new_spot = self._ws_message_to_spot(data)
+                        if new_spot:
+                            self._submit(new_spot)
+
+                        self.status = "OK"
+                        self.last_update_time = datetime.now(pytz.UTC)
+                        logging.debug("Received data from " + self.name + " spot API.")
+
+                    except Exception:
+                        logging.exception(
+                            "Exception processing message from Websocket Spot Provider (" + self.name + ")")
+
+            except Exception as e:
+                self.status = "Error"
+                logging.exception("Exception in Websocket Spot Provider (" + self.name + ")", e)
+            else:
+                self.status = "Disconnected"
+            sleep(5)  # Wait before trying to reconnect
+
+    def _ws_message_to_spot(self, b):
+        """Convert a WS message received from the API into a spot. The exact message data (in bytes) is provided here so the
+        subclass implementations can handle the message as string, JSON, XML, whatever the API actually provides."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/spot/wota.py

@@ -0,0 +1,81 @@
+import logging
+import re
+from datetime import datetime
+from typing import cast
+
+import pytz
+from rss_parser import Parser
+from rss_parser.models.rss import RSS
+
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class WOTA(HTTPSpotProvider):
+    """Spot provider for Wainwrights on the Air"""
+
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://www.wota.org.uk/spots_rss.php"
+    LIST_URL = "https://www.wota.org.uk/mapping/data/summits.json"
+    RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
+
+    def __init__(self, provider_config):
+        super().__init__("WOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_spots(self, http_response):
+        new_spots = []
+        rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
+        # Iterate through source data
+        for source_spot in rss.channel.items:
+
+            try:
+                # Reject GUID missing or zero
+                if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0":
+                    continue
+
+                # Pick apart the title
+                title_split = source_spot.title.split(" on ")
+                dx_call = title_split[0]
+                ref = None
+                ref_name = None
+                if len(title_split) > 1:
+                    ref_split = title_split[1].split(" - ")
+                    ref = str(ref_split[0])
+                    if len(ref_split) > 1:
+                        ref_name = str(ref_split[1])
+
+                # Pick apart the description
+                desc_split = source_spot.description.split(". ")
+                freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
+                freq_mode_split = re.split(r'[\-\s]+', freq_mode)
+                freq_hz = float(freq_mode_split[0].replace("'", ".")) * 1000000
+                mode = None
+                if len(freq_mode_split) > 1:
+                    mode = freq_mode_split[1].upper()
+
+                comment = None
+                if len(desc_split) > 1:
+                    comment = desc_split[1].strip()
+                spotter = None
+                if len(desc_split) > 2:
+                    spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").upper().strip()
+
+                time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+
+                # Convert to our spot format
+                spot = Spot(source=self.name,
+                            source_id=source_spot.guid.content,
+                            dx_call=dx_call,
+                            de_call=spotter,
+                            freq=freq_hz,
+                            mode=mode,
+                            comment=comment,
+                            sig="WOTA",
+                            sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
+                            time=time.timestamp())
+
+                new_spots.append(spot)
+            except Exception as e:
+                logging.error("Exception parsing WOTA spot", e)
+        return new_spots

providers/spot/wwbota.py

@@ -0,0 +1,43 @@
+import json
+from datetime import datetime
+
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.sse_spot_provider import SSESpotProvider
+
+
+class WWBOTA(SSESpotProvider):
+    """Spot provider for Worldwide Bunkers on the Air"""
+
+    SPOTS_URL = "https://api.wwbota.net/spots/"
+
+    def __init__(self, provider_config):
+        super().__init__("WWBOTA", provider_config, self.SPOTS_URL)
+
+    def _sse_message_to_spot(self, message_data):
+        source_spot = json.loads(message_data)
+        # Convert to our spot format. First we unpack references, because WWBOTA spots can have more than one for
+        # n-fer activations.
+        refs = []
+        for ref in source_spot["references"]:
+            sigref = SIGRef(id=ref["reference"], sig="WWBOTA", name=ref["name"])
+            refs.append(sigref)
+
+        spot = Spot(source=self.name,
+                    dx_call=source_spot["call"].upper(),
+                    de_call=source_spot["spotter"].upper(),
+                    freq=float(source_spot["freq"]) * 1000000,
+                    mode=source_spot["mode"].upper(),
+                    comment=source_spot["comment"],
+                    sig="WWBOTA",
+                    sig_refs=refs,
+                    time=datetime.fromisoformat(source_spot["time"].replace("Z", "+00:00")).timestamp(),
+                    # WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
+                    # now, we will just pick the first one to use as our grid, latitude and longitude.
+                    dx_grid=source_spot["references"][0]["locator"],
+                    dx_latitude=source_spot["references"][0]["lat"],
+                    dx_longitude=source_spot["references"][0]["long"],
+                    qrt=source_spot["type"] == "QRT")
+
+        # WWBOTA does support a special "Test" spot type, we need to avoid adding that.
+        return spot if source_spot["type"] != "Test" else None

providers/spot/wwff.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+
+import pytz
+
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class WWFF(HTTPSpotProvider):
+    """Spot provider for Worldwide Flora & Fauna"""
+
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://spots.wwff.co/static/spots.json"
+
+    def __init__(self, provider_config):
+        super().__init__("WWFF", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_spots(self, http_response):
+        new_spots = []
+        # Iterate through source data
+        for source_spot in http_response.json():
+            # Convert to our spot format
+            spot = Spot(source=self.name,
+                        source_id=source_spot["id"],
+                        dx_call=source_spot["activator"].upper(),
+                        de_call=source_spot["spotter"].upper(),
+                        freq=float(source_spot["frequency_khz"]) * 1000,
+                        mode=source_spot["mode"].upper(),
+                        comment=source_spot["remarks"],
+                        sig="WWFF",
+                        sig_refs=[SIGRef(id=source_spot["reference"], sig="WWFF", name=source_spot["reference_name"])],
+                        time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
+                        dx_latitude=source_spot["latitude"],
+                        dx_longitude=source_spot["longitude"])
+
+            # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
+            # that for us.
+            new_spots.append(spot)
+        return new_spots

providers/spot/xota.py

@@ -0,0 +1,42 @@
+import json
+from datetime import datetime
+
+import pytz
+
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.websocket_spot_provider import WebsocketSpotProvider
+
+
+class XOTA(WebsocketSpotProvider):
+    """Spot provider for servers based on the "xOTA" software at https://github.com/nischu/xOTA/
+    The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG which we can
+    then use for lookups. This functionality is implemented for Toilets on the Air events, of which there are
+    several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA, which is why we also
+    provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3 might become
+    "C3 T-01". This allows us to provide location lookups for TOTA at several conferences."""
+
+    LOCATION_DATA = {}
+    SIG = None
+
+    def __init__(self, provider_config):
+        name = provider_config["name"] if "name" in provider_config else "xOTA"
+        super().__init__(name, provider_config, provider_config["url"])
+        self.SIG = str(provider_config["sig"]) if "sig" in provider_config else None
+        self._sig_ref_prefix = str(provider_config["sig-ref-prefix"]) if "sig-ref-prefix" in provider_config else ""
+
+    def _ws_message_to_spot(self, b):
+        string = b.decode("utf-8")
+        source_spot = json.loads(string)
+        ref_id = self._sig_ref_prefix + " " + source_spot["reference"]["title"]
+        spot = Spot(source=self.name,
+                    source_id=source_spot["id"],
+                    dx_call=source_spot["stationCallSign"].upper(),
+                    freq=float(source_spot["freq"]) * 1000,
+                    mode=source_spot["mode"].upper(),
+                    sig=self.SIG,
+                    sig_refs=[
+                        SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"])],
+                    time=datetime.now(pytz.UTC).timestamp(),
+                    qrt=source_spot["state"] != "active")
+        return spot

providers/spot/zlota.py

@@ -0,0 +1,43 @@
+from datetime import datetime
+
+import pytz
+
+from data.sig_ref import SIGRef
+from data.spot import Spot
+from providers.spot.http_spot_provider import HTTPSpotProvider
+
+
+class ZLOTA(HTTPSpotProvider):
+    """Spot provider for ZLOTA"""
+
+    POLL_INTERVAL_SEC = 120
+    SPOTS_URL = "https://ontheair.nz/api/spots?zlota_only=true"
+    LIST_URL = "https://ontheair.nz/assets/assets.json"
+
+    def __init__(self, provider_config):
+        super().__init__("ZLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_spots(self, http_response):
+        new_spots = []
+        # Iterate through source data
+        for source_spot in http_response.json():
+            # Frequency is often inconsistent as to whether it's in Hz or kHz. Make a guess.
+            freq_hz = float(source_spot["frequency"])
+            if freq_hz < 1000000:
+                freq_hz = freq_hz * 1000
+
+            # Convert to our spot format
+            spot = Spot(source=self.name,
+                        source_id=source_spot["id"],
+                        dx_call=source_spot["activator"].upper(),
+                        de_call=source_spot["spotter"].upper(),
+                        freq=freq_hz,
+                        mode=source_spot["mode"].upper().strip(),
+                        comment=source_spot["comments"],
+                        sig="ZLOTA",
+                        sig_refs=[SIGRef(id=source_spot["reference"], sig="ZLOTA", name=source_spot["name"])],
+                        time=datetime.fromisoformat(source_spot["referenced_time"].replace("Z", "+00:00")).astimezone(
+                            pytz.UTC).timestamp())
+
+            new_spots.append(spot)
+        return new_spots

providers/staticdata/cqzonedata.py

@@ -0,0 +1,36 @@
+import json
+import logging
+
+import geopandas
+from shapely import prepare
+
+from core.data_store import DATA_STORE
+from providers.staticdata.local_file_static_data_provider import LocalFileStaticDataProvider
+
+
+class CQZoneData(LocalFileStaticDataProvider):
+    """Static data provider for CQ zone geodata."""
+
+    PATH = "datafiles/cqzones.geojson"
+
+    def __init__(self, provider_config):
+        super().__init__("CQ Zone Data", provider_config, self.PATH)
+
+    def _load_data(self, path):
+        try:
+            with open(path) as f:
+                cq_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
+            for idx in cq_zone_data.index:
+                prepare(cq_zone_data.at[idx, 'geometry'])
+
+                # Bail out if a stop has been requested, i.e. the program is shutting down - no need to prepare the rest
+                # of the data in this case
+                if self.stop:
+                    break
+
+            DATA_STORE.cq_zone_data = cq_zone_data
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading CQ zone data.", e, exc_info=True)
+            return False

providers/staticdata/file_download_static_data_provider.py

@@ -0,0 +1,77 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from providers.staticdata.static_data_provider import StaticDataProvider
+
+
+class FileDownloadStaticDataProvider(StaticDataProvider):
+    """Generic static reference data provider class for providers that fetch their data from the web by downloading a
+     file."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
+        super().__init__(name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+        self._url_data_cache = URLDataCache("staticdata_" + name)
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info(
+            "Set up query of " + self.name + " static reference data every " + str(self._poll_interval) + " days.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
+                break
+
+    def _poll(self):
+        try:
+            # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
+            logging.debug("Downloading " + self.name + " static reference data...")
+            http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                ok = self._handle_http_response(http_response)
+                if ok:
+                    self.status = "OK"
+                    self.last_update_time = datetime.now(pytz.UTC)
+                    logging.info("Updated static reference data for " + self.name)
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when downloading static reference data for {self.name}.")
+
+        except ConnectionError:
+            self.status = "Error"
+            logging.warning(f"Connection error when downloading static reference data for {self.name}.")
+        except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
+            logging.warning(f"Timeout when downloading static reference data for {self.name}.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP static reference data provider (" + self.name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _handle_http_response(self, http_response):
+        """Handle an HTTP response returned by the server and load the data from it. Return true if successful,
+         false otherwise."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/staticdata/ituzonedata.py

@@ -0,0 +1,36 @@
+import json
+import logging
+
+import geopandas
+from shapely import prepare
+
+from core.data_store import DATA_STORE
+from providers.staticdata.local_file_static_data_provider import LocalFileStaticDataProvider
+
+
+class ITUZoneData(LocalFileStaticDataProvider):
+    """Static data provider for ITU zone geodata."""
+
+    PATH = "datafiles/ituzones.geojson"
+
+    def __init__(self, provider_config):
+        super().__init__("ITU Zone Data", provider_config, self.PATH)
+
+    def _load_data(self, path):
+        try:
+            with open(path) as f:
+                itu_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
+            for idx in itu_zone_data.index:
+                prepare(itu_zone_data.at[idx, 'geometry'])
+
+                # Bail out if a stop has been requested, i.e. the program is shutting down - no need to prepare the rest
+                # of the data in this case
+                if self.stop:
+                    break
+
+            DATA_STORE.itu_zone_data = itu_zone_data
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading ITU zone data.", e, exc_info=True)
+            return False

providers/staticdata/k0swe.py

@@ -0,0 +1,46 @@
+import logging
+
+from core.data_store import DATA_STORE
+from providers.staticdata.file_download_static_data_provider import FileDownloadStaticDataProvider
+
+
+class K0SWE(FileDownloadStaticDataProvider):
+    """Static data provider for K0SWE's dxcc.json, which provides callsign regex to DXCC entity mapping, plus DXCC to
+    continent, flag emoji etc."""
+
+    POLL_INTERVAL_DAYS = 7
+    DATA_URL = "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json"
+
+    def __init__(self, provider_config):
+        super().__init__("K0SWE DXCC JSON", provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _handle_http_response(self, http_response):
+        try:
+            dxcc_list = http_response.json()["dxcc"]
+            # Reformat as a map for to place in the data store
+            dxcc_map = {}
+            for dxcc in dxcc_list:
+                dxcc_map[dxcc["entityCode"]] = dxcc
+
+                # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
+                # of the data in this case
+                if self._stop_event.is_set():
+                    break
+
+            # Add to data store
+            for k, v in dxcc_map.items():
+                DATA_STORE.dxcc_data[k] = v
+
+                if self._stop_event.is_set():
+                    break
+
+            # Regenerate in-memory regex-to-DXCC-entity-code map.
+            DATA_STORE.regenerate_call_regex_to_dxcc_entity_map()
+
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading K0SWE dxcc.json.", e, exc_info=True)
+            return False
+
+

providers/staticdata/local_file_static_data_provider.py

@@ -0,0 +1,38 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from providers.staticdata.static_data_provider import StaticDataProvider
+
+
+class LocalFileStaticDataProvider(StaticDataProvider):
+    """Generic static reference data provider class for providers that fetch their data from a local file on startup."""
+
+    def __init__(self, name, provider_config, path):
+        super().__init__(name, provider_config)
+        self._path = path
+        self._stop = False
+
+    def start(self):
+        logging.debug("Loading " + self.name + " static reference data from file.")
+        try:
+            ok = self._load_data(self._path)
+            if ok:
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.info("Updated static reference data for " + self.name)
+            else:
+                self.status = "Error"
+                logging.error("Failed to load data for " + self.name)
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception in local file Static Data Provider (" + self.name + ")", e, exc_info=True)
+
+    def stop(self):
+        self._stop = True
+
+    def _load_data(self, path):
+        """Load data from the given file path. Return true if successful, false otherwise."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/staticdata/static_data_provider.py

@@ -0,0 +1,28 @@
+from datetime import datetime
+
+import pytz
+
+
+class StaticDataProvider:
+    """Generic static reference data provider class. Subclasses of this query the individual URLs or files for data."""
+
+    def __init__(self, name, provider_config):
+        """Constructor"""
+
+        self.name = name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+        self.reference_count = 0
+
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")