Refactor of caching & data storage part 9 #118

providers/alert/alert_provider.py

@@ -0,0 +1,45 @@
+from datetime import datetime
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+
+class AlertProvider:
+    """Generic alert provider class. Subclasses of this query the individual APIs for alerts."""
+
+    def __init__(self, name, provider_config):
+        """Constructor"""
+
+        self.name = name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+        self._alerts = DATA_STORE.alerts
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def _submit_batch(self, alerts):
+        """Submit a batch of alerts retrieved from the provider. There is no timestamp checking like there is for spots,
+        because alerts could be created at any point for any time in the future. Rely on hashcode-based id matching
+        to deal with duplicates."""
+
+        # Sort the batch so that earliest ones go in first. This helps keep the ordering correct when alerts are fired
+        # off to SSE listeners.
+        alerts = sorted(alerts, key=lambda a: (a.start_time if a and a.start_time else 0))
+        for alert in alerts:
+            # Fill in any blanks and add to the list
+            alert.infer_missing()
+            self._add_alert(alert)
+
+    def _add_alert(self, alert):
+        if not alert.expired():
+            self._alerts.set(alert.id, alert)
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/alert/bota.py

@@ -0,0 +1,62 @@
+from datetime import datetime, timedelta
+
+import pytz
+from bs4 import BeautifulSoup
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class BOTA(HTTPAlertProvider):
+    """Alert provider for Beaches on the Air"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://www.beachesontheair.com/"
+
+    def __init__(self, provider_config):
+        super().__init__("BOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Find the table of upcoming alerts
+        bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
+        if not bs.body:
+            return new_alerts
+        div = bs.body.find('div', attrs={'class': 'view-activations-public'})
+        if div:
+            table = div.find('table', attrs={'class': 'views-table'})
+            if table:
+                tbody = table.find('tbody')
+                if not tbody:
+                    return new_alerts
+                for row in tbody.find_all('tr'):
+                    cells = row.find_all('td')
+                    first_cell_anchor = cells[0].find('a') if len(cells) > 0 else None
+                    second_cell_anchor = cells[1].find('a') if len(cells) > 1 else None
+                    if not first_cell_anchor or not second_cell_anchor:
+                        continue
+                    first_cell_text = first_cell_anchor.get_text().strip()
+                    ref_name = first_cell_text.split(" by ")[0]
+                    dx_call = second_cell_anchor.get_text().strip().upper()
+
+                    # Get the date, dealing with the fact we get no year so have to figure out if it's last year or next year
+                    date_span = cells[2].find('span') if len(cells) > 2 else None
+                    if not date_span:
+                        continue
+                    date_text = date_span.get_text().strip()
+                    date_time = datetime.strptime(date_text, "%d %b - %H:%M UTC").replace(tzinfo=pytz.UTC)
+                    date_time = date_time.replace(year=datetime.now(pytz.UTC).year)
+                    # If this was more than a day ago, activation is actually next year
+                    if date_time < datetime.now(pytz.UTC) - timedelta(days=1):
+                        date_time = date_time.replace(year=datetime.now(pytz.UTC).year + 1)
+
+                    # Convert to our alert format
+                    alert = Alert(source=self.name,
+                                  dx_calls=[dx_call],
+                                  sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
+                                  start_time=date_time.timestamp(),
+                                  is_dxpedition=False)
+
+                    new_alerts.append(alert)
+        return new_alerts

providers/alert/http_alert_provider.py

@@ -0,0 +1,75 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+
+from providers.alert.alert_provider import AlertProvider
+from core.constants import HTTP_HEADERS
+
+
+class HTTPAlertProvider(AlertProvider):
+    """Generic alert provider class for providers that request data via HTTP(S). Just for convenience to avoid code
+    duplication. Subclasses of this query the individual APIs for data."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        super().__init__(name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info("Set up query of " + self.name + " alert API every " + str(self._poll_interval) + " seconds.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval):
+                break
+
+    def _poll(self):
+        try:
+            # Request data from API
+            logging.debug("Polling " + self.name + " alert API...")
+            http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                new_alerts = self._http_response_to_alerts(http_response)
+                # Submit the new alerts for processing. There might not be any alerts for the less popular programs.
+                if new_alerts:
+                    self._submit_batch(new_alerts)
+
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.debug("Received data from " + self.name + " alert API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} alerts API.")
+
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} alerts API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} alerts API.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP JSON Alert Provider (" + self.name + ")")
+            # Brief pause on error before the next poll, but still respond promptly to stop()
+            self._stop_event.wait(timeout=1)
+
+    def _http_response_to_alerts(self, http_response):
+        """Convert an HTTP response returned by the API into alert data. The whole response is provided here so the subclass
+        implementations can check for HTTP status codes if necessary, and handle the response as JSON, XML, text, whatever
+        the API actually provides."""
+
+        raise NotImplementedError("Subclasses must implement this method")

providers/alert/ng3k.py

@@ -0,0 +1,89 @@
+import re
+from datetime import datetime
+from typing import cast
+
+import pytz
+from rss_parser import Parser
+from rss_parser.models.rss import RSS
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+
+
+class NG3K(HTTPAlertProvider):
+    """Alert provider NG3K DXpedition list"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://www.ng3k.com/adxo.xml"
+    AS_CALL_PATTERN = re.compile("as ([a-z0-9/]+)", re.IGNORECASE)
+
+    def __init__(self, provider_config):
+        super().__init__("NG3K", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
+        # Iterate through source data
+        for source_alert in rss.channel.items:
+            # Deal with "the format"...
+            parts = source_alert.description.split(" --\n")
+
+            start_string = parts[0].split("-")[0]
+            end_string = parts[0].split("-")[1]
+
+            end_year = end_string.split(", ")[1].strip()
+
+            if ", " in start_string:
+                start_year = start_string.split(", ")[1].strip()
+                start_mon = start_string.split(", ")[0][0:3].strip()
+                start_day = start_string.split(", ")[0][4:].strip()
+            else:
+                start_year = end_year
+                start_mon = start_string[0:3].strip()
+                start_day = start_string[4:].strip()
+
+            if " " in end_string.split(", ")[0]:
+                end_mon = end_string.split(", ")[0].split(" ")[0].strip()
+                end_day = end_string.split(", ")[0].split(" ")[1].strip()
+            else:
+                end_day = end_string.split(", ")[0].strip()
+                end_mon = start_mon
+
+            start_timestamp = datetime.strptime(start_year + " " + start_mon + " " + start_day, "%Y %b %d").replace(
+                tzinfo=pytz.UTC).timestamp()
+            end_timestamp = datetime.strptime(end_year + " " + end_mon + " " + end_day + " 23:59",
+                                              "%Y %b %d %H:%M").replace(
+                tzinfo=pytz.UTC).timestamp()
+
+            # Sometimes the DX callsign is "real", sometimes you just get a prefix with the real working callsigns being
+            # provided in the "by" field. e.g. call="JW", by="By LA7XK as JW7XK, LA6VM as JW6VM, LA9DL as JW9DL". So
+            # if there are "as" callsigns in the "by" field, we extract them and use them, otherwise we fall back to the
+            # "real" call field.
+            extra_parts = parts[5].split("; ")
+            by = extra_parts[0]
+            dx_calls = [x.group(1) for x in re.finditer(self.AS_CALL_PATTERN, by)]
+            if not dx_calls:
+                dx_calls = [parts[2].upper()]
+
+            # "Calls" of TBA, TBC or TBD are not real attempts at Turkish callsigns
+            dx_calls = list(filter(lambda a: a != "TBA" and a != "TBC" and a != "TBD", dx_calls))
+
+            dx_country = parts[1]
+            qsl_info = parts[3]
+            bands = extra_parts[1] if len(extra_parts) > 1 else ""
+            modes = extra_parts[2] if len(extra_parts) > 2 else ""
+            comment = extra_parts[3] if len(extra_parts) > 3 else ""
+
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          dx_calls=dx_calls,
+                          dx_country=dx_country,
+                          freqs_modes=bands + (("; " + modes) if modes != "" else ""),
+                          comment=by + "; " + comment + "; " + qsl_info,
+                          start_time=start_timestamp,
+                          end_time=end_timestamp,
+                          is_dxpedition=True)
+
+            # Add to our list.
+            new_alerts.append(alert)
+        return new_alerts

providers/alert/parksnpeaks.py

@@ -0,0 +1,61 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class ParksNPeaks(HTTPAlertProvider):
+    """Alert provider for Parks n Peaks"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://parksnpeaks.org/api/ALERTS/"
+
+    def __init__(self, provider_config):
+        super().__init__("ParksNPeaks", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json():
+            # Calculate some things
+            sig = source_alert["Class"].upper()
+            if " - " in source_alert["Location"]:
+                split = source_alert["Location"].split(" - ")
+                sig_ref = split[0]
+                sig_ref_name = split[1]
+            else:
+                sig_ref = source_alert["WWFFID"]
+                sig_ref_name = source_alert["Location"]
+            start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(
+                tzinfo=pytz.UTC).timestamp()
+
+            sigrefs = []
+            # PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list,
+            # so mask this out if we got it.
+            if sig != "QRP":
+                sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)]
+
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert["alID"],
+                          dx_calls=[source_alert["CallSign"].upper()],
+                          freqs_modes=source_alert["Freq"] + " " + source_alert["MODE"],
+                          comment=source_alert["Comments"],
+                          sig_refs=sigrefs,
+                          start_time=start_time,
+                          is_dxpedition=False)
+
+            # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
+            if sig and sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
+                logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!")
+
+            # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
+            # the alert list. Note that while ZLOTA has its own spots API, it doesn't have its own alerts API. So that
+            # means the PnP *spot* provider rejects ZLOTA spots here, but the PnP *alerts* provider here allows ZLOTA.
+            if sig not in ["POTA", "SOTA", "WWFF"]:
+                new_alerts.append(alert)
+        return new_alerts

providers/alert/pota.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+
+import pytz
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class POTA(HTTPAlertProvider):
+    """Alert provider for Parks on the Air"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://api.pota.app/activation"
+
+    def __init__(self, provider_config):
+        super().__init__("POTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json():
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert["scheduledActivitiesId"],
+                          dx_calls=[source_alert["activator"].upper()],
+                          freqs_modes=source_alert["frequencies"],
+                          comment=source_alert["comments"],
+                          sig_refs=[SIGRef(id=source_alert["reference"], sig="POTA", name=source_alert["name"],
+                                           url="https://pota.app/#/park/" + source_alert["reference"])],
+                          start_time=datetime.strptime(source_alert["startDate"] + source_alert["startTime"],
+                                                       "%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
+                          end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"],
+                                                     "%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
+                          is_dxpedition=False)
+
+            # Add to our list, but exclude any old spots that POTA can sometimes give us where even the end time is
+            # in the past. Don't worry about de-duping, removing old alerts etc. at this point; other code will do
+            # that for us.
+            if alert.end_time and alert.end_time > datetime.now(pytz.UTC).timestamp():
+                new_alerts.append(alert)
+        return new_alerts

providers/alert/sota.py

@@ -0,0 +1,44 @@
+from datetime import datetime
+
+import pytz
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class SOTA(HTTPAlertProvider):
+    """Alert provider for Summits on the Air"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://api-db2.sota.org.uk/api/alerts/365/all/all"
+
+    def __init__(self, provider_config):
+        super().__init__("SOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json():
+            # Convert to our alert format
+            details = source_alert["summitDetails"].split(", ")
+            summit_name = details[0]
+            summit_points = None
+            if len(details) > 2:
+                summit_points = int(details[-1].split(" ")[0])
+            alert = Alert(source=self.name,
+                          source_id=source_alert["id"],
+                          dx_calls=[source_alert["activatingCallsign"].upper()],
+                          dx_names=[source_alert["activatorName"].upper()],
+                          freqs_modes=source_alert["frequency"],
+                          comment=source_alert["comments"],
+                          sig_refs=[
+                              SIGRef(id=source_alert["associationCode"] + "/" + source_alert["summitCode"], sig="SOTA",
+                                     name=summit_name, activation_score=summit_points)],
+                          start_time=datetime.strptime(source_alert["dateActivated"],
+                                                       "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=pytz.UTC).timestamp(),
+                          is_dxpedition=False)
+
+            # Add to our list
+            new_alerts.append(alert)
+        return new_alerts

providers/alert/wota.py

@@ -0,0 +1,64 @@
+from datetime import datetime
+from typing import cast
+
+import pytz
+from rss_parser import Parser as RSSParser
+from rss_parser.models.rss import RSS
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class WOTA(HTTPAlertProvider):
+    """Alert provider for Wainwrights on the Air"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://www.wota.org.uk/alerts_rss.php"
+    RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
+
+    def __init__(self, provider_config):
+        super().__init__("WOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig")))
+        # Iterate through source data
+        for source_alert in rss.channel.items:
+
+            # Reject GUID missing or zero
+            if not source_alert.guid or not source_alert.guid.content or source_alert.guid.content == "http://www.wota.org.uk/alerts/0":
+                continue
+
+            # Pick apart the title
+            title_split = source_alert.title.split(" on ")
+            dx_call = title_split[0]
+            ref = None
+            ref_name = None
+            if len(title_split) > 1:
+                ref_split = title_split[1].split(" - ")
+                ref = str(ref_split[0])
+                if len(ref_split) > 1:
+                    ref_name = str(ref_split[1])
+
+            # Pick apart the description
+            desc_split = source_alert.description.split(". ")
+            freqs_modes = desc_split[0].replace("Frequencies/modes:", "").strip()
+            comment = None
+            if len(desc_split) > 1:
+                comment = desc_split[1].strip()
+
+            time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert.guid.content,
+                          dx_calls=[dx_call],
+                          freqs_modes=freqs_modes,
+                          comment=comment,
+                          sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
+                          start_time=time.timestamp())
+
+            # Add to our list.
+            new_alerts.append(alert)
+        return new_alerts

providers/alert/wwff.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+
+import pytz
+
+from providers.alert.http_alert_provider import HTTPAlertProvider
+from data.alert import Alert
+from data.sig_ref import SIGRef
+
+
+class WWFF(HTTPAlertProvider):
+    """Alert provider for Worldwide Flora and Fauna"""
+
+    POLL_INTERVAL_SEC = 1800
+    ALERTS_URL = "https://spots.wwff.co/static/agendas.json"
+
+    def __init__(self, provider_config):
+        super().__init__("WWFF", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+
+    def _http_response_to_alerts(self, http_response):
+        new_alerts = []
+        # Iterate through source data
+        for source_alert in http_response.json():
+            # Convert to our alert format
+            alert = Alert(source=self.name,
+                          source_id=source_alert["id"],
+                          dx_calls=[source_alert["activator_call"].upper()],
+                          freqs_modes=source_alert["band"] + " " + source_alert["mode"],
+                          comment=source_alert["remarks"],
+                          sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")],
+                          start_time=datetime.strptime(source_alert["utc_start"],
+                                                       "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
+                          end_time=datetime.strptime(source_alert["utc_end"],
+                                                     "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
+                          is_dxpedition=False)
+
+            # Add to our list
+            new_alerts.append(alert)
+        return new_alerts