diff --git a/providers/alert/wota.py b/providers/alert/wota.py index 83a1d22..4dd1522 100644 --- a/providers/alert/wota.py +++ b/providers/alert/wota.py @@ -1,6 +1,7 @@ import logging from datetime import datetime from typing import cast +from xml.parsers.expat import ExpatError import pytz from rss_parser import Parser as RSSParser @@ -25,56 +26,62 @@ class WOTA(HTTPAlertProvider): def _http_response_to_alerts(self, http_response): new_alerts = [] - rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig"))) - # Iterate through source data - for source_alert in rss.channel.items: - # Reject GUID missing or zero - if ( - not source_alert.guid - or not source_alert.guid.content - or source_alert.guid.content == "http://www.wota.org.uk/alerts/0" - ): - continue - # Pick apart the title - dx_call = None - ref = None - ref_name = None - try: - title_split = source_alert.title.split(" on ") - dx_call = title_split[0] - if len(title_split) > 1: - ref_split = title_split[1].split(" - ") - ref = str(ref_split[0]) - if len(ref_split) > 1: - ref_name = str(ref_split[1]) - except Exception: - logger.warning(f"Could not parse WOTA alert title: {source_alert.description}") + try: + rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig"))) + # Iterate through source data + for source_alert in rss.channel.items: + # Reject GUID missing or zero + if ( + not source_alert.guid + or not source_alert.guid.content + or source_alert.guid.content == "http://www.wota.org.uk/alerts/0" + ): + continue - # Pick apart the description - comment = None - freqs_modes = None - try: - desc_split = source_alert.description.split(". ") - freqs_modes = desc_split[0].replace("Frequencies/modes:", "").strip() - if len(desc_split) > 1: - comment = desc_split[1].strip() - except Exception: - logger.warning(f"Could not parse WOTA alert description: {source_alert.description}") + # Pick apart the title + dx_call = None + ref = None + ref_name = None + try: + title_split = source_alert.title.split(" on ") + dx_call = title_split[0] + if len(title_split) > 1: + ref_split = title_split[1].split(" - ") + ref = str(ref_split[0]) + if len(ref_split) > 1: + ref_name = str(ref_split[1]) + except Exception: + logger.warning(f"Could not parse WOTA alert title: {source_alert.description}") - time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC) + # Pick apart the description + comment = None + freqs_modes = None + try: + desc_split = source_alert.description.split(". ") + freqs_modes = desc_split[0].replace("Frequencies/modes:", "").strip() + if len(desc_split) > 1: + comment = desc_split[1].strip() + except Exception: + logger.warning(f"Could not parse WOTA alert description: {source_alert.description}") - # Convert to our alert format - alert = Alert( - source=self.name, - source_id=source_alert.guid.content, - dx_calls=[dx_call], - freqs_modes=freqs_modes, - comment=comment, - sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [], - start_time=time.timestamp(), - ) + time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC) + + # Convert to our alert format + alert = Alert( + source=self.name, + source_id=source_alert.guid.content, + dx_calls=[dx_call], + freqs_modes=freqs_modes, + comment=comment, + sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [], + start_time=time.timestamp(), + ) + + # Add to our list. + new_alerts.append(alert) + + except ExpatError: + logger.warning("WOTA alert RSS feed was fetched but was invalid") - # Add to our list. - new_alerts.append(alert) return new_alerts diff --git a/providers/solarconditions/giroionosonde.py b/providers/solarconditions/giroionosonde.py index 91ab322..06d862f 100644 --- a/providers/solarconditions/giroionosonde.py +++ b/providers/solarconditions/giroionosonde.py @@ -13,10 +13,10 @@ from providers.solarconditions.solar_conditions_provider import SolarConditionsP logger = logging.getLogger(__name__) -# Each station gets polled roughly once every hour (3600 seconds). Note that to avoid a burst of requests to the server -# every hour, the requests for data from each station are spaced out throughout the hour, leading to one request being -# sent every 1-2 minutes. -POLL_INTERVAL = 3600 +# Each station gets polled roughly once every two hours (7200 seconds). Note that to avoid a burst of requests to the +# server every two hours, the requests for data from each station are spaced out throughout the period, leading to one +# request being sent every 3-4 minutes. +POLL_INTERVAL = 7200 # To avoid looking up all stations in the GIRO system and working out which ones are providing live data, this has been # manually determined and a CSV provided of all the stations that we can query for live data. STATIONS_INDEX = "datafiles/didbase-stations.csv" diff --git a/providers/spot/tiles.py b/providers/spot/tiles.py index baca9ef..b291314 100644 --- a/providers/spot/tiles.py +++ b/providers/spot/tiles.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime import requests @@ -7,6 +8,8 @@ from data.sig_ref import SIGRef from data.spot import Spot from providers.spot.http_spot_provider import HTTPSpotProvider +logger = logging.getLogger(__name__) + class Tiles(HTTPSpotProvider): """Spot provider for Tiles on the Air""" @@ -39,13 +42,19 @@ class Tiles(HTTPSpotProvider): # Iterate through source data for source_spot in http_response.json()["spots"]: # Convert to our spot format + freq = None + try: + freq = float(strip_extra_decimal_points(source_spot["frequency"])) * 1000000 + except ValueError: + logger.debug(f"Malformed frequency received from Tiles API: {source_spot['frequency']}") + spot = Spot( source=self.name, source_id=source_spot["id"], dx_call=source_spot["call_sign"].upper(), # No separate spotter callsign, assume all spots are self-spots de_call=source_spot["call_sign"].upper(), - freq=float(strip_extra_decimal_points(source_spot["frequency"])) * 1000000, + freq=freq, mode=source_spot["mode"].upper(), comment=source_spot["notes"], sig="Tiles", diff --git a/providers/spot/wota.py b/providers/spot/wota.py index c08b32c..717d325 100644 --- a/providers/spot/wota.py +++ b/providers/spot/wota.py @@ -2,6 +2,7 @@ import logging import re from datetime import datetime from typing import cast +from xml.parsers.expat import ExpatError import pytz from rss_parser import Parser @@ -26,73 +27,78 @@ class WOTA(HTTPSpotProvider): def _http_response_to_spots(self, http_response): new_spots = [] - rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig"))) - # Iterate through source data - for source_spot in rss.channel.items: - try: - # Reject GUID missing or zero - if ( - not source_spot.guid - or not source_spot.guid.content - or source_spot.guid.content == "http://www.wota.org.uk/spots/0" - ): - continue - - # Pick apart the title - dx_call = None - ref = None - ref_name = None + try: + rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig"))) + # Iterate through source data + for source_spot in rss.channel.items: try: - title_split = source_spot.title.split(" on ") - dx_call = title_split[0] - if len(title_split) > 1: - ref_split = title_split[1].split(" - ") - ref = str(ref_split[0]) - if len(ref_split) > 1: - ref_name = str(ref_split[1]) + # Reject GUID missing or zero + if ( + not source_spot.guid + or not source_spot.guid.content + or source_spot.guid.content == "http://www.wota.org.uk/spots/0" + ): + continue + + # Pick apart the title + dx_call = None + ref = None + ref_name = None + try: + title_split = source_spot.title.split(" on ") + dx_call = title_split[0] + if len(title_split) > 1: + ref_split = title_split[1].split(" - ") + ref = str(ref_split[0]) + if len(ref_split) > 1: + ref_name = str(ref_split[1]) + except Exception: + logger.warning(f"Could not parse WOTA spot title: {source_spot.title}") + + # Pick apart the description + freq_hz = None + mode = None + comment = None + spotter = None + try: + desc_split = source_spot.description.split(". ") + freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip() + if freq_mode and freq_mode != "-": + freq_mode_split = re.split(r"[\-\s]+", freq_mode) + freq_hz = float(freq_mode_split[0].replace("'", ".")) * 1000000 + if len(freq_mode_split) > 1: + mode = freq_mode_split[1].upper() + + if len(desc_split) > 1: + comment = desc_split[1].strip() + if len(desc_split) > 2: + spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").upper().strip() + except Exception: + logger.warning(f"Could not parse WOTA spot description: {source_spot.description}") + + time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC) + + # Convert to our spot format + spot = Spot( + source=self.name, + source_id=source_spot.guid.content, + dx_call=dx_call, + de_call=spotter, + freq=freq_hz, + mode=mode, + comment=comment, + sig="WOTA", + sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [], + time=time.timestamp(), + ) + + new_spots.append(spot) except Exception: - logger.warning(f"Could not parse WOTA spot title: {source_spot.title}") + logger.exception("Exception parsing WOTA spot") - # Pick apart the description - freq_hz = None - mode = None - comment = None - spotter = None - try: - desc_split = source_spot.description.split(". ") - freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip() - if freq_mode and freq_mode != "-": - freq_mode_split = re.split(r"[\-\s]+", freq_mode) - freq_hz = float(freq_mode_split[0].replace("'", ".")) * 1000000 - if len(freq_mode_split) > 1: - mode = freq_mode_split[1].upper() + except ExpatError: + logger.warning("WOTA spot RSS feed was fetched but was invalid") - if len(desc_split) > 1: - comment = desc_split[1].strip() - if len(desc_split) > 2: - spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").upper().strip() - except Exception: - logger.warning(f"Could not parse WOTA spot description: {source_spot.description}") - - time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC) - - # Convert to our spot format - spot = Spot( - source=self.name, - source_id=source_spot.guid.content, - dx_call=dx_call, - de_call=spotter, - freq=freq_hz, - mode=mode, - comment=comment, - sig="WOTA", - sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [], - time=time.timestamp(), - ) - - new_spots.append(spot) - except Exception: - logger.exception("Exception parsing WOTA spot") return new_spots def can_submit_spot(self, sig): diff --git a/templates/add_spot.html b/templates/add_spot.html index 00fd09f..87b09ea 100644 --- a/templates/add_spot.html +++ b/templates/add_spot.html @@ -76,7 +76,7 @@ - + diff --git a/templates/alerts.html b/templates/alerts.html index 730b9c3..065cbfa 100644 --- a/templates/alerts.html +++ b/templates/alerts.html @@ -84,7 +84,7 @@ - + diff --git a/templates/bands.html b/templates/bands.html index 27fb195..6e1dca6 100644 --- a/templates/bands.html +++ b/templates/bands.html @@ -76,8 +76,8 @@ - - + + diff --git a/templates/base.html b/templates/base.html index 4234539..e6eacf6 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,6 +1,6 @@ {% extends "skeleton.html" %} {% block head_extra %} - + @@ -15,10 +15,10 @@ window.fetchEventSource = fetchEventSource; - - - - + + + + {% end %} {% block body %}
diff --git a/templates/conditions.html b/templates/conditions.html index aa91400..c7caabf 100644 --- a/templates/conditions.html +++ b/templates/conditions.html @@ -284,7 +284,7 @@
- + diff --git a/templates/map.html b/templates/map.html index 0c80574..eceb479 100644 --- a/templates/map.html +++ b/templates/map.html @@ -109,8 +109,8 @@ - - + + diff --git a/templates/spots.html b/templates/spots.html index 543ec3d..f10bf07 100644 --- a/templates/spots.html +++ b/templates/spots.html @@ -113,8 +113,8 @@ - - + + diff --git a/templates/status.html b/templates/status.html index 62c483a..07473d6 100644 --- a/templates/status.html +++ b/templates/status.html @@ -86,7 +86,7 @@ - +