diff --git a/alertproviders/bota.py b/alertproviders/bota.py index 003a405..a275430 100644 --- a/alertproviders/bota.py +++ b/alertproviders/bota.py @@ -20,7 +20,7 @@ class BOTA(HTTPAlertProvider): def _http_response_to_alerts(self, http_response): new_alerts = [] # Find the table of upcoming alerts - bs = BeautifulSoup(http_response.content.decode(), features="lxml") + bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml") if not bs.body: return new_alerts div = bs.body.find('div', attrs={'class': 'view-activations-public'}) diff --git a/alertproviders/ng3k.py b/alertproviders/ng3k.py index 8b69d46..b32e54e 100644 --- a/alertproviders/ng3k.py +++ b/alertproviders/ng3k.py @@ -22,7 +22,7 @@ class NG3K(HTTPAlertProvider): def _http_response_to_alerts(self, http_response): new_alerts = [] - rss = cast(RSS, Parser.parse(http_response.content.decode())) + rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig"))) # Iterate through source data for source_alert in rss.channel.items: # Deal with "the format"... diff --git a/alertproviders/parksnpeaks.py b/alertproviders/parksnpeaks.py index 7dfb61b..fbfd7d5 100644 --- a/alertproviders/parksnpeaks.py +++ b/alertproviders/parksnpeaks.py @@ -22,7 +22,7 @@ class ParksNPeaks(HTTPAlertProvider): # Iterate through source data for source_alert in http_response.json(): # Calculate some things - sig = source_alert["Class"] + sig = source_alert["Class"].upper() if " - " in source_alert["Location"]: split = source_alert["Location"].split(" - ") sig_ref = split[0] @@ -50,7 +50,7 @@ class ParksNPeaks(HTTPAlertProvider): is_dxpedition=False) # Log a warning for the developer if PnP gives us an unknown programme we've never seen before - if sig and sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]: + if sig and sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]: logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!") # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to diff --git a/alertproviders/wota.py b/alertproviders/wota.py index 7712eca..abcf2d3 100644 --- a/alertproviders/wota.py +++ b/alertproviders/wota.py @@ -22,7 +22,7 @@ class WOTA(HTTPAlertProvider): def _http_response_to_alerts(self, http_response): new_alerts = [] - rss = cast(RSS, RSSParser.parse(http_response.content.decode())) + rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig"))) # Iterate through source data for source_alert in rss.channel.items: diff --git a/config-example.yml b/config-example.yml index 81a56c5..10ce78e 100644 --- a/config-example.yml +++ b/config-example.yml @@ -121,7 +121,7 @@ spot-providers: url: "wss://39c3.totawatch.de/api/spot/live" # For the "XOTA" provider, a SIG must be set menually here because xOTA is a generic backend for xOTA # programmes and so different URLs potentially provide different programmes. - sig: "TOTA" + sig: "Toilets" # For Toilets on the Air, we prefix the SIG references (T-01 etc) with some characters that define the conference: # C3, EH or HOPE - so we can look up the correct locations in our database, because each conference starts from T-01 # but refers to a toilet in a different building (or continent!) @@ -131,14 +131,14 @@ spot-providers: name: "EH23 TOTA" enabled: false url: "wss://eh23.totawatch.de/api/spot/live" - sig: "TOTA" + sig: "Toilets" sig-ref-prefix: "EH" - class: "XOTA" name: "HOPE26 TOTA" enabled: false url: "wss://hope-26.totawatch.de/api/spot/live" - sig: "TOTA" + sig: "Toilets" sig-ref-prefix: "HOPE" @@ -169,9 +169,36 @@ alert-providers: # SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF # park IDs to their name and location. sig-ref-data-providers: + - class: "POTA" + enabled: true + + - class: "SOTA" + enabled: true + - class: "WWFF" enabled: true + - class: "WWBOTA" + enabled: true + + - class: "GMA" + enabled: true + + - class: "MOTA" + enabled: true + + - class: "ILLW" + enabled: true + + - class: "ARLHS" + enabled: true + + - class: "WCA" + enabled: true + + - class: "IOTA" + enabled: true + - class: "WOTA" enabled: true @@ -184,10 +211,13 @@ sig-ref-data-providers: - class: "LLOTA" enabled: true + - class: "Towers" + enabled: true + - class: "DME" enabled: true - - class: "TOTA" + - class: "Toilets" enabled: true # Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band diff --git a/core/constants.py b/core/constants.py index f412f6b..c0d42c5 100644 --- a/core/constants.py +++ b/core/constants.py @@ -19,9 +19,9 @@ SIGS = [ SIG(name="HEMA", comment_names=["HEMA"], description="HuMPs Excluding Marilyns Award", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}"), SIG(name="IOTA", comment_names=["IOTA"], description="Islands on the Air", ref_regex=r"[A-Z]{2}\-\d{3}"), SIG(name="MOTA", comment_names=["MOTA"], description="Mills on the Air", ref_regex=r"X\d{4,6}"), - SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}\-\d{3,4}"), + SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}[\- ]\d{3,4}"), SIG(name="ILLW", comment_names=["ILLW"], description="International Lighthouse & Lightship Weekend", ref_regex=r"[A-Z]{2}\d{4}"), - SIG(name="SiOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"), + SIG(name="SIOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"), SIG(name="WCA", comment_names=["WCA"], description="World Castles Award", ref_regex=r"[A-Z0-9]{1,3}\-\d{5}"), SIG(name="ZLOTA", comment_names=["ZLOTA"], description="New Zealand on the Air", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}"), SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"), diff --git a/core/data_store.py b/core/data_store.py index 7e23e27..c5b3ec7 100644 --- a/core/data_store.py +++ b/core/data_store.py @@ -1,8 +1,10 @@ +import logging from pathlib import Path import diskcache from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE +from core.constants import SIGS from core.live_data_cache import LiveDataCache from data.solar_conditions import SolarConditions @@ -12,51 +14,65 @@ class DataStore: lookup data using different caching strategies for each.""" def __init__(self): - self.CACHE_DIR = "./cache" - self.MAX_SPOT_COUNT = 10000 - self.MAX_ALERT_COUNT = 10000 - self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300 - self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60 + self._CACHE_DIR = "./cache" + self._MAX_SPOT_COUNT = 100000 + self._MAX_ALERT_COUNT = 100000 + self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300 + self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60 + self.alerts = None + self.spots = None + self.callsigns = None + self.sigrefs = None + self.status_data = None + self._status = None + self.solar_conditions = None + self._solar = None def setup(self): - Path(self.CACHE_DIR).mkdir(parents=True, exist_ok=True) + Path(self._CACHE_DIR).mkdir(parents=True, exist_ok=True) # Standard disk cache for solar data and status data, but each cache contains only a single object which we # expose to the wider application - self.solar = diskcache.Cache(self.CACHE_DIR + "/solar") - if "solar_conditions" not in self.solar: - self.solar.add("solar_conditions", SolarConditions()) - self.solar_conditions = self.solar.get("solar_conditions") - self.status = diskcache.Cache(self.CACHE_DIR + "/status") - if "status_data" not in self.status: - self.status.add("status_data", {}) - self.status_data = self.status.get("status_data") + self._solar = diskcache.Cache(self._CACHE_DIR + "/solar") + if "solar_conditions" not in self._solar: + self._solar.add("solar_conditions", SolarConditions()) + self.solar_conditions = self._solar.get("solar_conditions") + self._status = diskcache.Cache(self._CACHE_DIR + "/status") + if "status_data" not in self._status: + self._status.add("status_data", {}) + self.status_data = self._status.get("status_data") # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis # but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG # name and then reference ID, with the final value being a SIGRef object. - self.sigrefs = diskcache.Cache(self.CACHE_DIR + "/sigrefs") + self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs") + for k in list(self.sigrefs.iterkeys()): + logging.info(f"Loaded data for %d references in %s SIG.", len(self.sigrefs[k]), k) # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup. # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign # we've seen, so we rely on them timing out and this triggering another lookup. - self.callsigns = diskcache.Cache(self.CACHE_DIR + "/callsigns") + self.callsigns = diskcache.Cache(self._CACHE_DIR + "/callsigns") + logging.info(f"Loaded data for %d callsigns.", len(self.callsigns)) # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache # before the spots and alerts are live in the system. - self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE, - snapshot_dir=self.CACHE_DIR + "/spots", - snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) - self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE, - snapshot_dir=self.CACHE_DIR + "/alerts", - snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) + self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE, + snapshot_dir=self._CACHE_DIR + "/spots", + snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) + logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys())) + + self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE, + snapshot_dir=self._CACHE_DIR + "/alerts", + snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) + logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys())) def close(self): self.spots.close() self.alerts.close() - self.solar.close() - self.status.close() + self._solar.close() + self._status.close() self.sigrefs.close() self.callsigns.close() diff --git a/core/sig_utils.py b/core/sig_utils.py index 2c16c9d..68c2a34 100644 --- a/core/sig_utils.py +++ b/core/sig_utils.py @@ -35,157 +35,25 @@ def populate_sig_ref_info(sig_ref): SIG we are getting data for. Note there is currently no support for KRMNPA location lookup, see issue #61.""" - if sig_ref.sig is None or sig_ref.id is None or sig_ref.id == "": + if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "": logging.debug("Failed to look up sig_ref info, sig or id were not set.") return sig_ref - sig = sig_ref.sig or "" + sig = sig_ref.sig ref_id = sig_ref.id + + # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all activators + # add leading zeros. + if sig.upper() == "DME": + ref_id = ref_id.zfill(5) + try: - if sig.upper() == "POTA": - response = URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS) - if response.ok: - data = response.json() - if data: - fullname = str(data["name"]) if "name" in data else None - if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "": - fullname = fullname + " " + data["parktypeDesc"] - sig_ref.name = fullname - sig_ref.url = "https://pota.app/#/park/" + ref_id - sig_ref.grid = data["grid6"] if "grid6" in data else None - sig_ref.latitude = data["latitude"] if "latitude" in data else None - sig_ref.longitude = data["longitude"] if "longitude" in data else None - elif not response.from_cache: - logging.warning("Malformed response looking up %s ref %s", sig, ref_id) - elif not response.from_cache: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - - elif sig.upper() == "SOTA": - response = URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id, - headers=HTTP_HEADERS) - if response.ok: - data = response.json() - if data: - sig_ref.name = data["name"] if "name" in data else None - sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id - sig_ref.grid = data["locator"] if "locator" in data else None - sig_ref.latitude = data["latitude"] if "latitude" in data else None - sig_ref.longitude = data["longitude"] if "longitude" in data else None - sig_ref.activation_score = data["points"] if "points" in data else None - elif not response.from_cache: - logging.warning("Malformed response looking up %s ref %s", sig, ref_id) - elif not response.from_cache: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - - elif sig.upper() == "WWBOTA": - response = URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id, - headers=HTTP_HEADERS) - if response.ok: - data = response.json() - if data: - sig_ref.name = data["name"] if "name" in data else None - sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None - sig_ref.grid = data["locator"] if "locator" in data else None - sig_ref.latitude = data["lat"] if "lat" in data else None - sig_ref.longitude = data["long"] if "long" in data else None - elif not response.from_cache: - logging.warning("Malformed response looking up %s ref %s", sig, ref_id) - elif not response.from_cache: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - - elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA": - response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id, - headers=HTTP_HEADERS) - if response.ok: - data = response.json() - if data: - sig_ref.name = data["name"] if "name" in data else None - sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id - sig_ref.grid = data["locator"] if "locator" in data else None - - # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are - # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this - # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters, - # just use the single point we have. - if data.get("latitude") is not None and data.get("longitude") is not None and data.get( - "lat2") is not None and data.get("lng2") is not None: - sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0 - sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0 - else: - sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None - sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None - elif not response.from_cache: - logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id) - elif not response.from_cache: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - - elif sig.upper() == "WWFF": - lookup_data = DATA_STORE.sigrefs["WWFF"][ref_id] if ref_id in DATA_STORE.sigrefs["WWFF"] else None - if lookup_data: - # Copy new sig ref data into existing object - sig_ref.__dict__.update(lookup_data.__dict__) - else: - logging.warning("WWFF database did not contain data for ref %s", ref_id) - - elif sig.upper() == "SIOTA": - lookup_data = DATA_STORE.sigrefs["SIOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["SIOTA"] else None - if lookup_data: - # Copy new sig ref data into existing object - sig_ref.__dict__.update(lookup_data.__dict__) - else: - logging.warning("SIOTA database did not contain data for ref %s", ref_id) - - elif sig.upper() == "WOTA": - lookup_data = DATA_STORE.sigrefs["WOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["WOTA"] else None - if lookup_data: - # Copy new sig ref data into existing object - sig_ref.__dict__.update(lookup_data.__dict__) - else: - logging.warning("WOTA database did not contain data for ref %s", ref_id) - - elif sig.upper() == "ZLOTA": - lookup_data = DATA_STORE.sigrefs["ZLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["ZLOTA"] else None - if lookup_data: - # Copy new sig ref data into existing object - sig_ref.__dict__.update(lookup_data.__dict__) - else: - logging.warning("ZLOTA database did not contain data for ref %s", ref_id) - - elif sig.upper() == "BOTA": - if not sig_ref.name: - sig_ref.name = sig_ref.id - sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-") - - elif sig.upper() == "LLOTA": - lookup_data = DATA_STORE.sigrefs["LLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["LLOTA"] else None - if lookup_data: - # Copy new sig ref data into existing object - sig_ref.__dict__.update(lookup_data.__dict__) - else: - logging.warning("LLOTA database did not contain data for ref %s", ref_id) - - elif sig.upper() == "DME": - # Zero-pad to 5 digits to match our source data - lookup_data = DATA_STORE.sigrefs["DME"][ref_id.zfill(5)] if ref_id.zfill(5) in DATA_STORE.sigrefs["DME"] else None - if lookup_data: - # Copy new sig ref data into existing object - sig_ref.__dict__.update(lookup_data.__dict__) - else: - logging.warning("DME database did not contain data for ref %s", ref_id) - - elif sig.upper() == "TOTA": - lookup_data = DATA_STORE.sigrefs["TOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["TOTA"] else None - if lookup_data: - # Copy new sig ref data into existing object - sig_ref.__dict__.update(lookup_data.__dict__) - else: - logging.warning("TOTA database did not contain data for ref %s", ref_id) - - elif sig.upper() == "WWTOTA": - if not sig_ref.name: - sig_ref.name = sig_ref.id - sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + str(sig_ref.name) + # If the SIG is HEMA or KRMNPA, we have no current lookup for this so just skip it. + if sig.upper() == "HEMA" or sig.upper() == "KRMNPA": + return sig_ref + # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can + # calculate all the information we are going to get directly. So handle those cases first elif sig.upper() == "TILES": # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same if not sig_ref.name: @@ -208,10 +76,54 @@ def populate_sig_ref_info(sig_ref): except: logging.warning("Invalid lat/lon received for WAB/WAI reference") - except ConnectionError: - logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id) - except (ConnectTimeout, ReadTimeout): - logging.warning(f"Timeout when looking up sig_ref info for " + sig + " ref " + ref_id) + elif sig.upper() == "BOTA": + # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs + if not sig_ref.name: + sig_ref.name = sig_ref.id + sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-") + + # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for + # this SIG. If so, check for the reference data and use that. + elif sig in DATA_STORE.sigrefs: + lookup_data = DATA_STORE.sigrefs[sig][ref_id] if ref_id in DATA_STORE.sigrefs[sig] else None + if lookup_data: + # Copy new sig ref data into existing object + sig_ref.__dict__.update(lookup_data.__dict__) + else: + logging.warning("%s database did not contain data for ref %s", sig, ref_id) + + elif False: + # TODO remove + # OK, this is not a SIG we have stored data for. Maybe it's of a type we can query information for live. + if sig.upper() == "IOTA": + response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id, + headers=HTTP_HEADERS) + if response.ok: + data = response.json() + if data: + sig_ref.name = data["name"] if "name" in data else None + sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id + sig_ref.grid = data["locator"] if "locator" in data else None + + # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are + # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this + # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters, + # just use the single point we have. + if data.get("latitude") is not None and data.get("longitude") is not None and data.get( + "lat2") is not None and data.get("lng2") is not None: + sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0 + sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0 + else: + sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None + sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None + elif not response.from_cache: + logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id) + elif not response.from_cache: + logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + + else: + logging.warning(f"Tried to look up a SIG called %s but Spothole does not know what that is.", sig) + except Exception: logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True) return sig_ref diff --git a/core/status_reporter.py b/core/status_reporter.py index 82ada84..7fc38b0 100644 --- a/core/status_reporter.py +++ b/core/status_reporter.py @@ -77,7 +77,8 @@ class StatusReporter: DATA_STORE.status_data["sig_ref_data_providers"] = list( map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status, "last_updated": p.last_update_time.replace( - tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, + tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0, + "reference_count": p.reference_count}, self._sig_ref_data_providers)) DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"], "last_api_access": self._web_server.web_server_metrics[ diff --git a/data/alert.py b/data/alert.py index 2052d09..62c40f7 100644 --- a/data/alert.py +++ b/data/alert.py @@ -1,6 +1,7 @@ import copy import hashlib import json +import logging from dataclasses import dataclass from datetime import datetime, timedelta @@ -64,69 +65,73 @@ class Alert: def infer_missing(self, credentials=None): """Infer missing parameters where possible""" - # If we somehow don't have a start time, set it to zero so it sorts off the bottom of any list but - # clients can still reliably parse it as a number. - if not self.start_time: - self.start_time = 0 + try: + # If we somehow don't have a start time, set it to zero so it sorts off the bottom of any list but + # clients can still reliably parse it as a number. + if not self.start_time: + self.start_time = 0 - # If we don't have a received time, this has just been received so set that to "now" - if not self.received_time: - self.received_time = datetime.now(pytz.UTC).timestamp() + # If we don't have a received time, this has just been received so set that to "now" + if not self.received_time: + self.received_time = datetime.now(pytz.UTC).timestamp() - # Fill in ISO versions of times, in case the client prefers that - if self.start_time and not self.start_time_iso: - self.start_time_iso = datetime.fromtimestamp(self.start_time, pytz.UTC).isoformat() - if self.end_time and not self.end_time_iso: - self.end_time_iso = datetime.fromtimestamp(self.end_time, pytz.UTC).isoformat() - if self.received_time and not self.received_time_iso: - self.received_time_iso = datetime.fromtimestamp(self.received_time, pytz.UTC).isoformat() + # Fill in ISO versions of times, in case the client prefers that + if self.start_time and not self.start_time_iso: + self.start_time_iso = datetime.fromtimestamp(self.start_time, pytz.UTC).isoformat() + if self.end_time and not self.end_time_iso: + self.end_time_iso = datetime.fromtimestamp(self.end_time, pytz.UTC).isoformat() + if self.received_time and not self.received_time_iso: + self.received_time_iso = datetime.fromtimestamp(self.received_time, pytz.UTC).isoformat() - # DX country, continent, zones etc. from callsign. CQ/ITU zone are better looked up with a location but we don't - # have a real location for alerts. - if self.dx_calls and self.dx_calls[0] and not self.dx_country: - self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_calls[0], credentials) - if self.dx_calls and self.dx_calls[0] and not self.dx_continent: - self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_calls[0], credentials) - if self.dx_calls and self.dx_calls[0] and not self.dx_cq_zone: - self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_calls[0], credentials) - if self.dx_calls and self.dx_calls[0] and not self.dx_itu_zone: - self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_calls[0], credentials) - if self.dx_calls and self.dx_calls[0] and not self.dx_dxcc_id: - self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_calls[0], credentials) - if self.dx_dxcc_id and not self.dx_flag: - self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id) + # DX country, continent, zones etc. from callsign. CQ/ITU zone are better looked up with a location but we don't + # have a real location for alerts. + if self.dx_calls and self.dx_calls[0] and not self.dx_country: + self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_calls[0], credentials) + if self.dx_calls and self.dx_calls[0] and not self.dx_continent: + self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_calls[0], credentials) + if self.dx_calls and self.dx_calls[0] and not self.dx_cq_zone: + self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_calls[0], credentials) + if self.dx_calls and self.dx_calls[0] and not self.dx_itu_zone: + self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_calls[0], credentials) + if self.dx_calls and self.dx_calls[0] and not self.dx_dxcc_id: + self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_calls[0], credentials) + if self.dx_dxcc_id and not self.dx_flag: + self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id) - # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference - # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs - # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths - if self.sig_refs and len(self.sig_refs) > 0: - for sig_ref in self.sig_refs: - populate_sig_ref_info(sig_ref) + # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference + # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs + # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths + if self.sig_refs and len(self.sig_refs) > 0: + for sig_ref in self.sig_refs: + populate_sig_ref_info(sig_ref) - # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG - # and apply it to the whole spot. - if self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0] and not self.sig: - self.sig = self.sig_refs[0].sig + # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG + # and apply it to the whole spot. + if self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0] and not self.sig: + self.sig = self.sig_refs[0].sig - # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index - # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical - # apart from that they were retrieved from the API at different times. Note that the simple Python hash() - # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we - # use diskcache to store our data between runs, so we use SHA256 which does not include this random element. - # The ID is computed before the online lookups below so that it is stable regardless of whether credentials - # are provided, allowing the enriched API response to be matched to the stored alert by ID. - if not self.id: - self_copy = copy.deepcopy(self) - self_copy.received_time = 0 - self_copy.received_time_iso = "" - self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest() + # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index + # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical + # apart from that they were retrieved from the API at different times. Note that the simple Python hash() + # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we + # use diskcache to store our data between runs, so we use SHA256 which does not include this random element. + # The ID is computed before the online lookups below so that it is stable regardless of whether credentials + # are provided, allowing the enriched API response to be matched to the stored alert by ID. + if not self.id: + self_copy = copy.deepcopy(self) + self_copy.received_time = 0 + self_copy.received_time_iso = "" + self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest() - # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data - # from the actual alerting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon - # instead of the one from the park reference they're at. - if self.dx_calls and not self.dx_names: - self.dx_names = list( - map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c, credentials), self.dx_calls)) + # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data + # from the actual alerting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon + # instead of the one from the park reference they're at. + if self.dx_calls and not self.dx_names: + self.dx_names = list( + map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c, credentials), self.dx_calls)) + + except Exception as e: + logging.error("Exception while inferring missing data from spot", e, exc_info=True) def to_json(self): """JSON serialise""" diff --git a/data/spot.py b/data/spot.py index 56f2e78..bf32145 100644 --- a/data/spot.py +++ b/data/spot.py @@ -147,295 +147,299 @@ class Spot: def infer_missing(self, credentials=None): """Infer missing parameters where possible""" - # If we somehow don't have a spot time, set it to zero so it sorts off the bottom of any list but - # clients can still reliably parse it as a number. - if not self.time: - self.time = 0 + try: + # If we somehow don't have a spot time, set it to zero so it sorts off the bottom of any list but + # clients can still reliably parse it as a number. + if not self.time: + self.time = 0 - # If we don't have a received time, this has just been received so set that to "now" - if not self.received_time: - self.received_time = datetime.now(pytz.UTC).timestamp() + # If we don't have a received time, this has just been received so set that to "now" + if not self.received_time: + self.received_time = datetime.now(pytz.UTC).timestamp() - # Fill in ISO versions of times, in case the client prefers that - if self.time and not self.time_iso: - self.time_iso = datetime.fromtimestamp(self.time, pytz.UTC).isoformat() - if self.received_time and not self.received_time_iso: - self.received_time_iso = datetime.fromtimestamp(self.received_time, pytz.UTC).isoformat() + # Fill in ISO versions of times, in case the client prefers that + if self.time and not self.time_iso: + self.time_iso = datetime.fromtimestamp(self.time, pytz.UTC).isoformat() + if self.received_time and not self.received_time_iso: + self.received_time_iso = datetime.fromtimestamp(self.received_time, pytz.UTC).isoformat() - # Clean up DX call if it has an SSID or -# from RBN - if self.dx_call and "-" in self.dx_call: - split = self.dx_call.split("-") - self.dx_call = split[0] - if len(split) > 1 and split[1] != "#": - self.dx_ssid = split[1] + # Clean up DX call if it has an SSID or -# from RBN + if self.dx_call and "-" in self.dx_call: + split = self.dx_call.split("-") + self.dx_call = split[0] + if len(split) > 1 and split[1] != "#": + self.dx_ssid = split[1] - # DX country, continent etc. from callsign - if self.dx_call and not self.dx_country: - self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_call, credentials) - if self.dx_call and not self.dx_continent: - self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_call, credentials) - if self.dx_call and not self.dx_dxcc_id: - self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_call, credentials) - if self.dx_dxcc_id and not self.dx_flag: - self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id) + # DX country, continent etc. from callsign + if self.dx_call and not self.dx_country: + self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_call, credentials) + if self.dx_call and not self.dx_continent: + self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_call, credentials) + if self.dx_call and not self.dx_dxcc_id: + self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_call, credentials) + if self.dx_dxcc_id and not self.dx_flag: + self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id) - # Clean up spotter call if it has an SSID or -# from RBN - if self.de_call and "-" in self.de_call: - split = self.de_call.split("-") - self.de_call = split[0] - if len(split) > 1 and split[1] != "#": - self.de_ssid = split[1] + # Clean up spotter call if it has an SSID or -# from RBN + if self.de_call and "-" in self.de_call: + split = self.de_call.split("-") + self.de_call = split[0] + if len(split) > 1 and split[1] != "#": + self.de_ssid = split[1] - # If we have a spotter of "RBNHOLE", we should have the actual spotter callsign in the comment, so extract it. - # RBNHole posts come from a number of providers, so it's dealt with here in the generic spot handling code. - if self.de_call == "RBNHOLE" and self.comment: - rbnhole_call_match = re.search(r"\Wat ([a-z0-9/]+)\W", self.comment, re.IGNORECASE) - if rbnhole_call_match: - self.de_call = rbnhole_call_match.group(1).upper() + # If we have a spotter of "RBNHOLE", we should have the actual spotter callsign in the comment, so extract it. + # RBNHole posts come from a number of providers, so it's dealt with here in the generic spot handling code. + if self.de_call == "RBNHOLE" and self.comment: + rbnhole_call_match = re.search(r"\Wat ([a-z0-9/]+)\W", self.comment, re.IGNORECASE) + if rbnhole_call_match: + self.de_call = rbnhole_call_match.group(1).upper() - # If we have a spotter of "SOTAMAT", we might have the actual spotter callsign in the comment, if so extract it. - # SOTAMAT can do POTA as well as SOTA, so it's dealt with here in the generic spot handling code. - if self.de_call == "SOTAMAT" and self.comment: - sotamat_call_match = re.search(r"\Wfrom ([a-z0-9/]+)]", self.comment, re.IGNORECASE) - if sotamat_call_match: - self.de_call = sotamat_call_match.group(1).upper() + # If we have a spotter of "SOTAMAT", we might have the actual spotter callsign in the comment, if so extract it. + # SOTAMAT can do POTA as well as SOTA, so it's dealt with here in the generic spot handling code. + if self.de_call == "SOTAMAT" and self.comment: + sotamat_call_match = re.search(r"\Wfrom ([a-z0-9/]+)]", self.comment, re.IGNORECASE) + if sotamat_call_match: + self.de_call = sotamat_call_match.group(1).upper() - # Spotter country, continent, zones etc. from callsign. - # DE call with no digits, or APRS servers starting "T2" are not things we can look up location for - if self.de_call and any(char.isdigit() for char in self.de_call) and not ( - self.de_call.startswith("T2") and self.source == "APRS-IS"): - if not self.de_country: - self.de_country = lookup_helper.infer_country_from_callsign(self.de_call, credentials) - if not self.de_continent: - self.de_continent = lookup_helper.infer_continent_from_callsign(self.de_call, credentials) - if not self.de_dxcc_id: - self.de_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.de_call, credentials) - if self.de_dxcc_id and not self.de_flag: - self.de_flag = lookup_helper.get_flag_for_dxcc(self.de_dxcc_id) + # Spotter country, continent, zones etc. from callsign. + # DE call with no digits, or APRS servers starting "T2" are not things we can look up location for + if self.de_call and any(char.isdigit() for char in self.de_call) and not ( + self.de_call.startswith("T2") and self.source == "APRS-IS"): + if not self.de_country: + self.de_country = lookup_helper.infer_country_from_callsign(self.de_call, credentials) + if not self.de_continent: + self.de_continent = lookup_helper.infer_continent_from_callsign(self.de_call, credentials) + if not self.de_dxcc_id: + self.de_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.de_call, credentials) + if self.de_dxcc_id and not self.de_flag: + self.de_flag = lookup_helper.get_flag_for_dxcc(self.de_dxcc_id) - # Remove NaNs in frequency - if self.freq and self.freq == float("nan"): - self.freq = None + # Remove NaNs in frequency + if self.freq and self.freq == float("nan"): + self.freq = None - # Band from frequency - if self.freq and not self.band: - band = infer_band_from_freq(self.freq) - self.band = band.name + # Band from frequency + if self.freq and not self.band: + band = infer_band_from_freq(self.freq) + self.band = band.name - # Mode from comments or bandplan - if self.mode: - self.mode_source = "SPOT" - if self.comment and not self.mode: - self.mode = infer_mode_from_comment(self.comment) - self.mode_source = "COMMENT" - if self.freq and not self.mode: - self.mode = infer_mode_from_frequency(self.freq) - self.mode_source = "BANDPLAN" + # Mode from comments or bandplan + if self.mode: + self.mode_source = "SPOT" + if self.comment and not self.mode: + self.mode = infer_mode_from_comment(self.comment) + self.mode_source = "COMMENT" + if self.freq and not self.mode: + self.mode = infer_mode_from_frequency(self.freq) + self.mode_source = "BANDPLAN" - # Normalise mode if necessary. - if self.mode in MODE_ALIASES: - self.mode = MODE_ALIASES[self.mode] + # Normalise mode if necessary. + if self.mode in MODE_ALIASES: + self.mode = MODE_ALIASES[self.mode] - # Mode type from mode - if self.mode and not self.mode_type: - self.mode_type = infer_mode_type_from_mode(self.mode) + # Mode type from mode + if self.mode and not self.mode_type: + self.mode_type = infer_mode_type_from_mode(self.mode) - # If we have a latitude or grid at this point, it can only have been provided by the spot itself - if self.dx_latitude or self.dx_grid: - self.dx_location_source = "SPOT" + # If we have a latitude or grid at this point, it can only have been provided by the spot itself + if self.dx_latitude or self.dx_grid: + self.dx_location_source = "SPOT" - # Set the top-level "SIG" if it is missing but we have at least one SIG ref. - if not self.sig and self.sig_refs and len(self.sig_refs) > 0: - self.sig = self.sig_refs[0].sig.upper() + # Set the top-level "SIG" if it is missing but we have at least one SIG ref. + if not self.sig and self.sig_refs and len(self.sig_refs) > 0: + self.sig = self.sig_refs[0].sig.upper() - # See if we already have a SIG reference, but the comment looks like it contains more for the same SIG. This - # should catch e.g. POTA comments like "2-fer: GB-0001 GB-0002". - if self.comment and self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0].sig: - sig = self.sig_refs[0].sig.upper() - regex = get_ref_regex_for_sig(sig) - if regex: - all_comment_ref_matches = re.finditer(r"(^|\W)(" + regex + r")(^|\W)", self.comment, re.IGNORECASE) - for ref_match in all_comment_ref_matches: - self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(2).upper(), sig=sig)) + # See if we already have a SIG reference, but the comment looks like it contains more for the same SIG. This + # should catch e.g. POTA comments like "2-fer: GB-0001 GB-0002". + if self.comment and self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0].sig: + sig = self.sig_refs[0].sig.upper() + regex = get_ref_regex_for_sig(sig) + if regex: + all_comment_ref_matches = re.finditer(r"(^|\W)(" + regex + r")(^|\W)", self.comment, re.IGNORECASE) + for ref_match in all_comment_ref_matches: + self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(2).upper(), sig=sig)) - # See if the comment looks like it contains any SIGs (and optionally SIG references) that we can - # add to the spot. This should catch cluster spot comments like "POTA GB-0001 WWFF GFF-0001" and e.g. POTA - # comments like "also WWFF GFF-0001". - if self.comment: - sig_matches = re.finditer(r"(^|\W)" + ANY_SIG_REGEX + r"($|\W)", self.comment, re.IGNORECASE) - for sig_match in sig_matches: - # First of all, if we haven't got a SIG for this spot set yet, now we have. This covers things like cluster - # spots where the comment is just "POTA". - found_sig = get_sig_name_from_comment_name(sig_match.group(2)) - if not self.sig: - self.sig = found_sig + # See if the comment looks like it contains any SIGs (and optionally SIG references) that we can + # add to the spot. This should catch cluster spot comments like "POTA GB-0001 WWFF GFF-0001" and e.g. POTA + # comments like "also WWFF GFF-0001". + if self.comment: + sig_matches = re.finditer(r"(^|\W)" + ANY_SIG_REGEX + r"($|\W)", self.comment, re.IGNORECASE) + for sig_match in sig_matches: + # First of all, if we haven't got a SIG for this spot set yet, now we have. This covers things like cluster + # spots where the comment is just "POTA". + found_sig = get_sig_name_from_comment_name(sig_match.group(2)) + if not self.sig: + self.sig = found_sig - # Now look to see if that SIG name was followed by something that looks like a reference ID for that SIG. - # If so, add that to the sig_refs list for this spot. - ref_regex = get_ref_regex_for_sig(found_sig) - if ref_regex: - ref_matches = re.finditer(r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)", self.comment, - re.IGNORECASE) - for ref_match in ref_matches: - self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig)) + # Now look to see if that SIG name was followed by something that looks like a reference ID for that SIG. + # If so, add that to the sig_refs list for this spot. + ref_regex = get_ref_regex_for_sig(found_sig) + if ref_regex: + ref_matches = re.finditer(r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)", self.comment, + re.IGNORECASE) + for ref_match in ref_matches: + self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig)) - # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference - # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs - # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths - if self.sig_refs and len(self.sig_refs) > 0: - for sig_ref in self.sig_refs: - sig_ref = populate_sig_ref_info(sig_ref) - # If the spot itself doesn't have location yet, but the SIG ref does, extract it - if sig_ref.grid and not self.dx_grid: - self.dx_grid = sig_ref.grid - if sig_ref.latitude and not self.dx_latitude: - self.dx_latitude = sig_ref.latitude - self.dx_longitude = sig_ref.longitude - if self.sig == "WAB" or self.sig == "WAI": - self.dx_location_source = "WAB/WAI GRID" - else: - self.dx_location_source = "SIG REF LOOKUP" + # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference + # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs + # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths + if self.sig_refs and len(self.sig_refs) > 0: + for sig_ref in self.sig_refs: + sig_ref = populate_sig_ref_info(sig_ref) + # If the spot itself doesn't have location yet, but the SIG ref does, extract it + if sig_ref.grid and not self.dx_grid: + self.dx_grid = sig_ref.grid + if sig_ref.latitude and not self.dx_latitude: + self.dx_latitude = sig_ref.latitude + self.dx_longitude = sig_ref.longitude + if self.sig == "WAB" or self.sig == "WAI": + self.dx_location_source = "WAB/WAI GRID" + else: + self.dx_location_source = "SIG REF LOOKUP" - # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG - # and apply it to the whole spot. - if self.sig_refs and len(self.sig_refs) > 0 and not self.sig: - self.sig = self.sig_refs[0].sig + # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG + # and apply it to the whole spot. + if self.sig_refs and len(self.sig_refs) > 0 and not self.sig: + self.sig = self.sig_refs[0].sig - # Parse "de_griddx_grid" structures from the comment, e.g. "JN61ES(ES)JM56XT" or "JO02GQ<>KN17LG". - # These are common on cluster spots and can provide grid references in preference to e.g. QRZ lookup, as well as - # being the only source we have for propagation mode. Brace for nightmare regex from hell. - if self.comment: - grid_mode_grid_match = re.search( - r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b', - self.comment) - if grid_mode_grid_match: - # regex matches, so extract grids: - if not self.de_grid: - self.de_grid = grid_mode_grid_match.group(1).upper() - if not self.dx_grid: - self.dx_grid = grid_mode_grid_match.group(4).upper() - self.dx_location_source = "SPOT" + # Parse "de_griddx_grid" structures from the comment, e.g. "JN61ES(ES)JM56XT" or "JO02GQ<>KN17LG". + # These are common on cluster spots and can provide grid references in preference to e.g. QRZ lookup, as well as + # being the only source we have for propagation mode. Brace for nightmare regex from hell. + if self.comment: + grid_mode_grid_match = re.search( + r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b', + self.comment) + if grid_mode_grid_match: + # regex matches, so extract grids: + if not self.de_grid: + self.de_grid = grid_mode_grid_match.group(1).upper() + if not self.dx_grid: + self.dx_grid = grid_mode_grid_match.group(4).upper() + self.dx_location_source = "SPOT" - # And extract propagation mode (group 2 for <...>, group 3 for (...)): - mode_tag = (grid_mode_grid_match.group(2) or grid_mode_grid_match.group(3) or "").upper() - if mode_tag and not self.propagation_mode: - if mode_tag in PROPAGATION_MODES: - self.propagation_mode = PROPAGATION_MODES[mode_tag] - else: - self.propagation_mode = mode_tag - logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag) + # And extract propagation mode (group 2 for <...>, group 3 for (...)): + mode_tag = (grid_mode_grid_match.group(2) or grid_mode_grid_match.group(3) or "").upper() + if mode_tag and not self.propagation_mode: + if mode_tag in PROPAGATION_MODES: + self.propagation_mode = PROPAGATION_MODES[mode_tag] + else: + self.propagation_mode = mode_tag + logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag) - # DX Grid to lat/lon and vice versa in case one is missing - if self.dx_grid and not self.dx_latitude: - try: - ll = locator_to_latlong(self.dx_grid) - self.dx_latitude = ll[0] - self.dx_longitude = ll[1] - except: - logging.debug("Invalid grid received for spot") - if self.dx_latitude and self.dx_longitude and not self.dx_grid: - try: - self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8) - except: - logging.debug("Invalid lat/lon received for spot") + # DX Grid to lat/lon and vice versa in case one is missing + if self.dx_grid and not self.dx_latitude: + try: + ll = locator_to_latlong(self.dx_grid) + self.dx_latitude = ll[0] + self.dx_longitude = ll[1] + except: + logging.debug("Invalid grid received for spot") + if self.dx_latitude and self.dx_longitude and not self.dx_grid: + try: + self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8) + except: + logging.debug("Invalid lat/lon received for spot") - # QRT comment detection - if self.comment and not self.qrt: - self.qrt = "QRT" in self.comment.upper() + # QRT comment detection + if self.comment and not self.qrt: + self.qrt = "QRT" in self.comment.upper() - # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index - # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical - # apart from that they were retrieved from the API at different times. Note that the simple Python hash() - # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we - # use diskcache to store our data between runs, so we use SHA256 which does not include this random element. - # The ID is computed before the online lookups below so that it is stable regardless of whether credentials - # are provided, allowing the enriched API response to be matched to the stored spot by ID. - if not self.id: - self_copy = copy.deepcopy(self) - self_copy.received_time = 0 - self_copy.received_time_iso = "" - self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest() + # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index + # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical + # apart from that they were retrieved from the API at different times. Note that the simple Python hash() + # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we + # use diskcache to store our data between runs, so we use SHA256 which does not include this random element. + # The ID is computed before the online lookups below so that it is stable regardless of whether credentials + # are provided, allowing the enriched API response to be matched to the stored spot by ID. + if not self.id: + self_copy = copy.deepcopy(self) + self_copy.received_time = 0 + self_copy.received_time_iso = "" + self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest() - # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data - # from the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon - # instead of the one from the park reference they're at. - if self.dx_call and not self.dx_name: - self.dx_name = lookup_helper.infer_name_from_callsign_online_lookup(self.dx_call, credentials) - if self.dx_call and not self.dx_latitude: - latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.dx_call, credentials) - if latlon: - self.dx_latitude = latlon[0] - self.dx_longitude = latlon[1] - self.dx_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.dx_call, credentials) - self.dx_location_source = "HOME QTH" - - # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable string, - # otherwise see what they have set on an online lookup service. - if self.sig_refs and len(self.sig_refs) > 0: - qth = self.sig_refs[0].id - if self.sig_refs[0].name: - qth += " " + self.sig_refs[0].name - self.dx_qth = qth - else: - self.dx_qth = lookup_helper.infer_qth_from_callsign_online_lookup(self.dx_call, credentials) - - # Last resort for getting a DX position, use the DXCC entity. - if self.dx_call and not self.dx_latitude: - latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.dx_call) - if latlon: - self.dx_latitude = latlon[0] - self.dx_longitude = latlon[1] - self.dx_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.dx_call) - self.dx_location_source = "DXCC" - - # It looks like we can sometimes get a string into lat/lon, so try to parse as float, reject if not valid - if isinstance(self.dx_latitude, str) or isinstance(self.dx_longitude, str): - try: - self.dx_latitude = float(str(self.dx_latitude)) - self.dx_longitude = float(str(self.dx_longitude)) - except (TypeError, ValueError): - logging.warning("Received non-numeric strings in lat/lon (" + str(self.dx_latitude) + ", " + str( - self.dx_longitude) + ") for call " + str(self.dx_call) + ", rejecting it") - self.dx_latitude = None - self.dx_longitude = None - - # CQ and ITU zone lookup, preferably from location but failing that, from callsign - if not self.dx_cq_zone: - if self.dx_latitude: - self.dx_cq_zone = lat_lon_to_cq_zone(self.dx_latitude, self.dx_longitude) - elif self.dx_call: - self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_call, credentials) - if not self.dx_itu_zone: - if self.dx_latitude: - self.dx_itu_zone = lat_lon_to_itu_zone(self.dx_latitude, self.dx_longitude) - elif self.dx_call: - self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_call, credentials) - - # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator - # is likely at home. - self.dx_location_good = bool(self.dx_latitude and self.dx_longitude and ( - self.dx_location_source == "SPOT" or self.dx_location_source == "SIG REF LOOKUP" - or self.dx_location_source == "WAB/WAI GRID" - or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or "")))) - - # DE with no digits and APRS servers starting "T2" are not things we can look up location for - if self.de_call and any(char.isdigit() for char in self.de_call) and not ( - self.de_call.startswith("T2") and self.source == "APRS-IS"): - # DE operator position lookup, using QRZ.com/HamQTH. - if not self.de_latitude: - latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.de_call, credentials) + # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data + # from the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon + # instead of the one from the park reference they're at. + if self.dx_call and not self.dx_name: + self.dx_name = lookup_helper.infer_name_from_callsign_online_lookup(self.dx_call, credentials) + if self.dx_call and not self.dx_latitude: + latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.dx_call, credentials) if latlon: - self.de_latitude = latlon[0] - self.de_longitude = latlon[1] - self.de_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.de_call, credentials) + self.dx_latitude = latlon[0] + self.dx_longitude = latlon[1] + self.dx_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.dx_call, credentials) + self.dx_location_source = "HOME QTH" - # Last resort for getting a DE position, use the DXCC entity. - if not self.de_latitude: - latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.de_call) + # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable string, + # otherwise see what they have set on an online lookup service. + if self.sig_refs and len(self.sig_refs) > 0: + qth = self.sig_refs[0].id + if self.sig_refs[0].name: + qth += " " + self.sig_refs[0].name + self.dx_qth = qth + else: + self.dx_qth = lookup_helper.infer_qth_from_callsign_online_lookup(self.dx_call, credentials) + + # Last resort for getting a DX position, use the DXCC entity. + if self.dx_call and not self.dx_latitude: + latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.dx_call) if latlon: - self.de_latitude = latlon[0] - self.de_longitude = latlon[1] - self.de_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.de_call) + self.dx_latitude = latlon[0] + self.dx_longitude = latlon[1] + self.dx_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.dx_call) + self.dx_location_source = "DXCC" + + # It looks like we can sometimes get a string into lat/lon, so try to parse as float, reject if not valid + if isinstance(self.dx_latitude, str) or isinstance(self.dx_longitude, str): + try: + self.dx_latitude = float(str(self.dx_latitude)) + self.dx_longitude = float(str(self.dx_longitude)) + except (TypeError, ValueError): + logging.warning("Received non-numeric strings in lat/lon (" + str(self.dx_latitude) + ", " + str( + self.dx_longitude) + ") for call " + str(self.dx_call) + ", rejecting it") + self.dx_latitude = None + self.dx_longitude = None + + # CQ and ITU zone lookup, preferably from location but failing that, from callsign + if not self.dx_cq_zone: + if self.dx_latitude: + self.dx_cq_zone = lat_lon_to_cq_zone(self.dx_latitude, self.dx_longitude) + elif self.dx_call: + self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_call, credentials) + if not self.dx_itu_zone: + if self.dx_latitude: + self.dx_itu_zone = lat_lon_to_itu_zone(self.dx_latitude, self.dx_longitude) + elif self.dx_call: + self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_call, credentials) + + # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator + # is likely at home. + self.dx_location_good = bool(self.dx_latitude and self.dx_longitude and ( + self.dx_location_source == "SPOT" or self.dx_location_source == "SIG REF LOOKUP" + or self.dx_location_source == "WAB/WAI GRID" + or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or "")))) + + # DE with no digits and APRS servers starting "T2" are not things we can look up location for + if self.de_call and any(char.isdigit() for char in self.de_call) and not ( + self.de_call.startswith("T2") and self.source == "APRS-IS"): + # DE operator position lookup, using QRZ.com/HamQTH. + if not self.de_latitude: + latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.de_call, credentials) + if latlon: + self.de_latitude = latlon[0] + self.de_longitude = latlon[1] + self.de_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.de_call, credentials) + + # Last resort for getting a DE position, use the DXCC entity. + if not self.de_latitude: + latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.de_call) + if latlon: + self.de_latitude = latlon[0] + self.de_longitude = latlon[1] + self.de_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.de_call) + + except Exception as e: + logging.error("Exception while inferring missing data from spot", e, exc_info=True) def to_json(self): """JSON serialise""" diff --git a/datafiles/tota.csv b/datafiles/toilets.csv similarity index 100% rename from datafiles/tota.csv rename to datafiles/toilets.csv diff --git a/server/webserver.py b/server/webserver.py index 8ba1c93..131b11d 100644 --- a/server/webserver.py +++ b/server/webserver.py @@ -72,7 +72,8 @@ class WebServer: {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts}), (r"/api/v1/alerts/stream", APIAlertsStreamHandler, {"sse_alert_broadcaster": self._alert_broadcaster, **handler_opts}), - (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._data_store.solar, **handler_opts}), + (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._data_store.solar_conditions, + **handler_opts}), (r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._data_store.spots, **handler_opts}), (r"/api/v1/options", APIOptionsHandler, {"status_data": self._data_store.status_data, **handler_opts}), (r"/api/v1/status", APIStatusHandler, {"status_data": self._data_store.status_data, **handler_opts}), diff --git a/sigrefdataproviders/arlhs.py b/sigrefdataproviders/arlhs.py new file mode 100644 index 0000000..20d8a2a --- /dev/null +++ b/sigrefdataproviders/arlhs.py @@ -0,0 +1,30 @@ +import csv + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class ARLHS(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Amateur Radio Light House Society""" + + POLL_INTERVAL_DAYS = 30 + SIG = "ARLHS" + DATA_URL = "https://www.gma.rocks/download/lighthouse.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): + if "ARLHS" in row: + ref_id = row["ARLHS"] + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, + url="https://www.cqgma.org/zinfo.php?ref=" + ref_id, + latitude=float(row["Latitude"]) if "Latitude" in row and row[ + "Latitude"] != "" else None, + longitude=float(row["Longitude"]) if "Longitude" in row and row[ + "Longitude"] != "" else None, + grid=row["Maidenhead Locator"]) + + return new_data diff --git a/sigrefdataproviders/file_download_sig_ref_data_provider.py b/sigrefdataproviders/file_download_sig_ref_data_provider.py index 9f46bc1..364b564 100644 --- a/sigrefdataproviders/file_download_sig_ref_data_provider.py +++ b/sigrefdataproviders/file_download_sig_ref_data_provider.py @@ -3,11 +3,11 @@ from datetime import datetime from threading import Thread, Event import pytz -import requests from requests import ReadTimeout from requests.exceptions import ConnectionError, ConnectTimeout from core.constants import HTTP_HEADERS +from core.url_data_cache import URL_DATA_CACHE from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider @@ -15,6 +15,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider): """Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file.""" def __init__(self, sig_name, provider_config, url, poll_interval): + """ Set up the provider, note poll_interval is in *days*.""" super().__init__(sig_name, provider_config) self._url = url self._poll_interval = poll_interval @@ -25,7 +26,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider): # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between # subsequent polls, so start() returns immediately and the application can continue starting. logging.info( - "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " seconds.") + "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " days.") self._thread = Thread(target=self._run, daemon=True) self._thread.start() @@ -35,14 +36,15 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider): def _run(self): while True: self._poll() - if self._stop_event.wait(timeout=self._poll_interval): + if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24): break def _poll(self): try: - # Request data from API + # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for + # caching, but just so continual restarts of the software during testing don't hammer the servers. logging.debug("Downloading " + self.sig_name + " SIG ref data...") - http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30)) + http_response = URL_DATA_CACHE.get(self._url, headers=HTTP_HEADERS) # Check response code was good if http_response.ok: # Pass off to the subclass for processing diff --git a/sigrefdataproviders/gma.py b/sigrefdataproviders/gma.py new file mode 100644 index 0000000..ae58549 --- /dev/null +++ b/sigrefdataproviders/gma.py @@ -0,0 +1,29 @@ +import csv + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class GMA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Global Mountain Activity""" + + POLL_INTERVAL_DAYS = 30 + SIG = "GMA" + DATA_URL = "https://www.gma.rocks/download/summits.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): + ref_id = row["Reference"] + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, + url="https://www.cqgma.org/zinfo.php?ref=" + ref_id, + latitude=float(row["Latitude"]) if "Latitude" in row and row[ + "Latitude"] != "" else None, + longitude=float(row["Longitude"]) if "Longitude" in row and row[ + "Longitude"] != "" else None, + grid=row["Maidenhead Locator"]) + + return new_data diff --git a/sigrefdataproviders/illw.py b/sigrefdataproviders/illw.py new file mode 100644 index 0000000..af17391 --- /dev/null +++ b/sigrefdataproviders/illw.py @@ -0,0 +1,30 @@ +import csv + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class ILLW(FileDownloadSIGRefDataProvider): + """SIG ref data provider for International Lighthouse & Lightship Weekend""" + + POLL_INTERVAL_DAYS = 30 + SIG = "ILLW" + DATA_URL = "https://www.gma.rocks/download/lighthouse.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): + if "ILLW" in row: + ref_id = row["ILLW"] + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, + url="https://www.cqgma.org/zinfo.php?ref=" + ref_id, + latitude=float(row["Latitude"]) if "Latitude" in row and row[ + "Latitude"] != "" else None, + longitude=float(row["Longitude"]) if "Longitude" in row and row[ + "Longitude"] != "" else None, + grid=row["Maidenhead Locator"]) + + return new_data diff --git a/sigrefdataproviders/iota.py b/sigrefdataproviders/iota.py new file mode 100644 index 0000000..3213a24 --- /dev/null +++ b/sigrefdataproviders/iota.py @@ -0,0 +1,36 @@ +import logging + +from pyhamtools.locator import latlong_to_locator + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class IOTA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Islands on the Air""" + + POLL_INTERVAL_DAYS = 365 + SIG = "IOTA" + DATA_URL = "https://www.iota-world.org/islands-on-the-air/downloads/download-file.html?path=groups.json" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) + + def _http_response_to_data(self, http_response): + new_data = {} + data = http_response.json() + if isinstance(data, list): + for ref in data: + ref_id = ref["refno"] + latitude = float(ref["latitude_min"]) + float(ref["latitude_max"]) / 2.0 + longitude = float(ref["longitude_min"]) + float(ref["longitude_max"]) / 2.0 + grid = None + try: + grid = latlong_to_locator(latitude, longitude, 6) + except ValueError: + logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude) + + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"], + grid=grid, latitude=latitude, longitude=longitude) + + return new_data diff --git a/sigrefdataproviders/llota.py b/sigrefdataproviders/llota.py index e9cc8ce..d0513d4 100644 --- a/sigrefdataproviders/llota.py +++ b/sigrefdataproviders/llota.py @@ -7,12 +7,12 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload class LLOTA(FileDownloadSIGRefDataProvider): """SIG ref data provider for Lagos y Lagunas on the Air""" - POLL_INTERVAL_SEC = 7 * 24 * 60 * 60 # 7 days + POLL_INTERVAL_DAYS = 7 SIG = "LLOTA" DATA_URL = "https://llota.app/api/public/references" def __init__(self, provider_config): - super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC) + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): new_data = {} diff --git a/sigrefdataproviders/local_file_sig_ref_data_provider.py b/sigrefdataproviders/local_file_sig_ref_data_provider.py index ba427d3..780a0a6 100644 --- a/sigrefdataproviders/local_file_sig_ref_data_provider.py +++ b/sigrefdataproviders/local_file_sig_ref_data_provider.py @@ -14,7 +14,7 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider): self._path = path def start(self): - logging.info("Loading " + self.sig_name + " SIG ref data from file.") + logging.debug("Loading " + self.sig_name + " SIG ref data from file.") try: new_data = self._file_to_data(self._path) if new_data: diff --git a/sigrefdataproviders/mota.py b/sigrefdataproviders/mota.py new file mode 100644 index 0000000..f5579dc --- /dev/null +++ b/sigrefdataproviders/mota.py @@ -0,0 +1,29 @@ +import csv + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class MOTA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Mills on the Air""" + + POLL_INTERVAL_DAYS = 30 + SIG = "MOTA" + DATA_URL = "https://www.gma.rocks/download/mills.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): + ref_id = row["Reference"] + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, + url="https://www.cqgma.org/zinfo.php?ref=" + ref_id, + latitude=float(row["Latitude"]) if "Latitude" in row and row[ + "Latitude"] != "" else None, + longitude=float(row["Longitude"]) if "Longitude" in row and row[ + "Longitude"] != "" else None, + grid=row["Maidenhead Locator"]) + + return new_data diff --git a/sigrefdataproviders/pota.py b/sigrefdataproviders/pota.py new file mode 100644 index 0000000..5a4e87c --- /dev/null +++ b/sigrefdataproviders/pota.py @@ -0,0 +1,29 @@ +import csv + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class POTA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Parks on the Air""" + + POLL_INTERVAL_DAYS = 7 + SIG = "POTA" + DATA_URL = "https://pota.app/all_parks_ext.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): + ref_id = row["reference"] + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None, + url="https://pota.app/#/park/" + ref_id, + grid=row["grid"] if "grid" in row else None, + latitude=float(row["latitude"]) if "latitude" in row and row[ + "latitude"] != "" else None, + longitude=float(row["longitude"]) if "longitude" in row and row[ + "longitude"] != "" else None) + + return new_data diff --git a/sigrefdataproviders/sig_ref_data_provider.py b/sigrefdataproviders/sig_ref_data_provider.py index 96e265b..9664731 100644 --- a/sigrefdataproviders/sig_ref_data_provider.py +++ b/sigrefdataproviders/sig_ref_data_provider.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime import pytz @@ -16,6 +17,7 @@ class SIGRefDataProvider: self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC) self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC) self.status = "Not Started" if self.enabled else "Disabled" + self.reference_count = 0 # Create an empty dict to store data if one doesn't already exist if not sig_name in DATA_STORE.sigrefs: @@ -38,3 +40,5 @@ class SIGRefDataProvider: objects.""" DATA_STORE.sigrefs[self.sig_name] = new_data + self.reference_count = len(new_data) + logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name) diff --git a/sigrefdataproviders/siota.py b/sigrefdataproviders/siota.py index 8d60272..9b20593 100644 --- a/sigrefdataproviders/siota.py +++ b/sigrefdataproviders/siota.py @@ -7,16 +7,16 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload class SIOTA(FileDownloadSIGRefDataProvider): """SIG ref data provider for Silos on the Air""" - POLL_INTERVAL_SEC = 30 * 24 * 60 * 60 # 30 days + POLL_INTERVAL_DAYS = 30 SIG = "SIOTA" DATA_URL = "https://www.silosontheair.com/data/silos.csv" def __init__(self, provider_config): - super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC) + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): new_data = {} - for row in csv.DictReader(http_response.content.decode().splitlines()): + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): ref_id = row["SILO_CODE"] new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None, grid=row["LOCATOR"] if "LOCATOR" in row else None, diff --git a/sigrefdataproviders/sota.py b/sigrefdataproviders/sota.py new file mode 100644 index 0000000..2f5bdd5 --- /dev/null +++ b/sigrefdataproviders/sota.py @@ -0,0 +1,33 @@ +import csv + +from pyhamtools.locator import latlong_to_locator + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class SOTA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Summits on the Air""" + + POLL_INTERVAL_DAYS = 30 + SIG = "SOTA" + DATA_URL = "https://storage.sota.org.uk/summitslist.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): + ref_id = row["SummitCode"] + latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None + longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None, + url="https://www.sotadata.org.uk/en/summit/" + ref_id, + latitude=latitude, + longitude=longitude, + activation_score=int(row["Points"]) if "Points" in row else None) + if latitude and longitude: + new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6) + + return new_data diff --git a/sigrefdataproviders/tota.py b/sigrefdataproviders/toilets.py similarity index 87% rename from sigrefdataproviders/tota.py rename to sigrefdataproviders/toilets.py index ad7b546..7bffab3 100644 --- a/sigrefdataproviders/tota.py +++ b/sigrefdataproviders/toilets.py @@ -4,11 +4,11 @@ from data.sig_ref import SIGRef from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider -class TOTA(LocalFileSIGRefDataProvider): +class Toilets(LocalFileSIGRefDataProvider): """SIG ref data provider for Toilets on the Air""" - SIG = "TOTA" - PATH = "datafiles/tota.csv" + SIG = "Toilets" + PATH = "datafiles/toilets.csv" def __init__(self, provider_config): super().__init__(self.SIG, provider_config, self.PATH) diff --git a/sigrefdataproviders/towers.py b/sigrefdataproviders/towers.py new file mode 100644 index 0000000..bf122f4 --- /dev/null +++ b/sigrefdataproviders/towers.py @@ -0,0 +1,27 @@ +import csv + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class Towers(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Towers on the Air""" + + POLL_INTERVAL_DAYS = 30 + SIG = "Towers" + DATA_URL = "https://wwtota.com/servis/generate_csv.php?ref=&filter=all" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"): + ref_id = row["Ref"] + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None, + url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id, + grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None, + latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None, + longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None) + + return new_data diff --git a/sigrefdataproviders/wca.py b/sigrefdataproviders/wca.py new file mode 100644 index 0000000..07a85ff --- /dev/null +++ b/sigrefdataproviders/wca.py @@ -0,0 +1,44 @@ +import csv +import logging + +from pyhamtools.locator import latlong_to_locator + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class WCA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for World Castles Award""" + + POLL_INTERVAL_DAYS = 30 + SIG = "WCA" + DATA_URL = "https://polo.ham2k.com/data/activities/wca/all-castles.csv" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): + ref_id = row["REF"] + + coordsStr = row["COORDINATES"] + latitude = None + longitude = None + grid = None + try: + if coordsStr: + split = coordsStr.split(", ") + latitude = float(split[0]) + longitude = float(split[1]) + grid = latlong_to_locator(latitude, longitude) + except ValueError: + logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id) + + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None, + url="https://www.cqgma.org/zinfo.php?ref=" + ref_id, + latitude=latitude, + longitude=longitude, + grid=grid) + + return new_data diff --git a/sigrefdataproviders/wota.py b/sigrefdataproviders/wota.py index 4e2b8a8..c221845 100644 --- a/sigrefdataproviders/wota.py +++ b/sigrefdataproviders/wota.py @@ -5,12 +5,12 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload class WOTA(FileDownloadSIGRefDataProvider): """SIG ref data provider for Wainwrights on the Air""" - POLL_INTERVAL_SEC = 365 * 24 * 60 * 60 # 365 days + POLL_INTERVAL_DAYS = 365 SIG = "WOTA" DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json" def __init__(self, provider_config): - super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC) + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): new_data = {} diff --git a/sigrefdataproviders/wwbota.py b/sigrefdataproviders/wwbota.py new file mode 100644 index 0000000..52dd296 --- /dev/null +++ b/sigrefdataproviders/wwbota.py @@ -0,0 +1,27 @@ +import csv + +from data.sig_ref import SIGRef +from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider + + +class WWBOTA(FileDownloadSIGRefDataProvider): + """SIG ref data provider for Worldwide Bunkers on the Air""" + + POLL_INTERVAL_DAYS = 30 + SIG = "WWBOTA" + DATA_URL = "https://api.wwbota.org/bunkers/?format=CSV" + + def __init__(self, provider_config): + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) + + def _http_response_to_data(self, http_response): + new_data = {} + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): + ref_id = row["Reference"] + new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, + url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None, + grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None, + latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None, + longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None) + + return new_data diff --git a/sigrefdataproviders/wwff.py b/sigrefdataproviders/wwff.py index 492bc91..bb8f033 100644 --- a/sigrefdataproviders/wwff.py +++ b/sigrefdataproviders/wwff.py @@ -7,16 +7,16 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload class WWFF(FileDownloadSIGRefDataProvider): """SIG ref data provider for Worldwide Flora & Fauna""" - POLL_INTERVAL_SEC = 30 * 24 * 60 * 60 # 30 days + POLL_INTERVAL_DAYS = 30 SIG = "WWFF" DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv" def __init__(self, provider_config): - super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC) + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): new_data = {} - for row in csv.DictReader(http_response.content.decode().splitlines()): + for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): ref_id = row["reference"] new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None, url="https://wwff.co/directory/?showRef=" + ref_id, diff --git a/sigrefdataproviders/zlota.py b/sigrefdataproviders/zlota.py index 84546df..6323f4d 100644 --- a/sigrefdataproviders/zlota.py +++ b/sigrefdataproviders/zlota.py @@ -7,12 +7,12 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload class ZLOTA(FileDownloadSIGRefDataProvider): """SIG ref data provider for New Zealand on the Air""" - POLL_INTERVAL_SEC = 30 * 24 * 60 * 60 # 30 days + POLL_INTERVAL_DAYS = 30 SIG = "ZLOTA" DATA_URL = "https://ontheair.nz/assets/assets.json" def __init__(self, provider_config): - super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC) + super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): new_data = {} diff --git a/spotproviders/towers.py b/spotproviders/towers.py index 6d09022..2553872 100644 --- a/spotproviders/towers.py +++ b/spotproviders/towers.py @@ -30,8 +30,8 @@ class Towers(HTTPSpotProvider): dx_call=source_spot["call"].upper(), freq=likely_freq, comment=source_spot["comment"], - sig="WWTOTA", - sig_refs=[SIGRef(id=source_spot["ref"], sig="WWTOTA")], + sig="Towers", + sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")], time=datetime.strptime(response_json["updated"][:10] + source_spot["time"], "%Y-%m-%d%H:%M").timestamp()) diff --git a/spotproviders/wota.py b/spotproviders/wota.py index 079968f..f8321c4 100644 --- a/spotproviders/wota.py +++ b/spotproviders/wota.py @@ -25,7 +25,7 @@ class WOTA(HTTPSpotProvider): def _http_response_to_spots(self, http_response): new_spots = [] - rss = cast(RSS, Parser.parse(http_response.content.decode())) + rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig"))) # Iterate through source data for source_spot in rss.channel.items: diff --git a/spotproviders/xota.py b/spotproviders/xota.py index 14e7526..95f5843 100644 --- a/spotproviders/xota.py +++ b/spotproviders/xota.py @@ -11,7 +11,7 @@ from spotproviders.websocket_spot_provider import WebsocketSpotProvider class XOTA(WebsocketSpotProvider): """Spot provider for servers based on the "xOTA" software at https://github.com/nischu/xOTA/ The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG which we can - then use for lookups. This functionality is implemented for TOTA events, of which there are + then use for lookups. This functionality is implemented for Toilets on the Air events, of which there are several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA, which is why we also provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3 might become "C3 T-01". This allows us to provide location lookups for TOTA at several conferences.""" diff --git a/static/apidocs/openapi.yml b/static/apidocs/openapi.yml index abe3b6b..4b852ad 100644 --- a/static/apidocs/openapi.yml +++ b/static/apidocs/openapi.yml @@ -796,7 +796,7 @@ components: - HEMA - WCA - MOTA - - SiOTA + - SIOTA - ARLHS - ILLW - ZLOTA @@ -1742,6 +1742,10 @@ components: The last time at which this provider received data, UTC seconds since UNIX epoch. If this is zero, the provider has never updated. example: 1759579508 + reference_count: + type: number + description: The number of references fetched using this provider. + example: 1234 SpotList: type: array diff --git a/static/js/status.js b/static/js/status.js index 2d8a977..a4ac8da 100644 --- a/static/js/status.js +++ b/static/js/status.js @@ -46,6 +46,7 @@ function loadStatus() {
${p["sig_name"]}
Status: ${p["status"]}
Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}
+
Reference count: ${(p["enabled"] && p["reference_count"] > 0) ? p["reference_count"] : "N/A"}
`); }); });