diff --git a/config-example.yml b/config-example.yml index 9a4e0c4..ec81825 100644 --- a/config-example.yml +++ b/config-example.yml @@ -192,15 +192,6 @@ web-server-port: 8080 max-spot-age-sec: 3600 max-alert-age-sec: 604800 -# Login for QRZ.com to look up information. Optional. You will need an "XML Subscriber" (paid) package to retrieve all -# the data for a callsign via their system. -qrz-username: "" -qrz-password: "" - -# Login for HamQTH to look up information. Optional. -hamqth-username: "" -hamqth-password: "" - # API key for Clublog to look up information. Optional. You sill need to request one via their helpdesk portal if you # want to use callsign lookups from Clublog. clublog-api-key: "" diff --git a/core/lookup_helper.py b/core/lookup_helper.py index 7f68d27..a0a340a 100644 --- a/core/lookup_helper.py +++ b/core/lookup_helper.py @@ -5,6 +5,7 @@ import re import urllib.parse from datetime import timedelta +import requests import xmltodict from diskcache import Cache from pyhamtools import LookupLib, Callinfo, callinfo @@ -17,6 +18,38 @@ from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE from core.config import config from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \ HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES +from data.lookup_credentials import LookupCredentials + +# QRZ XML field names differ from pyhamtools' normalised names; map them here. +_QRZ_FIELD_MAP = { + "lat": "latitude", + "lon": "longitude", + "grid": "locator", + "ituzone": "ituz", + "cqzone": "cqz", +} +_QRZ_INT_FIELDS = {"adif", "cqz", "ituz"} +_QRZ_FLOAT_FIELDS = {"latitude", "longitude"} + + +def _normalize_qrz_data(raw): + data = {} + for k, v in raw.items(): + if v is None: + continue + mapped_key = _QRZ_FIELD_MAP.get(k, k) + if mapped_key in _QRZ_INT_FIELDS: + try: + v = int(v) + except (ValueError, TypeError): + pass + elif mapped_key in _QRZ_FLOAT_FIELDS: + try: + v = float(v) + except (ValueError, TypeError): + pass + data[mapped_key] = v + return data class LookupHelper: @@ -36,9 +69,10 @@ class LookupHelper: self._clublog_cty_xml_cache = None self._clublog_api_key = None self._qrz_callsign_data_cache = None - self._lookup_lib_qrz = None - self._qrz_available = None - self._hamqth_available = None + self._qrz_base_url = "https://xmldata.qrz.com/xml/current/" + # QRZ session keys expire after an hour; cache the login response for 55 minutes. + self._qrz_session_cache = CachedSession("cache/qrz_session_cache", + expire_after=timedelta(minutes=55)) self._hamqth_callsign_data_cache = None self._hamqth_base_url = "https://www.hamqth.com/xml.php" # HamQTH session keys expire after an hour. Rather than working out how much time has passed manually, we cheat @@ -67,13 +101,8 @@ class LookupHelper: self._lookup_lib_basic = LookupLib(lookuptype="countryfile") self._call_info_basic = Callinfo(self._lookup_lib_basic) - self._qrz_available = config["qrz-username"] != "" and config["qrz-password"] != "" - if self._qrz_available: - self._lookup_lib_qrz = LookupLib(lookuptype="qrz", username=config["qrz-username"], - pwd=config["qrz-password"]) self._qrz_callsign_data_cache = Cache('cache/qrz_callsign_lookup_cache') - self._hamqth_available = config["hamqth-username"] != "" and config["hamqth-password"] != "" self._hamqth_callsign_data_cache = Cache('cache/hamqth_callsign_lookup_cache') self._clublog_api_key = config["clublog-api-key"] @@ -166,7 +195,7 @@ class LookupHelper: logging.error("Exception when downloading Clublog cty.xml", e) return False - def infer_country_from_callsign(self, call): + def infer_country_from_callsign(self, call, credentials=None): """Infer a country name from a callsign""" try: @@ -176,12 +205,12 @@ class LookupHelper: country = None # Couldn't get anything from basic call info database, try QRZ.com if not country: - qrz_data = self._get_qrz_data_for_callsign(call) + qrz_data = self._get_qrz_data_for_callsign(call, credentials) if qrz_data and "country" in qrz_data: country = qrz_data["country"] # Couldn't get anything from QRZ.com database, try HamQTH if not country: - hamqth_data = self._get_hamqth_data_for_callsign(call) + hamqth_data = self._get_hamqth_data_for_callsign(call, credentials) if hamqth_data and "country" in hamqth_data: country = hamqth_data["country"] # Couldn't get anything from HamQTH database, try Clublog data @@ -200,7 +229,7 @@ class LookupHelper: country = dxcc_data["name"] return country - def infer_dxcc_id_from_callsign(self, call): + def infer_dxcc_id_from_callsign(self, call, credentials=None): """Infer a DXCC ID from a callsign""" try: @@ -210,12 +239,12 @@ class LookupHelper: dxcc = None # Couldn't get anything from basic call info database, try QRZ.com if not dxcc: - qrz_data = self._get_qrz_data_for_callsign(call) + qrz_data = self._get_qrz_data_for_callsign(call, credentials) if qrz_data and "adif" in qrz_data: dxcc = qrz_data["adif"] # Couldn't get anything from QRZ.com database, try HamQTH if not dxcc: - hamqth_data = self._get_hamqth_data_for_callsign(call) + hamqth_data = self._get_hamqth_data_for_callsign(call, credentials) if hamqth_data and "adif" in hamqth_data: dxcc = hamqth_data["adif"] # Couldn't get anything from HamQTH database, try Clublog data @@ -234,7 +263,7 @@ class LookupHelper: dxcc = dxcc_data["entityCode"] return dxcc - def infer_continent_from_callsign(self, call): + def infer_continent_from_callsign(self, call, credentials=None): """Infer a continent shortcode from a callsign""" try: @@ -244,7 +273,7 @@ class LookupHelper: continent = None # Couldn't get anything from basic call info database, try HamQTH if not continent: - hamqth_data = self._get_hamqth_data_for_callsign(call) + hamqth_data = self._get_hamqth_data_for_callsign(call, credentials) if hamqth_data and "continent" in hamqth_data: continent = hamqth_data["continent"] # Couldn't get anything from HamQTH database, try Clublog data @@ -264,7 +293,7 @@ class LookupHelper: continent = dxcc_data["continent"][0] return continent - def infer_cq_zone_from_callsign(self, call): + def infer_cq_zone_from_callsign(self, call, credentials=None): """Infer a CQ zone from a callsign""" try: @@ -274,12 +303,12 @@ class LookupHelper: cqz = None # Couldn't get anything from basic call info database, try QRZ.com if not cqz: - qrz_data = self._get_qrz_data_for_callsign(call) + qrz_data = self._get_qrz_data_for_callsign(call, credentials) if qrz_data and "cqz" in qrz_data: cqz = qrz_data["cqz"] # Couldn't get anything from QRZ.com database, try HamQTH if not cqz: - hamqth_data = self._get_hamqth_data_for_callsign(call) + hamqth_data = self._get_hamqth_data_for_callsign(call, credentials) if hamqth_data and "cq" in hamqth_data: cqz = hamqth_data["cq"] # Couldn't get anything from HamQTH database, try Clublog data @@ -299,7 +328,7 @@ class LookupHelper: cqz = dxcc_data["cq"][0] return cqz - def infer_itu_zone_from_callsign(self, call): + def infer_itu_zone_from_callsign(self, call, credentials=None): """Infer a ITU zone from a callsign""" try: @@ -309,12 +338,12 @@ class LookupHelper: ituz = None # Couldn't get anything from basic call info database, try QRZ.com if not ituz: - qrz_data = self._get_qrz_data_for_callsign(call) + qrz_data = self._get_qrz_data_for_callsign(call, credentials) if qrz_data and "ituz" in qrz_data: ituz = qrz_data["ituz"] # Couldn't get anything from QRZ.com database, try HamQTH if not ituz: - hamqth_data = self._get_hamqth_data_for_callsign(call) + hamqth_data = self._get_hamqth_data_for_callsign(call, credentials) if hamqth_data and "itu" in hamqth_data: ituz = hamqth_data["itu"] # Couldn't get anything from HamQTH database, Clublog doesn't provide this, so try DXCC data @@ -330,31 +359,31 @@ class LookupHelper: return self._dxcc_data[dxcc]["flag"] if dxcc in self._dxcc_data else None - def infer_name_from_callsign_online_lookup(self, call): + def infer_name_from_callsign_online_lookup(self, call, credentials=None): """Infer an operator name from a callsign (requires QRZ.com/HamQTH)""" - data = self._get_qrz_data_for_callsign(call) + data = self._get_qrz_data_for_callsign(call, credentials) if data and "fname" in data: name = data["fname"] if "name" in data: name = name + " " + data["name"] return name - data = self._get_hamqth_data_for_callsign(call) + data = self._get_hamqth_data_for_callsign(call, credentials) if data and "nick" in data: return data["nick"] else: return None - def infer_latlon_from_callsign_online_lookup(self, call): + def infer_latlon_from_callsign_online_lookup(self, call, credentials=None): """Infer a latitude and longitude from a callsign (requires QRZ.com/HamQTH) Coordinates that look default are rejected (apologies if your position really is 0,0, enjoy your voyage)""" - data = self._get_qrz_data_for_callsign(call) + data = self._get_qrz_data_for_callsign(call, credentials) if data and "latitude" in data and "longitude" in data and ( float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float( data["latitude"]) < 89.9: return [float(data["latitude"]), float(data["longitude"])] - data = self._get_hamqth_data_for_callsign(call) + data = self._get_hamqth_data_for_callsign(call, credentials) if data and "latitude" in data and "longitude" in data and ( float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float( data["latitude"]) < 89.9: @@ -362,28 +391,28 @@ class LookupHelper: else: return None - def infer_grid_from_callsign_online_lookup(self, call): + def infer_grid_from_callsign_online_lookup(self, call, credentials=None): """Infer a grid locator from a callsign (requires QRZ.com/HamQTH). Grids that look default are rejected (apologies if your grid really is AA00aa, enjoy your research)""" - data = self._get_qrz_data_for_callsign(call) + data = self._get_qrz_data_for_callsign(call, credentials) if data and "locator" in data and data["locator"].upper() != "AA00" and data["locator"].upper() != "AA00AA" and \ data["locator"].upper() != "AA00AA00": return data["locator"] - data = self._get_hamqth_data_for_callsign(call) + data = self._get_hamqth_data_for_callsign(call, credentials) if data and "grid" in data and data["grid"].upper() != "AA00" and data["grid"].upper() != "AA00AA" and data[ "grid"].upper() != "AA00AA00": return data["grid"] else: return None - def infer_qth_from_callsign_online_lookup(self, call): + def infer_qth_from_callsign_online_lookup(self, call, credentials=None): """Infer a textual QTH from a callsign (requires QRZ.com/HamQTH)""" - data = self._get_qrz_data_for_callsign(call) + data = self._get_qrz_data_for_callsign(call, credentials) if data and "addr2" in data: return data["addr2"] - data = self._get_hamqth_data_for_callsign(call) + data = self._get_hamqth_data_for_callsign(call, credentials) if data and "qth" in data: return data["qth"] else: @@ -422,79 +451,116 @@ class LookupHelper: logging.debug("Invalid lat/lon received for DXCC") return grid - def _get_qrz_data_for_callsign(self, call): - """Utility method to get QRZ.com data from cache if possible, if not get it from the API and cache it""" + def _get_qrz_data_for_callsign(self, call, credentials): + """Utility method to get QRZ.com data from cache if possible, if not get it from the API and cache it. + Returns None immediately if no credentials are provided.""" - # Fetch from cache if we can, otherwise fetch from the API and cache it + # Return from cache if available (a cached None means 'not found in QRZ') if call in self._qrz_callsign_data_cache: return self._qrz_callsign_data_cache.get(call) - elif self._qrz_available: + + # Obtain session key from credentials + session_key = None + if credentials and credentials.qrz_session_key: + session_key = credentials.qrz_session_key + elif credentials and credentials.qrz_username and credentials.qrz_password: try: - data = self._lookup_lib_qrz.lookup_callsign(callsign=call) - self._qrz_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds - return data - except (KeyError, ValueError): - # QRZ had no info for the call, but maybe it had prefixes or suffixes. Try again with the base call. - try: - data = self._lookup_lib_qrz.lookup_callsign(callsign=callinfo.Callinfo.get_homecall(call)) - self._qrz_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds - return data - except (KeyError, ValueError): - # QRZ had no info for the call, that's OK. Cache a None so we don't try to look this up again - self._qrz_callsign_data_cache.add(call, None, expire=604800) # 1 week in seconds + login_response = self._qrz_session_cache.get( + self._qrz_base_url + "?username=" + urllib.parse.quote_plus(credentials.qrz_username) + + "&password=" + urllib.parse.quote_plus(credentials.qrz_password) + "&agent=spothole", + headers=HTTP_HEADERS).content + login_data = xmltodict.parse(login_response) + session = login_data.get("QRZDatabase", {}).get("Session", {}) + if "Key" in session: + session_key = session["Key"] + else: + logging.warning("QRZ.com login details incorrect, failed to look up with QRZ.") return None except Exception: - # General exception like a timeout when communicating with QRZ. Return None this time, but don't cache - # that, so we can try again next time. - logging.error("Exception when looking up QRZ data") + logging.error("Exception when getting QRZ.com session key") return None - else: + + if not session_key: return None - def _get_hamqth_data_for_callsign(self, call): - """Utility method to get HamQTH data from cache if possible, if not get it from the API and cache it""" + # Try the call as given, then fall back to the base call (strips /P, /M etc.) + calls_to_try = [call] + home_call = callinfo.Callinfo.get_homecall(call) + if home_call != call: + calls_to_try.append(home_call) - # Fetch from cache if we can, otherwise fetch from the API and cache it + for lookup_call in calls_to_try: + try: + lookup_response = requests.get( + self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call), + headers=HTTP_HEADERS, timeout=10).content + raw = xmltodict.parse(lookup_response).get("QRZDatabase", {}).get("Callsign") + if raw: + data = _normalize_qrz_data(raw) + self._qrz_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds + return data + except (KeyError, ValueError): + continue + except Exception: + logging.error("Exception when looking up QRZ data") + return None + + # Not found in QRZ; cache None so we don't keep retrying + self._qrz_callsign_data_cache.add(call, None, expire=604800) # 1 week in seconds + return None + + def _get_hamqth_data_for_callsign(self, call, credentials): + """Utility method to get HamQTH data from cache if possible, if not get it from the API and cache it. + Returns None immediately if no credentials are provided.""" + + # Return from cache if available if call in self._hamqth_callsign_data_cache: return self._hamqth_callsign_data_cache.get(call) - elif self._hamqth_available: + + # Obtain session ID from credentials + session_id = None + if credentials and credentials.hamqth_session_id: + session_id = credentials.hamqth_session_id + elif credentials and credentials.hamqth_username and credentials.hamqth_password: try: - # First we need to log in and get a session token. session_data = self._hamqth_session_lookup_cache.get( - self._hamqth_base_url + "?u=" + urllib.parse.quote_plus(config["hamqth-username"]) + - "&p=" + urllib.parse.quote_plus(config["hamqth-password"]), headers=HTTP_HEADERS).content + self._hamqth_base_url + "?u=" + urllib.parse.quote_plus(credentials.hamqth_username) + + "&p=" + urllib.parse.quote_plus(credentials.hamqth_password), headers=HTTP_HEADERS).content dict_data = xmltodict.parse(session_data) if "session_id" in dict_data["HamQTH"]["session"]: session_id = dict_data["HamQTH"]["session"]["session_id"] - - # Now look up the actual data. - try: - lookup_data = SEMI_STATIC_URL_DATA_CACHE.get( - self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus( - call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS).content - data = xmltodict.parse(lookup_data)["HamQTH"]["search"] - self._hamqth_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds - return data - except (KeyError, ValueError): - # HamQTH had no info for the call, but maybe it had prefixes or suffixes. Try again with the base call. - try: - lookup_data = SEMI_STATIC_URL_DATA_CACHE.get( - self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus( - callinfo.Callinfo.get_homecall(call)) + "&prg=" + HAMQTH_PRG, - headers=HTTP_HEADERS).content - data = xmltodict.parse(lookup_data)["HamQTH"]["search"] - self._hamqth_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds - return data - except (KeyError, ValueError): - # HamQTH had no info for the call, that's OK. Cache a None so we don't try to look this up again - self._hamqth_callsign_data_cache.add(call, None, expire=604800) # 1 week in seconds - return None - else: logging.warning("HamQTH login details incorrect, failed to look up with HamQTH.") - except: + return None + except Exception: + logging.error("Exception when getting HamQTH session ID") + return None + + if not session_id: + return None + + # Try the call as given, then fall back to the base call (strips /P, /M etc.) + calls_to_try = [call] + home_call = callinfo.Callinfo.get_homecall(call) + if home_call != call: + calls_to_try.append(home_call) + + for lookup_call in calls_to_try: + try: + lookup_data = SEMI_STATIC_URL_DATA_CACHE.get( + self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus( + lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS).content + data = xmltodict.parse(lookup_data)["HamQTH"]["search"] + self._hamqth_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds + return data + except (KeyError, ValueError): + continue + except Exception: logging.error("Exception when looking up HamQTH data") return None + + # Not found in HamQTH; cache None so we don't keep retrying + self._hamqth_callsign_data_cache.add(call, None, expire=604800) # 1 week in seconds return None def _get_clublog_api_data_for_callsign(self, call): @@ -551,6 +617,7 @@ class LookupHelper: """Shutdown method to close down any caches neatly.""" self._qrz_callsign_data_cache.close() + self._hamqth_callsign_data_cache.close() self._clublog_callsign_data_cache.close() diff --git a/data/alert.py b/data/alert.py index 1765d9a..25187a0 100644 --- a/data/alert.py +++ b/data/alert.py @@ -61,7 +61,7 @@ class Alert: # The ID the source gave it, if any. source_id: str = None - def infer_missing(self): + def infer_missing(self, credentials=None): """Infer missing parameters where possible""" # If we somehow don't have a start time, set it to zero so it sorts off the bottom of any list but @@ -84,15 +84,15 @@ class Alert: # DX country, continent, zones etc. from callsign. CQ/ITU zone are better looked up with a location but we don't # have a real location for alerts. if self.dx_calls and self.dx_calls[0] and not self.dx_country: - self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_calls[0]) + self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_calls[0], credentials) if self.dx_calls and self.dx_calls[0] and not self.dx_continent: - self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_calls[0]) + self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_calls[0], credentials) if self.dx_calls and self.dx_calls[0] and not self.dx_cq_zone: - self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_calls[0]) + self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_calls[0], credentials) if self.dx_calls and self.dx_calls[0] and not self.dx_itu_zone: - self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_calls[0]) + self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_calls[0], credentials) if self.dx_calls and self.dx_calls[0] and not self.dx_dxcc_id: - self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_calls[0]) + self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_calls[0], credentials) if self.dx_dxcc_id and not self.dx_flag: self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id) @@ -108,21 +108,25 @@ class Alert: if self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0] and not self.sig: self.sig = self.sig_refs[0].sig - # DX operator details lookup, using QRZ.com. This should be the last resort compared to taking the data from - # the actual alertting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of - # the one from the park reference they're at. - if self.dx_calls and not self.dx_names: - self.dx_names = list(map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c), self.dx_calls)) - # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical # apart from that they were retrieved from the API at different times. Note that the simple Python hash() # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we # use diskcache to store our data between runs, so we use SHA256 which does not include this random element. - self_copy = copy.deepcopy(self) - self_copy.received_time = 0 - self_copy.received_time_iso = "" - self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest() + # The ID is computed before the online lookups below so that it is stable regardless of whether credentials + # are provided, allowing the enriched API response to be matched to the stored alert by ID. + if not self.id: + self_copy = copy.deepcopy(self) + self_copy.received_time = 0 + self_copy.received_time_iso = "" + self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest() + + # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data + # from the actual alerting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon + # instead of the one from the park reference they're at. + if self.dx_calls and not self.dx_names: + self.dx_names = list( + map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c, credentials), self.dx_calls)) def to_json(self): """JSON serialise""" diff --git a/data/lookup_credentials.py b/data/lookup_credentials.py new file mode 100644 index 0000000..b31c149 --- /dev/null +++ b/data/lookup_credentials.py @@ -0,0 +1,27 @@ +from dataclasses import dataclass + + +@dataclass +class LookupCredentials: + """Per-request credentials for QRZ.com and HamQTH online callsign lookups.""" + qrz_username: str = "" + qrz_password: str = "" + qrz_session_key: str = "" # alternative to username/password + hamqth_username: str = "" + hamqth_password: str = "" + hamqth_session_id: str = "" # alternative to username/password + + +def extract_credentials(query_params): + """Build a LookupCredentials from HTTP query params; returns None if no usable credentials are present.""" + creds = LookupCredentials( + qrz_username=query_params.get("qrz_username", ""), + qrz_password=query_params.get("qrz_password", ""), + qrz_session_key=query_params.get("qrz_session_key", ""), + hamqth_username=query_params.get("hamqth_username", ""), + hamqth_password=query_params.get("hamqth_password", ""), + hamqth_session_id=query_params.get("hamqth_session_id", ""), + ) + has_qrz = creds.qrz_session_key or (creds.qrz_username and creds.qrz_password) + has_hamqth = creds.hamqth_session_id or (creds.hamqth_username and creds.hamqth_password) + return creds if (has_qrz or has_hamqth) else None diff --git a/data/spot.py b/data/spot.py index 20fbcb4..e40f6cf 100644 --- a/data/spot.py +++ b/data/spot.py @@ -12,8 +12,9 @@ from pyhamtools.locator import locator_to_latlong, latlong_to_locator from core.config import MAX_SPOT_AGE from core.constants import MODE_ALIASES from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone -from core.lookup_helper import lookup_helper, infer_band_from_freq, infer_mode_from_comment, infer_mode_from_frequency, \ - infer_mode_type_from_mode +from core.lookup_helper import lookup_helper, infer_band_from_freq, infer_mode_from_comment, \ + infer_mode_from_frequency, infer_mode_type_from_mode +from data.lookup_credentials import LookupCredentials from core.sig_utils import populate_sig_ref_info, ANY_SIG_REGEX, get_ref_regex_for_sig from data.sig_ref import SIGRef @@ -131,7 +132,7 @@ class Spot: # The ID the source gave it, if any. source_id: str = None - def infer_missing(self): + def infer_missing(self, credentials=None): """Infer missing parameters where possible""" # If we somehow don't have a spot time, set it to zero so it sorts off the bottom of any list but @@ -158,11 +159,11 @@ class Spot: # DX country, continent etc. from callsign if self.dx_call and not self.dx_country: - self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_call) + self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_call, credentials) if self.dx_call and not self.dx_continent: - self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_call) + self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_call, credentials) if self.dx_call and not self.dx_dxcc_id: - self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_call) + self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_call, credentials) if self.dx_dxcc_id and not self.dx_flag: self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id) @@ -192,11 +193,11 @@ class Spot: if self.de_call and any(char.isdigit() for char in self.de_call) and not ( self.de_call.startswith("T2") and self.source == "APRS-IS"): if not self.de_country: - self.de_country = lookup_helper.infer_country_from_callsign(self.de_call) + self.de_country = lookup_helper.infer_country_from_callsign(self.de_call, credentials) if not self.de_continent: - self.de_continent = lookup_helper.infer_continent_from_callsign(self.de_call) + self.de_continent = lookup_helper.infer_continent_from_callsign(self.de_call, credentials) if not self.de_dxcc_id: - self.de_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.de_call) + self.de_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.de_call, credentials) if self.de_dxcc_id and not self.de_flag: self.de_flag = lookup_helper.get_flag_for_dxcc(self.de_dxcc_id) @@ -306,27 +307,40 @@ class Spot: if self.comment and not self.qrt: self.qrt = "QRT" in self.comment.upper() - # DX operator details lookup, using QRZ.com. This should be the last resort compared to taking the data from - # the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon instead of - # the one from the park reference they're at. + # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index + # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical + # apart from that they were retrieved from the API at different times. Note that the simple Python hash() + # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we + # use diskcache to store our data between runs, so we use SHA256 which does not include this random element. + # The ID is computed before the online lookups below so that it is stable regardless of whether credentials + # are provided, allowing the enriched API response to be matched to the stored spot by ID. + if not self.id: + self_copy = copy.deepcopy(self) + self_copy.received_time = 0 + self_copy.received_time_iso = "" + self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest() + + # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data + # from the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon + # instead of the one from the park reference they're at. if self.dx_call and not self.dx_name: - self.dx_name = lookup_helper.infer_name_from_callsign_online_lookup(self.dx_call) + self.dx_name = lookup_helper.infer_name_from_callsign_online_lookup(self.dx_call, credentials) if self.dx_call and not self.dx_latitude: - latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.dx_call) + latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.dx_call, credentials) if latlon: self.dx_latitude = latlon[0] self.dx_longitude = latlon[1] - self.dx_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.dx_call) + self.dx_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.dx_call, credentials) self.dx_location_source = "HOME QTH" - # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable stirng, + # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable string, # otherwise see what they have set on an online lookup service. if self.sig_refs and len(self.sig_refs) > 0: self.dx_qth = self.sig_refs[0].id if self.sig_refs[0].name: self.dx_qth = self.dx_qth + " " + self.sig_refs[0].name else: - self.dx_qth = lookup_helper.infer_qth_from_callsign_online_lookup(self.dx_call) + self.dx_qth = lookup_helper.infer_qth_from_callsign_online_lookup(self.dx_call, credentials) # Last resort for getting a DX position, use the DXCC entity. if self.dx_call and not self.dx_latitude: @@ -352,12 +366,12 @@ class Spot: if self.dx_latitude: self.dx_cq_zone = lat_lon_to_cq_zone(self.dx_latitude, self.dx_longitude) elif self.dx_call: - self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_call) + self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_call, credentials) if not self.dx_itu_zone: if self.dx_latitude: self.dx_itu_zone = lat_lon_to_itu_zone(self.dx_latitude, self.dx_longitude) elif self.dx_call: - self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_call) + self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_call, credentials) # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator # is likely at home. @@ -369,13 +383,13 @@ class Spot: # DE with no digits and APRS servers starting "T2" are not things we can look up location for if self.de_call and any(char.isdigit() for char in self.de_call) and not ( self.de_call.startswith("T2") and self.source == "APRS-IS"): - # DE operator position lookup, using QRZ.com. + # DE operator position lookup, using QRZ.com/HamQTH. if not self.de_latitude: - latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.de_call) + latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.de_call, credentials) if latlon: self.de_latitude = latlon[0] self.de_longitude = latlon[1] - self.de_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.de_call) + self.de_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.de_call, credentials) # Last resort for getting a DE position, use the DXCC entity. if not self.de_latitude: @@ -385,16 +399,6 @@ class Spot: self.de_longitude = latlon[1] self.de_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.de_call) - # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index - # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical - # apart from that they were retrieved from the API at different times. Note that the simple Python hash() - # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we - # use diskcache to store our data between runs, so we use SHA256 which does not include this random element. - self_copy = copy.deepcopy(self) - self_copy.received_time = 0 - self_copy.received_time_iso = "" - self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest() - def to_json(self): """JSON serialise""" diff --git a/server/handlers/api/alerts.py b/server/handlers/api/alerts.py index 2f336bf..b7740da 100644 --- a/server/handlers/api/alerts.py +++ b/server/handlers/api/alerts.py @@ -1,3 +1,4 @@ +import copy import json import logging from datetime import datetime @@ -9,6 +10,8 @@ import tornado_eventsource.handler from core.prometheus_metrics_handler import api_requests_counter from core.utils import serialize_everything, empty_queue +from data.lookup_credentials import extract_credentials + SSE_HANDLER_MAX_QUEUE_SIZE = 100 SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000 @@ -21,6 +24,15 @@ class APIAlertsHandler(tornado.web.RequestHandler): self._alerts = alerts self._web_server_metrics = web_server_metrics + @staticmethod + def _enrich(alerts, credentials): + enriched = [] + for alert in alerts: + alert_copy = copy.deepcopy(alert) + alert_copy.infer_missing(credentials) + enriched.append(alert_copy) + return enriched + def get(self): try: # Metrics @@ -33,8 +45,11 @@ class APIAlertsHandler(tornado.web.RequestHandler): # reduce that to just the first entry, and convert bytes to string query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()} - # Fetch all alerts matching the query + # Fetch all alerts matching the query, then optionally enrich with online data + credentials = extract_credentials(query_params) data = get_alert_list_with_filters(self._alerts, query_params) + if credentials: + data = self._enrich(data, credentials) self.write(json.dumps(data, default=serialize_everything)) self.set_status(200) except ValueError as e: @@ -73,6 +88,7 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler): # request.arguments contains lists for each param key because technically the client can supply multiple, # reduce that to just the first entry, and convert bytes to string self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()} + self._credentials = extract_credentials(self._query_params) # Create a alert queue and add it to the web server's list. The web server will fill this when alerts arrive self._alert_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE) @@ -110,6 +126,9 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler): alert = self._alert_queue.get() # If the new alert matches our param filters, send it to the client. If not, ignore it. if alert_allowed_by_query(alert, self._query_params): + if self._credentials: + alert = copy.deepcopy(alert) + alert.infer_missing(self._credentials) self.write_message(msg=json.dumps(alert, default=serialize_everything)) if self._alert_queue not in self._sse_alert_queues: diff --git a/server/handlers/api/lookups.py b/server/handlers/api/lookups.py index 3b6fef7..8f19f0f 100644 --- a/server/handlers/api/lookups.py +++ b/server/handlers/api/lookups.py @@ -11,6 +11,7 @@ from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_z from core.prometheus_metrics_handler import api_requests_counter from core.sig_utils import get_ref_regex_for_sig, populate_sig_ref_info from core.utils import serialize_everything +from data.lookup_credentials import extract_credentials from data.sig_ref import SIGRef from data.spot import Spot @@ -39,8 +40,9 @@ class APILookupCallHandler(tornado.web.RequestHandler): if re.match(r"^[A-Z0-9/\-]*$", call): # Take the callsign, make a "fake spot" so we can run infer_missing() on it, then repack the # resulting data in the correct way for the API response. + credentials = extract_credentials(query_params) fake_spot = Spot(dx_call=call) - fake_spot.infer_missing() + fake_spot.infer_missing(credentials) data = { "call": call, "name": fake_spot.dx_name, diff --git a/server/handlers/api/spots.py b/server/handlers/api/spots.py index b3579bf..0bf739c 100644 --- a/server/handlers/api/spots.py +++ b/server/handlers/api/spots.py @@ -1,3 +1,4 @@ +import copy import json import logging from datetime import datetime, timedelta @@ -9,6 +10,8 @@ import tornado_eventsource.handler from core.prometheus_metrics_handler import api_requests_counter from core.utils import serialize_everything, empty_queue +from data.lookup_credentials import extract_credentials + SSE_HANDLER_MAX_QUEUE_SIZE = 1000 SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000 @@ -21,6 +24,15 @@ class APISpotsHandler(tornado.web.RequestHandler): self._spots = spots self._web_server_metrics = web_server_metrics + @staticmethod + def _enrich(spots, credentials): + enriched = [] + for spot in spots: + spot_copy = copy.deepcopy(spot) + spot_copy.infer_missing(credentials) + enriched.append(spot_copy) + return enriched + def get(self): try: # Metrics @@ -33,8 +45,11 @@ class APISpotsHandler(tornado.web.RequestHandler): # reduce that to just the first entry, and convert bytes to string query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()} - # Fetch all spots matching the query + # Fetch all spots matching the query, then optionally enrich with online data + credentials = extract_credentials(query_params) data = get_spot_list_with_filters(self._spots, query_params) + if credentials: + data = self._enrich(data, credentials) self.write(json.dumps(data, default=serialize_everything)) self.set_status(200) except ValueError as e: @@ -75,6 +90,7 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler): # request.arguments contains lists for each param key because technically the client can supply multiple, # reduce that to just the first entry, and convert bytes to string self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()} + self._credentials = extract_credentials(self._query_params) # Create a spot queue and add it to the web server's list. The web server will fill this when spots arrive self._spot_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE) @@ -112,6 +128,9 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler): spot = self._spot_queue.get() # If the new spot matches our param filters, send it to the client. If not, ignore it. if spot_allowed_by_query(spot, self._query_params): + if self._credentials: + spot = copy.deepcopy(spot) + spot.infer_missing(self._credentials) self.write_message(msg=json.dumps(spot, default=serialize_everything)) if self._spot_queue not in self._sse_spot_queues: diff --git a/templates/about.html b/templates/about.html index 658e264..dc8c707 100644 --- a/templates/about.html +++ b/templates/about.html @@ -67,7 +67,7 @@

This software is dedicated to the memory of Tom G1PJB, SK, a friend and colleague who sadly passed away around the time I started writing it in Autumn 2025. I was looking forward to showing it to you when it was done.

- + {% end %} \ No newline at end of file diff --git a/templates/add_spot.html b/templates/add_spot.html index 134f741..1db935c 100644 --- a/templates/add_spot.html +++ b/templates/add_spot.html @@ -69,8 +69,8 @@ - - + + {% end %} \ No newline at end of file diff --git a/templates/alerts.html b/templates/alerts.html index bd8ce68..1d46221 100644 --- a/templates/alerts.html +++ b/templates/alerts.html @@ -56,8 +56,8 @@ - - + + {% end %} \ No newline at end of file diff --git a/templates/bands.html b/templates/bands.html index c637882..2f1da40 100644 --- a/templates/bands.html +++ b/templates/bands.html @@ -62,9 +62,9 @@ - - - + + + {% end %} \ No newline at end of file diff --git a/templates/base.html b/templates/base.html index 11e9059..8674e44 100644 --- a/templates/base.html +++ b/templates/base.html @@ -24,7 +24,7 @@ Spothole - + @@ -46,9 +46,9 @@ crossorigin="anonymous"> - - - + + + diff --git a/templates/conditions.html b/templates/conditions.html index e3c9695..f532dd4 100644 --- a/templates/conditions.html +++ b/templates/conditions.html @@ -230,8 +230,8 @@ - - + + diff --git a/templates/map.html b/templates/map.html index 712b328..1d2cab8 100644 --- a/templates/map.html +++ b/templates/map.html @@ -79,9 +79,9 @@ - - - + + + {% end %} \ No newline at end of file diff --git a/templates/spots.html b/templates/spots.html index 217e101..8001cd5 100644 --- a/templates/spots.html +++ b/templates/spots.html @@ -90,9 +90,9 @@ - - - + + + {% end %} \ No newline at end of file diff --git a/templates/status.html b/templates/status.html index 14bc6ee..772d453 100644 --- a/templates/status.html +++ b/templates/status.html @@ -59,8 +59,8 @@ - - + + diff --git a/webassets/apidocs/openapi.yml b/webassets/apidocs/openapi.yml index c249265..0b3cc25 100644 --- a/webassets/apidocs/openapi.yml +++ b/webassets/apidocs/openapi.yml @@ -13,6 +13,10 @@ info: ## Changelog + ### 1.3 + + * `/spots`, `/spots/stream`, `/alerts`, `/alerts/stream`, and `/lookup/call` now accept optional QRZ.com and HamQTH credentials as query parameters. When supplied, returned data is enriched with operator name, home location etc. from those services. + ### 1.2 * Added `/dxstats` endpoint for inter-continent DX spot statistics. @@ -29,7 +33,7 @@ info: license: name: The Unlicense url: https://unlicense.org/#the-unlicense - version: v1.2 + version: v1.3 servers: - url: https://spothole.app/api/v1 paths: @@ -38,7 +42,7 @@ paths: tags: - Spots summary: Get spots - description: The main API call that retrieves spots from the system. Supply this with no query parameters to retrieve all spots known to the system. Supply query parameters to filter what is retrieved. + description: The main API call that retrieves spots from the system. Supply this with no query parameters to retrieve all spots known to the system. Supply query parameters to filter what is retrieved. If QRZ.com or HamQTH credentials are supplied, returned spots will be enriched with operator name, home location etc. from those services. operationId: spots parameters: - name: limit @@ -160,6 +164,12 @@ paths: schema: type: boolean default: true + - $ref: '#/components/parameters/QrzUsername' + - $ref: '#/components/parameters/QrzPassword' + - $ref: '#/components/parameters/QrzSessionKey' + - $ref: '#/components/parameters/HamqthUsername' + - $ref: '#/components/parameters/HamqthPassword' + - $ref: '#/components/parameters/HamqthSessionId' responses: '200': description: Success @@ -175,7 +185,7 @@ paths: tags: - Spots summary: Get spot stream - description: Request a Server-Sent Event stream which will return individual spots immediately when they are added to the system. Only spots that match the provided filters will be returned. + description: Request a Server-Sent Event stream which will return individual spots immediately when they are added to the system. Only spots that match the provided filters will be returned. If QRZ.com or HamQTH credentials are supplied, streamed spots will be enriched with operator name, home location etc. from those services. operationId: spots-stream parameters: - name: source @@ -266,6 +276,12 @@ paths: schema: type: boolean default: true + - $ref: '#/components/parameters/QrzUsername' + - $ref: '#/components/parameters/QrzPassword' + - $ref: '#/components/parameters/QrzSessionKey' + - $ref: '#/components/parameters/HamqthUsername' + - $ref: '#/components/parameters/HamqthPassword' + - $ref: '#/components/parameters/HamqthSessionId' responses: '200': description: Success @@ -280,7 +296,7 @@ paths: tags: - Alerts summary: Get alerts - description: Retrieves alerts (indications of upcoming activations) from the system. Supply this with no query parameters to retrieve all alerts known to the system. Supply query parameters to filter what is retrieved. + description: Retrieves alerts (indications of upcoming activations) from the system. Supply this with no query parameters to retrieve all alerts known to the system. Supply query parameters to filter what is retrieved. If QRZ.com or HamQTH credentials are supplied, returned alerts will be enriched with operator names from those services. operationId: alerts parameters: - name: limit @@ -337,6 +353,12 @@ paths: required: false schema: type: string + - $ref: '#/components/parameters/QrzUsername' + - $ref: '#/components/parameters/QrzPassword' + - $ref: '#/components/parameters/QrzSessionKey' + - $ref: '#/components/parameters/HamqthUsername' + - $ref: '#/components/parameters/HamqthPassword' + - $ref: '#/components/parameters/HamqthSessionId' responses: '200': description: Success @@ -353,7 +375,7 @@ paths: tags: - Alerts summary: Get alert stream - description: Request a Server-Sent Event stream which will return individual alerts immediately when they are added to the system. Only alerts that match the provided filters will be returned. + description: Request a Server-Sent Event stream which will return individual alerts immediately when they are added to the system. Only alerts that match the provided filters will be returned. If QRZ.com or HamQTH credentials are supplied, streamed alerts will be enriched with operator names from those services. operationId: alerts-stream parameters: - name: max_duration @@ -398,6 +420,12 @@ paths: required: false schema: type: string + - $ref: '#/components/parameters/QrzUsername' + - $ref: '#/components/parameters/QrzPassword' + - $ref: '#/components/parameters/QrzSessionKey' + - $ref: '#/components/parameters/HamqthUsername' + - $ref: '#/components/parameters/HamqthPassword' + - $ref: '#/components/parameters/HamqthSessionId' responses: '200': description: Success @@ -607,7 +635,7 @@ paths: tags: - Utilities summary: Look up callsign details - description: Perform a lookup of data about a certain callsign, using any of the lookup services available to the Spothole server. + description: Perform a lookup of data about a certain callsign, using any of the lookup services available to the Spothole server. If QRZ.com or HamQTH credentials are supplied, the response will be able to use these services to perform a lookup. operationId: call parameters: - name: call @@ -616,6 +644,12 @@ paths: required: true type: string example: M0TRT + - $ref: '#/components/parameters/QrzUsername' + - $ref: '#/components/parameters/QrzPassword' + - $ref: '#/components/parameters/QrzSessionKey' + - $ref: '#/components/parameters/HamqthUsername' + - $ref: '#/components/parameters/HamqthPassword' + - $ref: '#/components/parameters/HamqthSessionId' responses: '200': description: Success @@ -838,6 +872,50 @@ paths: example: "Failed" components: + parameters: + QrzUsername: + name: qrz_username + in: query + description: "QRZ.com username for online callsign lookup, which will enrich the returned spots and alerts with extra data. Requires a QRZ.com XML Subscriber (paid) account. Supply together with `qrz_password`, or supply `qrz_session_key` instead." + required: false + schema: + type: string + QrzPassword: + name: qrz_password + in: query + description: "QRZ.com password. Supply together with `qrz_username`." + required: false + schema: + type: string + QrzSessionKey: + name: qrz_session_key + in: query + description: "A pre-obtained QRZ.com XML session key, as an alternative to supplying `qrz_username` and `qrz_password`. See https://www.qrz.com/docs/xml/current_spec.html for details on how to obtain one for the user." + required: false + schema: + type: string + HamqthUsername: + name: hamqth_username + in: query + description: "HamQTH username for online callsign lookup, which will enrich the returned spots and alerts with extra data. Supply together with `hamqth_password`, or supply `hamqth_session_id` instead." + required: false + schema: + type: string + HamqthPassword: + name: hamqth_password + in: query + description: "HamQTH password. Supply together with `hamqth_username`." + required: false + schema: + type: string + HamqthSessionId: + name: hamqth_session_id + in: query + description: "A pre-obtained HamQTH session ID, as an alternative to supplying `hamqth_username` and `hamqth_password`. See https://www.hamqth.com/developers.php for details on how to retrieve one for a user." + required: false + schema: + type: string + schemas: Source: type: string