diff --git a/README.md b/README.md index ecb1b23..429f90a 100644 --- a/README.md +++ b/README.md @@ -166,8 +166,9 @@ source .venv/bin/activate python3 spothole.py ``` -The software can take a few seconds to start up, mostly because it is downloading an updated file to match callsigns to -countries. This is normal, don't panic! +The software can take a few seconds to start up, mostly because it is downloading updated files containing the various +bits of reference data it uses. This is normal, don't panic! Once you see `You can access your copy of Spothole at +http://localhost:8080` in the log, your server is good to go. If you see some errors on startup, check your configuration, e.g. in case you have specified a port for the web server that is already in use by something else. diff --git a/config-example.yml b/config-example.yml index cf46a15..33353e6 100644 --- a/config-example.yml +++ b/config-example.yml @@ -260,12 +260,19 @@ callsign-data-providers: enabled: true # API key for Clublog to look up information. Required in order to enable this provider. You will need to request # one via their helpdesk portal if you want to use callsign lookups from Clublog. - clublog-api-key: "" + api-key: "" - class: "ClublogAPI" - enabled: true + # Querying the Clublog API directly doesn't provide any more data than the XML version, it just provides slightly + # more up-to-date information in the rare case that the prefix data changes, at a significant cost of looking up + # every callsign via an API call. Normally left disabled but it exists as an option. + enabled: false # API key for Clublog to look up information. Required in order to enable this provider. - clublog-api-key: "" + api-key: "" + + - class: "QRZ" + enabled: true + # No server-side credentials for QRZ. Users must provide their own as per QRZ policy. # Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week diff --git a/core/data_store.py b/core/data_store.py index e2a72e0..f58720c 100644 --- a/core/data_store.py +++ b/core/data_store.py @@ -20,11 +20,15 @@ class DataStore: self._MAX_SPOT_COUNT = 100000 self._MAX_ALERT_COUNT = 100000 self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300 - self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60 + self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60 # Caches self.alerts = None self.spots = None - self.callsigns = None + self.callsign_data_countryfiles = None + self.callsign_data_clublogxml = None + self.callsign_data_clublogapi = None + self.callsign_data_qrz = None + self.callsign_data_hamqth = None self.dxcc_data = None self.dxcc_lookup_by_call_regex = [] self.sigrefs = None @@ -65,8 +69,16 @@ class DataStore: # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup. # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign # we've seen, so we rely on them timing out and this triggering another lookup. - self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns") - logging.info(f"Loaded data for %d callsigns.", len(self.callsigns)) + self.callsign_data_countryfiles = diskcache.Cache(CACHE_DIR + "callsign_data_countryfiles") + self.callsign_data_clublogxml = diskcache.Cache(CACHE_DIR + "callsign_data_clublogxml") + self.callsign_data_clublogapi = diskcache.Cache(CACHE_DIR + "callsign_data_clublogapi") + self.callsign_data_qrz = diskcache.Cache(CACHE_DIR + "callsign_data_qrz") + self.callsign_data_hamqth = diskcache.Cache(CACHE_DIR + "callsign_data_hamqth") + unique_keys = set() + for c in [self.callsign_data_countryfiles, self.callsign_data_clublogxml, self.callsign_data_clublogapi, + self.callsign_data_qrz, self.callsign_data_hamqth]: + unique_keys.update(c) + logging.info(f"Loaded data for %d callsigns.", len(unique_keys)) # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache @@ -97,7 +109,11 @@ class DataStore: self._status.close() self.dxcc_data.close() self.sigrefs.close() - self.callsigns.close() + self.callsign_data_countryfiles.close() + self.callsign_data_clublogxml.close() + self.callsign_data_clublogapi.close() + self.callsign_data_qrz.close() + self.callsign_data_hamqth.close() # Global object DATA_STORE = DataStore() \ No newline at end of file diff --git a/core/sig_lookup_helper.py b/core/sig_lookup_helper.py index 0325b75..339c289 100644 --- a/core/sig_lookup_helper.py +++ b/core/sig_lookup_helper.py @@ -86,7 +86,9 @@ def populate_sig_ref_info(sig_ref): if value is not None and sig_ref.__dict__.get(key) is None: sig_ref.__dict__[key] = value else: - logging.warning("%s database did not contain data for ref %s", sig, ref_id) + # Maybe a super new reference we don't know about yet, but more likely a typo or a test reference, + # just silently ignore it. + logging.debug("%s database did not contain data for ref %s", sig, ref_id) except Exception: logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True) diff --git a/providers/callsigndata/api_query_callsign_data_provider.py b/providers/callsigndata/api_query_callsign_data_provider.py index 6d52358..f1180e9 100644 --- a/providers/callsigndata/api_query_callsign_data_provider.py +++ b/providers/callsigndata/api_query_callsign_data_provider.py @@ -14,9 +14,9 @@ from providers.callsigndata.callsign_data_provider import CallsignDataProvider class APIQueryCallsignDataProvider(CallsignDataProvider): """Generic callsign data provider class for providers that fetch their data from the web on-demand using an API.""" - def __init__(self, name, provider_config): + def __init__(self, name, provider_config, storage): """ Set up the provider.""" - super().__init__(name, provider_config) + super().__init__(name, provider_config, storage) def start(self): pass diff --git a/providers/callsigndata/callsign_data_provider.py b/providers/callsigndata/callsign_data_provider.py index dc84e2c..6d36f99 100644 --- a/providers/callsigndata/callsign_data_provider.py +++ b/providers/callsigndata/callsign_data_provider.py @@ -2,18 +2,22 @@ from datetime import datetime import pytz +from core.data_store import DATA_STORE + class CallsignDataProvider: """Generic callsign reference data provider class. Subclasses of this set up the various mechanisms via which Spothole can look up data for callsigns.""" - def __init__(self, name, provider_config): - """Constructor""" + def __init__(self, name, provider_config, storage): + """Constructor. As well as name and config, provide the storage object from DATA_STORE that will be used to + store the result of lookups to speed up future access.""" self.name = name self.enabled = provider_config["enabled"] self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC) self.status = "Not Started" if self.enabled else "Disabled" + self._storage = storage def start(self): """Start the provider. This should return immediately after spawning threads to access remote resources, if @@ -31,6 +35,19 @@ class CallsignDataProvider: that have been provided by the user for this session (QRZ.com/HamQTH) to allow us to look up using those services on the user's behalf. (Clublog is looked up using an API key owned by the server and provided in its config file, so users need not provide their own.) Returns a Callsign object with as much data populated as - possible.""" + possible. Data is cached internally for a set period of 30 days to avoid the need to request data from servers + each time.""" + + if callsign in self._storage: + return self._storage[callsign] + else: + c = self._perform_new_lookup(callsign, lookup_credentials) + if c: + self._storage.set(callsign, c, expire=DATA_STORE.CALLSIGN_DATA_TTL_SEC) + return c + + + def _perform_new_lookup(self, callsign, lookup_credentials): + """Makes a new request to the data source for callsign data.""" raise NotImplementedError("Subclasses must implement this method") diff --git a/providers/callsigndata/clublogapi.py b/providers/callsigndata/clublogapi.py index 34a2096..036206e 100644 --- a/providers/callsigndata/clublogapi.py +++ b/providers/callsigndata/clublogapi.py @@ -4,6 +4,7 @@ from datetime import datetime import pytz from pyhamtools import LookupLib, Callinfo +from core.data_store import DATA_STORE from core.utils import get_callsign_object_from_pyhamtools_callinfo from data.callsign import Callsign from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider @@ -25,10 +26,12 @@ class ClublogAPI(APIQueryCallsignDataProvider): logging.warning( "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.") - super().__init__("Clublog API", provider_config) + super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi) + + self.status = "Ready" - def lookup(self, callsign, lookup_credentials): + def _perform_new_lookup(self, callsign, lookup_credentials): callsign_data = Callsign(call=callsign) try: diff --git a/providers/callsigndata/clublogxml.py b/providers/callsigndata/clublogxml.py index e4ba610..b1282aa 100644 --- a/providers/callsigndata/clublogxml.py +++ b/providers/callsigndata/clublogxml.py @@ -3,8 +3,8 @@ import logging from pyhamtools import LookupLib, Callinfo +from core.data_store import DATA_STORE from core.utils import get_callsign_object_from_pyhamtools_callinfo -from data.callsign import Callsign from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider @@ -26,7 +26,7 @@ class ClublogXML(FileDownloadCallsignDataProvider): "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.") super().__init__("Clublog XML", provider_config, self.DATA_URL + "?api=" + self._api_key, - self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS) + self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS, DATA_STORE.callsign_data_clublogxml) def _handle_file(self, path): try: @@ -48,5 +48,5 @@ class ClublogXML(FileDownloadCallsignDataProvider): logging.error("Exception when loading Clublog XML.", e, exc_info=True) return False - def lookup(self, callsign, lookup_credentials): + def _perform_new_lookup(self, callsign, lookup_credentials): return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo) diff --git a/providers/callsigndata/countryfiles.py b/providers/callsigndata/countryfiles.py index 5da8dab..41603af 100644 --- a/providers/callsigndata/countryfiles.py +++ b/providers/callsigndata/countryfiles.py @@ -2,8 +2,8 @@ import logging from pyhamtools import LookupLib, Callinfo +from core.data_store import DATA_STORE from core.utils import get_callsign_object_from_pyhamtools_callinfo -from data.callsign import Callsign from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider @@ -16,7 +16,8 @@ class CountryFiles(FileDownloadCallsignDataProvider): _callinfo = None def __init__(self, provider_config): - super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS) + super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS, + DATA_STORE.callsign_data_countryfiles) def _handle_file(self, path): try: @@ -28,5 +29,5 @@ class CountryFiles(FileDownloadCallsignDataProvider): logging.error("Exception when loading Country Files cty.plist.", e, exc_info=True) return False - def lookup(self, callsign, lookup_credentials): + def _perform_new_lookup(self, callsign, lookup_credentials): return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo) diff --git a/providers/callsigndata/file_download_callsign_data_provider.py b/providers/callsigndata/file_download_callsign_data_provider.py index 19585a3..781cc18 100644 --- a/providers/callsigndata/file_download_callsign_data_provider.py +++ b/providers/callsigndata/file_download_callsign_data_provider.py @@ -14,9 +14,9 @@ from providers.callsigndata.callsign_data_provider import CallsignDataProvider class FileDownloadCallsignDataProvider(CallsignDataProvider): """Generic callsign data provider class for providers that fetch their data from the web by downloading a file.""" - def __init__(self, name, provider_config, url, cache_file_path, poll_interval): + def __init__(self, name, provider_config, url, cache_file_path, poll_interval, storage): """ Set up the provider, note poll_interval is in *days*.""" - super().__init__(name, provider_config) + super().__init__(name, provider_config, storage) self._url = url self._cache_file_path = cache_file_path self._poll_interval = poll_interval diff --git a/providers/callsigndata/qrz.py b/providers/callsigndata/qrz.py new file mode 100644 index 0000000..99db4da --- /dev/null +++ b/providers/callsigndata/qrz.py @@ -0,0 +1,154 @@ +import logging +import urllib.parse +from datetime import timedelta + +import xmltodict +from pyhamtools import callinfo +from requests import ConnectTimeout, ReadTimeout +from requests_cache import CachedSession + +from core.constants import HTTP_HEADERS +from core.data_store import DATA_STORE, CACHE_DIR +from core.url_data_cache import URLDataCache +from data.callsign import Callsign +from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider + + +class QRZ(APIQueryCallsignDataProvider): + """Callsign data provider for QRZ.com.""" + + def __init__(self, provider_config): + super().__init__("QRZ.com", provider_config, DATA_STORE.callsign_data_qrz) + self._QRZ_BASE_URL = "https://xmldata.qrz.com/xml/current/" + self._URL_DATA_CACHE = URLDataCache("qrz") + # Separate URL cache for session key lookups. Once a session key is returned from logging in with a username + # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes. + self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/qrz-creds", + expire_after=timedelta(minutes=55)) + self.status = "Ready" + + def _perform_new_lookup(self, callsign, lookup_credentials): + # If we don't have QRZ credentials, skip this lookup + if not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password) + or lookup_credentials.qrz_session_key): + return None + + try: + # Obtain session key from credentials, by looking it up from username & password if necessary. + session_key = None + if lookup_credentials.qrz_session_key: + session_key = lookup_credentials.qrz_session_key + elif lookup_credentials.qrz_username and lookup_credentials.qrz_password: + try: + login_response = self._URL_DATA_CACHE.get( + self._QRZ_BASE_URL + "?username=" + urllib.parse.quote_plus(lookup_credentials.qrz_username) + + "&password=" + urllib.parse.quote_plus(lookup_credentials.qrz_password) + "&agent=spothole", + headers=HTTP_HEADERS).content + login_data = xmltodict.parse(login_response) + session = login_data.get("QRZDatabase", {}).get("Session", {}) + if "Key" in session: + session_key = str(session["Key"]) + else: + logging.warning("QRZ.com login details incorrect, failed to look up with QRZ.") + return None + except Exception: + logging.error("Exception when getting QRZ.com session key") + return None + + if not session_key: + return None + + # Try the call as given, then fall back to the base call (strips /P, /M etc.) + calls_to_try = [callsign] + try: + home_call = callinfo.Callinfo.get_homecall(callsign) + if home_call != callsign: + calls_to_try.append(home_call) + except ValueError: + logging.debug("Could not look up home call for callsign %s", callsign) + + # Try looking up each call using the API + for lookup_call in calls_to_try: + try: + response = self._URL_DATA_CACHE.get( + self._QRZ_BASE_URL + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call), + headers=HTTP_HEADERS, timeout=10) + if response.ok: + qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {}) + if qrz_response: + if "Callsign" in qrz_response: + # Found data, convert it to our object and return it + return self.qrz_response_to_callsign(callsign, qrz_response.get("Callsign")) + + elif "Session" in qrz_response and "Error" in qrz_response.get("Session"): + # Errors here are normally just "callsign not in database", no need to log that ourselves + # above debug level. + logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call, + qrz_response.get("Session").get("Error")) + + elif not response.from_cache: + logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call) + elif not response.from_cache: + logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call) + + except (KeyError, ValueError): + continue + except ConnectionError: + logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call) + continue + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call) + continue + except Exception: + logging.error("Exception when looking up callsign %s using QRZ", lookup_call, exc_info=True) + continue + + # Not found in QRZ; return a Callsign object with no data so we cache that and don't keep retrying + return Callsign(call=callsign) + + except Exception as e: + self.status = "Error" + logging.error("Exception when looking up data from QRZ.com", e, exc_info=True) + # Return None, this won't be cached so we will be asked to query data again for this call next time. + return None + + @staticmethod + def qrz_response_to_callsign(callsign, data): + """Convert the "Callsign" block in QRZ's API response to our own Callsign object.""" + + # Get a name + name = None + if "name_fmt" in data: + name = data["name_fmt"] + if "fname" in data: + name = data["fname"] + if "nick" in data: + name = name + " \"" + data["nick"] + "\"" + if "name" in data: + name = name + " " + data["name"] + + # Check for sensible latitudes + lat = None + lon = None + if "latitude" in data and "longitude" in data and ( + float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float( + data["latitude"]) < 89.9: + lat = float(data["latitude"]) + lon = float(data["longitude"]) + + # Check for sensible grids + grid = None + if "grid" in data and not data["grid"].startswith("AA00"): + grid = data["grid"] + + return Callsign(call=callsign, + home_call=callinfo.Callinfo.get_homecall(callsign), + name=name, + qth=data["addr2"] if "addr2" in data else None, + country=data["country"] if "country" in data else None, + latitude=lat, + longitude=lon, + grid=grid, + dxcc_id=int(data["adif"]) if "adif" in data else None, + cq_zone=int(data["cqzone"]) if "cqzone" in data else None, + itu_zone=int(data["ituzone"]) if "adif" in data else None)