Support Clublog lookup #38

.gitignore

@@ -1,11 +1,5 @@
 /.venv
 __pycache__
 *.pyc
-/.alerts_cache
-/.spots_cache
-/.qrz_callsign_lookup_cache
-/sota_summit_data_cache.sqlite
-/gma_ref_info_cache.sqlite
 /config.yml
-/siota_data_cache.sqlite
+/cache/
-/zlota_data_cache.sqlite

.idea/metaspot.iml

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$">
-      <excludeFolder url="file://$MODULE_DIR$/.venv" />
-    </content>
-    <orderEntry type="jdk" jdkName="Python 3.13 virtualenv at ~/code/spothole/.venv" jdkType="Python SDK" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

.idea/modules.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ProjectModuleManager">
     <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/metaspot.iml" filepath="$PROJECT_DIR$/.idea/metaspot.iml" />
+      <module fileurl="file://$PROJECT_DIR$/.idea/spothole.iml" filepath="$PROJECT_DIR$/.idea/spothole.iml" />
     </modules>
   </component>
 </project>

.idea/runConfigurations/Run.xml

@@ -1,6 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="Run" type="PythonConfigurationType" factoryName="Python">
-    <module name="metaspot" />
+    <module name="spothole" />
     <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />

README.md

@@ -34,7 +34,15 @@ cp config-example.yml config.yml
 
 Then edit `config.yml` in your text editor of choice to set up the software as you like it.
 
-Then, to run the software this time and any future times you want to run it directly from the command line:
+`config.yml` has some entries for QRZ.com username & password, and Clublog API keys. If provided, these allow Spothole to retrieve more information about DX spots, such as the country their callsign corresponds to. The software will work just fine without them, but you may find a few country flags etc. are less accurate or missing.
+
+Clublog API keys are free, but you'll need to get your own by submitting a helpdesk ticket and explaining what you'll use it for. The admin team are happy with the rate of requests made by my Spothole server, so unless you change the source code of yours to radically increase the rate of querying Clublog, I'm sure they will be fine with your server too.
+
+Free QRZ.com accounts offer only limited access to the site's data via their API. You'll have to sign up for one of their "XML Data Subscriber" plans to gain access to the full data, but if you're on a free account then the software will get what information it can.
+
+Once you're happy with the content of `config.yml`, you can proceed to running the software.
+
+To run the software this time and any future times you want to run it directly from the command line:
 
 ```bash
 source .venv/bin/activate
@@ -164,6 +172,7 @@ To navigate your way around the source code, this list may help.
 
 * `/` - Main script (`spothole.py`), pip `requirements.txt`, config, README, etc.
 * `/images` - Image sources
+* `/cache` - Directory where static-ish data downloaded from the internet is cached to avoid rapid re-requests, and where spot/alert data is cached so that it survives a software restart. Created on first run.
 
 ### Extending the server
 

config-example.yml

@@ -97,9 +97,14 @@ web-server-port: 8080
 max-spot-age-sec: 3600
 max-alert-age-sec: 604800
 
-# Login for QRZ.com to look up information. Optional.
+# Login for QRZ.com to look up information. Optional. You will need an "XML Subscriber" (paid) package to retrieve all
+# the data for a callsign via their system.
 qrz-username: "N0CALL"
 qrz-password: ""
 
+# API key for Clublog to look up information. Optional. You sill need to request one via their helpdesk portal if you
+# want to use callsign lookups from Clublog.
+clublog-api-key: ""
+
 # Allow submitting spots to the Spothole API?
 allow-spotting: true

core/utils.py

@@ -1,21 +1,56 @@
+import gzip
 import logging
+import urllib.request
 
 from diskcache import Cache
 from pyhamtools import LookupLib, Callinfo
+from pyhamtools.exceptions import APIKeyMissingError
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
 
 from core.config import config
 from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, QRZCQ_CALLSIGN_LOOKUP_DATA
 
-# Lookup helpers from pyhamtools
+CLUBLOG_API_KEY = config["clublog-api-key"]
+
+# Download the cty.xml (gzipped) file from Clublog on first startup, so we can use it in preference to querying the
+# database live if possible.
+def download_clublog_ctyxml():
+    try:
+        # Read the file inside the .gz archive located at url
+        with urllib.request.urlopen("https://cdn.clublog.org/cty.php?api=" + CLUBLOG_API_KEY) as response:
+            with gzip.GzipFile(fileobj=response) as uncompressed:
+                file_content = uncompressed.read()
+
+        # write to file in binary mode 'wb'
+        with open(CLUBLOG_XML_DOWNLOAD_LOCATION, "wb") as f:
+            f.write(file_content)
+            return True
+
+    except Exception as e:
+        logging.error("Exception when downloading Clublog cty.xml", e)
+        return False
+
+# Lookup helpers from pyhamtools. We use four (!) of these. The simplest is country-files.com, which downloads the data
+# once on startup, and requires no login/key, but does not have the best coverage.
+# If the user provides login details/API keys, we also set up helpers for QRZ.com, Clublog (live API request), and
+# Clublog (XML download). The lookup functions iterate through these in a sensible order, looking for suitable data.
 LOOKUP_LIB_BASIC = LookupLib(lookuptype="countryfile")
 CALL_INFO_BASIC = Callinfo(LOOKUP_LIB_BASIC)
 QRZ_AVAILABLE = config["qrz-password"] != ""
 if QRZ_AVAILABLE:
     LOOKUP_LIB_QRZ = LookupLib(lookuptype="qrz", username=config["qrz-username"], pwd=config["qrz-password"])
-# Cache of QRZ.com callsign lookups, so we don't repeatedly call the API for stuff we already know
+QRZ_CALLSIGN_DATA_CACHE = Cache('cache/qrz_callsign_lookup_cache')
-QRZ_CALLSIGN_DATA_CACHE = Cache('.qrz_callsign_lookup_cache')
+CLUBLOG_API_AVAILABLE = CLUBLOG_API_KEY != ""
+CLUBLOG_XML_DOWNLOAD_LOCATION = "cache/cty.xml"
+if CLUBLOG_API_AVAILABLE:
+    LOOKUP_LIB_CLUBLOG_API = LookupLib(lookuptype="clublogapi", apikey=CLUBLOG_API_KEY)
+    success = download_clublog_ctyxml()
+    CLUBLOG_XML_AVAILABLE = success
+    if success:
+        LOOKUP_LIB_CLUBLOG_XML = LookupLib(lookuptype="clublogxml", filename=CLUBLOG_XML_DOWNLOAD_LOCATION)
+CLUBLOG_CALLSIGN_DATA_CACHE = Cache('cache/clublog_callsign_lookup_cache')
+
 
 # Infer a mode from the comment
 def infer_mode_from_comment(comment):
@@ -24,6 +59,7 @@ def infer_mode_from_comment(comment):
             return mode
     return None
 
+
 # Infer a "mode family" from a mode.
 def infer_mode_type_from_mode(mode):
     if mode.upper() in CW_MODES:
@@ -37,6 +73,7 @@ def infer_mode_type_from_mode(mode):
             logging.warn("Found an unrecognised mode: " + mode + ". Developer should categorise this.")
         return None
 
+
 # Infer a band from a frequency in Hz
 def infer_band_from_freq(freq):
     for b in BANDS:
@@ -44,6 +81,7 @@ def infer_band_from_freq(freq):
             return b
     return UNKNOWN_BAND
 
+
 # Infer a country name from a callsign
 def infer_country_from_callsign(call):
     try:
@@ -60,13 +98,23 @@ def infer_country_from_callsign(call):
         qrz_data = get_qrz_data_for_callsign(call)
         if qrz_data and "country" in qrz_data:
             country = qrz_data["country"]
-    # Couldn't get anything from QRZ.com database, try QRZCQ data
+    # Couldn't get anything from QRZ.com database, try Clublog data
+    if not country:
+        clublog_data = get_clublog_xml_data_for_callsign(call)
+        if clublog_data and "Name" in clublog_data:
+            country = clublog_data["Name"]
+    if not country:
+        clublog_data = get_clublog_api_data_for_callsign(call)
+        if clublog_data and "Name" in clublog_data:
+            country = clublog_data["Name"]
+    # Couldn't get anything from Clublog database, try QRZCQ data
     if not country:
         qrzcq_data = get_qrzcq_data_for_callsign(call)
         if qrzcq_data and qrzcq_data["country"]:
             country = qrzcq_data["country"]
     return country
 
+
 # Infer a DXCC ID from a callsign
 def infer_dxcc_id_from_callsign(call):
     try:
@@ -84,13 +132,23 @@ def infer_dxcc_id_from_callsign(call):
         qrz_data = get_qrz_data_for_callsign(call)
         if qrz_data and "adif" in qrz_data:
             dxcc = qrz_data["adif"]
-    # Couldn't get anything from QRZ.com database, try QRZCQ data
+    # Couldn't get anything from QRZ.com database, try Clublog data
+    if not dxcc:
+        clublog_data = get_clublog_xml_data_for_callsign(call)
+        if clublog_data and "DXCC" in clublog_data:
+            dxcc = clublog_data["DXCC"]
+    if not dxcc:
+        clublog_data = get_clublog_api_data_for_callsign(call)
+        if clublog_data and "DXCC" in clublog_data:
+            dxcc = clublog_data["DXCC"]
+    # Couldn't get anything from Clublog database, try QRZCQ data
     if not dxcc:
         qrzcq_data = get_qrzcq_data_for_callsign(call)
         if qrzcq_data and qrzcq_data["dxcc"]:
             dxcc = qrzcq_data["dxcc"]
     return dxcc
 
+
 # Infer a continent shortcode from a callsign
 def infer_continent_from_callsign(call):
     try:
@@ -99,16 +157,26 @@ def infer_continent_from_callsign(call):
         continent = CALL_INFO_BASIC.get_continent(call)
         if not continent:
             base_call = max(call.split("/"), key=len)
-            continent =  CALL_INFO_BASIC.get_continent(base_call)
+            continent = CALL_INFO_BASIC.get_continent(base_call)
     except KeyError as e:
         continent = None
-    # Couldn't get anything from basic call info database, try QRZCQ data
+    # Couldn't get anything from basic call info database, try Clublog data
+    if not continent:
+        clublog_data = get_clublog_xml_data_for_callsign(call)
+        if clublog_data and "Continent" in clublog_data:
+            continent = clublog_data["Continent"]
+    if not continent:
+        clublog_data = get_clublog_api_data_for_callsign(call)
+        if clublog_data and "Continent" in clublog_data:
+            continent = clublog_data["Continent"]
+    # Couldn't get anything from Clublog database, try QRZCQ data
     if not continent:
         qrzcq_data = get_qrzcq_data_for_callsign(call)
         if qrzcq_data and qrzcq_data["continent"]:
             continent = qrzcq_data["continent"]
     return continent
 
+
 # Infer a CQ zone from a callsign
 def infer_cq_zone_from_callsign(call):
     try:
@@ -126,13 +194,23 @@ def infer_cq_zone_from_callsign(call):
         qrz_data = get_qrz_data_for_callsign(call)
         if qrz_data and "cqz" in qrz_data:
             cqz = qrz_data["cqz"]
-    # Couldn't get anything from QRZ.com database, try QRZCQ data
+    # Couldn't get anything from QRZ.com database, try Clublog data
+    if not cqz:
+        clublog_data = get_clublog_xml_data_for_callsign(call)
+        if clublog_data and "CQZ" in clublog_data:
+            cqz = clublog_data["CQZ"]
+    if not cqz:
+        clublog_data = get_clublog_api_data_for_callsign(call)
+        if clublog_data and "CQZ" in clublog_data:
+            cqz = clublog_data["CQZ"]
+    # Couldn't get anything from Clublog database, try QRZCQ data
     if not cqz:
         qrzcq_data = get_qrzcq_data_for_callsign(call)
         if qrzcq_data and qrzcq_data["cqz"]:
             cqz = qrzcq_data["cqz"]
     return cqz
 
+
 # Infer a ITU zone from a callsign
 def infer_itu_zone_from_callsign(call):
     try:
@@ -150,13 +228,14 @@ def infer_itu_zone_from_callsign(call):
         qrz_data = get_qrz_data_for_callsign(call)
         if qrz_data and "ituz" in qrz_data:
             ituz = qrz_data["ituz"]
-    # Couldn't get anything from QRZ.com database, try QRZCQ data
+    # Couldn't get anything from QRZ.com database, Clublog doesn't provide this, so try QRZCQ data
     if not ituz:
         qrzcq_data = get_qrzcq_data_for_callsign(call)
         if qrzcq_data and qrzcq_data["ituz"]:
             ituz = qrzcq_data["ituz"]
     return ituz
 
+
 # Utility method to get QRZ.com data from cache if possible, if not get it from the API and cache it
 def get_qrz_data_for_callsign(call):
     # Fetch from cache if we can, otherwise fetch from the API and cache it
@@ -166,7 +245,7 @@ def get_qrz_data_for_callsign(call):
     elif QRZ_AVAILABLE:
         try:
             data = LOOKUP_LIB_QRZ.lookup_callsign(callsign=call)
-            QRZ_CALLSIGN_DATA_CACHE.add(call, data, expire=604800) # 1 week in seconds
+            QRZ_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
             return data
         except KeyError:
             # QRZ had no info for the call, that's OK
@@ -174,6 +253,42 @@ def get_qrz_data_for_callsign(call):
     else:
         return None
 
+
+# Utility method to get Clublog API data from cache if possible, if not get it from the API and cache it
+def get_clublog_api_data_for_callsign(call):
+    # Fetch from cache if we can, otherwise fetch from the API and cache it
+    clublog_data = CLUBLOG_CALLSIGN_DATA_CACHE.get(call)
+    if clublog_data:
+        return clublog_data
+    elif CLUBLOG_API_AVAILABLE:
+        try:
+            data = LOOKUP_LIB_CLUBLOG_API.lookup_callsign(callsign=call)
+            CLUBLOG_CALLSIGN_DATA_CACHE.add(call, data, expire=604800)  # 1 week in seconds
+            return data
+        except KeyError:
+            # Clublog had no info for the call, that's OK
+            return None
+        except APIKeyMissingError:
+            # User API key was wrong, warn
+            logging.error("Could not look up via Clublog API, key " + CLUBLOG_API_KEY + " was rejected.")
+            return None
+    else:
+        return None
+
+
+# Utility method to get Clublog XML data from file
+def get_clublog_xml_data_for_callsign(call):
+    if CLUBLOG_XML_AVAILABLE:
+        try:
+            data = LOOKUP_LIB_CLUBLOG_XML.lookup_callsign(callsign=call)
+            return data
+        except KeyError:
+            # Clublog had no info for the call, that's OK
+            return None
+    else:
+        return None
+
+
 # Utility method to get QRZCQ data from our constants table, if we can find it
 def get_qrzcq_data_for_callsign(call):
     # Iterate in reverse order - see comments on the data structure itself
@@ -182,6 +297,7 @@ def get_qrzcq_data_for_callsign(call):
             return entry
     return None
 
+
 # Infer an operator name from a callsign (requires QRZ.com)
 def infer_name_from_callsign(call):
     data = get_qrz_data_for_callsign(call)
@@ -193,6 +309,7 @@ def infer_name_from_callsign(call):
     else:
         return None
 
+
 # Infer a latitude and longitude from a callsign (requires QRZ.com)
 def infer_latlon_from_callsign_qrz(call):
     data = get_qrz_data_for_callsign(call)
@@ -201,6 +318,7 @@ def infer_latlon_from_callsign_qrz(call):
     else:
         return None
 
+
 # Infer a grid locator from a callsign (requires QRZ.com)
 def infer_grid_from_callsign_qrz(call):
     data = get_qrz_data_for_callsign(call)
@@ -209,6 +327,7 @@ def infer_grid_from_callsign_qrz(call):
     else:
         return None
 
+
 # Infer a latitude and longitude from a callsign (using DXCC, probably very inaccurate)
 def infer_latlon_from_callsign_dxcc(call):
     try:
@@ -220,6 +339,7 @@ def infer_latlon_from_callsign_dxcc(call):
     except KeyError:
         return None
 
+
 # Infer a grid locator from a callsign (using DXCC, probably very inaccurate)
 def infer_grid_from_callsign_dxcc(call):
     latlon = infer_latlon_from_callsign_dxcc(call)
@@ -238,4 +358,4 @@ def infer_mode_from_frequency(freq):
 # Just converts objects to dict. Try to avoid doing anything clever here when serialising spots, because we also need
 # to receive spots without complex handling.
 def serialize_everything(obj):
-    return obj.__dict__
+    return obj.__dict__

spothole.py

@@ -13,8 +13,8 @@ from core.utils import QRZ_CALLSIGN_DATA_CACHE
 from server.webserver import WebServer
 
 # Globals
-spots = Cache('.spots_cache')
+spots = Cache('cache/spots_cache')
-alerts = Cache('.alerts_cache')
+alerts = Cache('cache/alerts_cache')
 status_data = {}
 spot_providers = []
 alert_providers = []

spotproviders/gma.py

@@ -15,7 +15,7 @@ class GMA(HTTPSpotProvider):
     # GMA spots don't contain the details of the programme they are for, we need a separate lookup for that
     REF_INFO_URL_ROOT = "https://www.cqgma.org/api/ref/?"
     REF_INFO_CACHE_TIME_DAYS = 30
-    REF_INFO_CACHE = CachedSession("gma_ref_info_cache", expire_after=timedelta(days=REF_INFO_CACHE_TIME_DAYS))
+    REF_INFO_CACHE = CachedSession("cache/gma_ref_info_cache", expire_after=timedelta(days=REF_INFO_CACHE_TIME_DAYS))
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)

spotproviders/parksnpeaks.py

@@ -15,10 +15,10 @@ class ParksNPeaks(HTTPSpotProvider):
     SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
     SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
     SIOTA_LIST_CACHE_TIME_DAYS = 30
-    SIOTA_LIST_CACHE = CachedSession("siota_data_cache", expire_after=timedelta(days=SIOTA_LIST_CACHE_TIME_DAYS))
+    SIOTA_LIST_CACHE = CachedSession("cache/siota_data_cache", expire_after=timedelta(days=SIOTA_LIST_CACHE_TIME_DAYS))
     ZLOTA_LIST_URL = "https://ontheair.nz/assets/assets.json"
     ZLOTA_LIST_CACHE_TIME_DAYS = 30
-    ZLOTA_LIST_CACHE = CachedSession("zlota_data_cache", expire_after=timedelta(days=ZLOTA_LIST_CACHE_TIME_DAYS))
+    ZLOTA_LIST_CACHE = CachedSession("cache/zlota_data_cache", expire_after=timedelta(days=ZLOTA_LIST_CACHE_TIME_DAYS))
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)

spotproviders/sota.py

@@ -18,7 +18,7 @@ class SOTA(HTTPSpotProvider):
     # SOTA spots don't contain lat/lon, we need a separate lookup for that
     SUMMIT_URL_ROOT = "https://api-db2.sota.org.uk/api/summits/"
     SUMMIT_DATA_CACHE_TIME_DAYS = 30
-    SUMMIT_DATA_CACHE = CachedSession("sota_summit_data_cache", expire_after=timedelta(days=SUMMIT_DATA_CACHE_TIME_DAYS))
+    SUMMIT_DATA_CACHE = CachedSession("cache/sota_summit_data_cache", expire_after=timedelta(days=SUMMIT_DATA_CACHE_TIME_DAYS))
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)