Download cty.plist separately so errors can be better handled

core/constants.py

@@ -4,6 +4,9 @@ from data.band import Band
 SOFTWARE_NAME = "Spothole by M0TRT"
 SOFTWARE_VERSION = "0.1"
 
+# HTTP headers used for spot providers that use HTTP
+HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + " " + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
+
 # Special Interest Groups
 SIGS = ["POTA", "SOTA", "WWFF", "GMA", "WWBOTA", "HEMA", "MOTA", "ARLHS", "SiOTA", "WCA"]
 

core/utils.py

@@ -1,26 +1,46 @@
 import gzip
 import logging
-import urllib.request
+from datetime import timedelta
 
 from diskcache import Cache
 from pyhamtools import LookupLib, Callinfo
 from pyhamtools.exceptions import APIKeyMissingError
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
+from requests_cache import CachedSession
 
 from core.config import config
-from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, QRZCQ_CALLSIGN_LOOKUP_DATA
+from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
+    QRZCQ_CALLSIGN_LOOKUP_DATA, HTTP_HEADERS
 
-CLUBLOG_API_KEY = config["clublog-api-key"]
+
+# Download the cty.plist file from country-files.com on first startup. The pyhamtools lib can actually download and use
+# this itself, but it's occasionally offline which causes it to throw an error. By downloading it separately, we can
+# catch errors and handle them, falling back to a previous copy of the file in the cache, and we can use the
+# requests_cache library to prevent re-downloading too quickly if the software keeps restarting.
+def download_country_files_cty_plist():
+    try:
+        response = COUNTRY_FILES_CTY_PLIST_CACHE.get("https://www.country-files.com/cty/cty.plist",
+                                headers=HTTP_HEADERS).text
+
+        # write to file
+        with open(COUNTRY_FILES_CTY_PLIST_DOWNLOAD_LOCATION, "w") as f:
+            f.write(response)
+            return True
+
+    except Exception as e:
+        logging.error("Exception when downloading Clublog cty.xml", e)
+        return False
 
 # Download the cty.xml (gzipped) file from Clublog on first startup, so we can use it in preference to querying the
 # database live if possible.
 def download_clublog_ctyxml():
     try:
         # Read the file inside the .gz archive located at url
-        with urllib.request.urlopen("https://cdn.clublog.org/cty.php?api=" + CLUBLOG_API_KEY) as response:
-            with gzip.GzipFile(fileobj=response) as uncompressed:
-                file_content = uncompressed.read()
+        response = CLUBLOG_CTY_XML_CACHE.get("https://cdn.clublog.org/cty.php?api=" + CLUBLOG_API_KEY,
+                                headers=HTTP_HEADERS).raw
+        with gzip.GzipFile(fileobj=response) as uncompressed:
+            file_content = uncompressed.read()
 
         # write to file in binary mode 'wb'
         with open(CLUBLOG_XML_DOWNLOAD_LOCATION, "wb") as f:
@@ -35,12 +55,18 @@ def download_clublog_ctyxml():
 # once on startup, and requires no login/key, but does not have the best coverage.
 # If the user provides login details/API keys, we also set up helpers for QRZ.com, Clublog (live API request), and
 # Clublog (XML download). The lookup functions iterate through these in a sensible order, looking for suitable data.
-LOOKUP_LIB_BASIC = LookupLib(lookuptype="countryfile")
+COUNTRY_FILES_CTY_PLIST_CACHE = CachedSession("cache/country_files_city_plist_cache", expire_after=timedelta(days=10))
+COUNTRY_FILES_CTY_PLIST_DOWNLOAD_LOCATION = "cache/cty.plist"
+LOOKUP_LIB_BASIC = LookupLib(lookuptype="countryfile", filename=COUNTRY_FILES_CTY_PLIST_DOWNLOAD_LOCATION)
 CALL_INFO_BASIC = Callinfo(LOOKUP_LIB_BASIC)
+
 QRZ_AVAILABLE = config["qrz-password"] != ""
 if QRZ_AVAILABLE:
     LOOKUP_LIB_QRZ = LookupLib(lookuptype="qrz", username=config["qrz-username"], pwd=config["qrz-password"])
 QRZ_CALLSIGN_DATA_CACHE = Cache('cache/qrz_callsign_lookup_cache')
+
+CLUBLOG_API_KEY = config["clublog-api-key"]
+CLUBLOG_CTY_XML_CACHE = CachedSession("cache/clublog_cty_xml_cache", expire_after=timedelta(days=10))
 CLUBLOG_API_AVAILABLE = CLUBLOG_API_KEY != ""
 CLUBLOG_XML_DOWNLOAD_LOCATION = "cache/cty.xml"
 if CLUBLOG_API_AVAILABLE: