Look up K0SWE's dxcc.json rather than using our own tables. Closes #80

README.md

@@ -196,10 +196,12 @@ Finally, simply add the appropriate config to the `providers` section of `config
 
 As well as being my work, I have also gratefully received feature patches from Steven, M1SDH.
 
-The project contains a self-hosted copy of Font Awesome's free library, in the `/webasset/fa/` directory. This is subject to Font Awesome's licence and is not covered by the overall licence declared in the `LICENSE` file. This approach was taken in preference to using their hosted kits due to the popularity of this project exceeding the page view limit for their free hosted offering.
+The project contains a self-hosted copy of Font Awesome's free library, in the `/webassets/fa/` directory. This is subject to Font Awesome's licence and is not covered by the overall licence declared in the `LICENSE` file. This approach was taken in preference to using their hosted kits due to the popularity of this project exceeding the page view limit for their free hosted offering.
+
+The project contains a set of flag icons generated using the "Noto Color Emoji" font on a Debian system, in the `/webassets/img/flags/` directory.
 
 The software uses a number of Python libraries as listed in `requirements.txt`, and a number of JavaScript libraries such as jQuery, Leaflet and Bootstrap. This project would not have been possible without these libraries, so many thanks to their developers.
 
-Particular thanks go to QRZCQ country-files.com for providing country lookup data for amateur radio, and to the developers of `pyhamtools` for making it easy to use this data as well as QRZ.com and Clublog lookup.
+Particular thanks go to country-files.com for providing country lookup data for amateur radio, to K0SWE for [this JSON-formatted DXCC data](https://github.com/k0swe/dxcc-json/), and to the developers of `pyhamtools` for making it easy to use country-files.com data as well as QRZ.com and Clublog lookup.
 
 The project's name was suggested by Harm, DK4HAA. Thanks!

core/lookup_helper.py

@@ -1,5 +1,7 @@
 import gzip
+import json
 import logging
+import re
 import urllib.parse
 from datetime import timedelta
 
@@ -14,7 +16,7 @@ from requests_cache import CachedSession
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.config import config
 from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
-    QRZCQ_CALLSIGN_LOOKUP_DATA, HTTP_HEADERS, HAMQTH_PRG
+    HTTP_HEADERS, HAMQTH_PRG
 
 
 # Singleton class that provides lookup functionality.
@@ -46,6 +48,8 @@ class LookupHelper:
         self.CALL_INFO_BASIC = None
         self.LOOKUP_LIB_BASIC = None
         self.COUNTRY_FILES_CTY_PLIST_DOWNLOAD_LOCATION = None
+        self.DXCC_JSON_DOWNLOAD_LOCATION = None
+        self.DXCC_DATA = None
 
     def start(self):
         # Lookup helpers from pyhamtools. We use five (!) of these. The simplest is country-files.com, which downloads
@@ -84,6 +88,19 @@ class LookupHelper:
                                                         filename=self.CLUBLOG_XML_DOWNLOAD_LOCATION)
         self.CLUBLOG_CALLSIGN_DATA_CACHE = Cache('cache/clublog_callsign_lookup_cache')
 
+        # We also get a lookup of DXCC data from K0SWE to use for additional lookups of e.g. flags.
+        self.DXCC_JSON_DOWNLOAD_LOCATION = "cache/dxcc.json"
+        success = self.download_dxcc_json()
+        if success:
+            with open(self.DXCC_JSON_DOWNLOAD_LOCATION) as f:
+                tmp_dxcc_data = json.load(f)["dxcc"]
+                # Reformat as a map for faster lookup
+                self.DXCC_DATA = {}
+                for dxcc in tmp_dxcc_data:
+                    self.DXCC_DATA[dxcc["entityCode"]] = dxcc
+        else:
+            logging.error("Could not download DXCC data, flags and similar data may be missing!")
+
     # Download the cty.plist file from country-files.com on first startup. The pyhamtools lib can actually download and use
     # this itself, but it's occasionally offline which causes it to throw an error. By downloading it separately, we can
     # catch errors and handle them, falling back to a previous copy of the file in the cache, and we can use the
@@ -103,6 +120,22 @@ class LookupHelper:
             logging.error("Exception when downloading Clublog cty.xml", e)
             return False
 
+    # Download the dxcc.json file on first startup.
+    def download_dxcc_json(self):
+        try:
+            logging.info("Downloading dxcc.json...")
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
+                                                      headers=HTTP_HEADERS).text
+
+            with open(self.DXCC_JSON_DOWNLOAD_LOCATION, "w") as f:
+                f.write(response)
+                f.flush()
+                return True
+
+        except Exception as e:
+            logging.error("Exception when downloading dxcc.json", e)
+            return False
+
     # Download the cty.xml (gzipped) file from Clublog on first startup, so we can use it in preference to querying the
     # database live if possible.
     def download_clublog_ctyxml(self):
@@ -175,11 +208,11 @@ class LookupHelper:
             clublog_data = self.get_clublog_api_data_for_callsign(call)
             if clublog_data and "Name" in clublog_data:
                 country = clublog_data["Name"]
-        # Couldn't get anything from Clublog database, try QRZCQ data
+        # Couldn't get anything from Clublog database, try DXCC data
         if not country:
-            qrzcq_data = self.get_qrzcq_data_for_callsign(call)
+            dxcc_data = self.get_dxcc_data_for_callsign(call)
-            if qrzcq_data and "country" in qrzcq_data:
+            if dxcc_data and "name" in dxcc_data:
-                country = qrzcq_data["country"]
+                country = dxcc_data["name"]
         return country
 
     # Infer a DXCC ID from a callsign
@@ -208,11 +241,11 @@ class LookupHelper:
             clublog_data = self.get_clublog_api_data_for_callsign(call)
             if clublog_data and "DXCC" in clublog_data:
                 dxcc = clublog_data["DXCC"]
-        # Couldn't get anything from Clublog database, try QRZCQ data
+        # Couldn't get anything from Clublog database, try DXCC data
         if not dxcc:
-            qrzcq_data = self.get_qrzcq_data_for_callsign(call)
+            dxcc_data = self.get_dxcc_data_for_callsign(call)
-            if qrzcq_data and "dxcc" in qrzcq_data:
+            if dxcc_data and "entityCode" in dxcc_data:
-                dxcc = qrzcq_data["dxcc"]
+                dxcc = dxcc_data["entityCode"]
         return dxcc
 
     # Infer a continent shortcode from a callsign
@@ -236,11 +269,12 @@ class LookupHelper:
             clublog_data = self.get_clublog_api_data_for_callsign(call)
             if clublog_data and "Continent" in clublog_data:
                 continent = clublog_data["Continent"]
-        # Couldn't get anything from Clublog database, try QRZCQ data
+        # Couldn't get anything from Clublog database, try DXCC data
         if not continent:
-            qrzcq_data = self.get_qrzcq_data_for_callsign(call)
+            dxcc_data = self.get_dxcc_data_for_callsign(call)
-            if qrzcq_data and "continent" in qrzcq_data:
+            # Some DXCCs are in two continents, if so don't use the continent data as we can't be sure
-                continent = qrzcq_data["continent"]
+            if dxcc_data and "continent" in dxcc_data and len(dxcc_data["continent"]) == 1:
+                continent = dxcc_data["continent"][0]
         return continent
 
     # Infer a CQ zone from a callsign
@@ -269,11 +303,12 @@ class LookupHelper:
             clublog_data = self.get_clublog_api_data_for_callsign(call)
             if clublog_data and "CQZ" in clublog_data:
                 cqz = clublog_data["CQZ"]
-        # Couldn't get anything from Clublog database, try QRZCQ data
+        # Couldn't get anything from Clublog database, try DXCC data
         if not cqz:
-            qrzcq_data = self.get_qrzcq_data_for_callsign(call)
+            dxcc_data = self.get_dxcc_data_for_callsign(call)
-            if qrzcq_data and "cqz" in qrzcq_data:
+            # Some DXCCs are in multiple zones, if so don't use the zone data as we can't be sure
-                cqz = qrzcq_data["cqz"]
+            if dxcc_data and "cq" in dxcc_data and len(dxcc_data["cq"]) == 1:
+                cqz = dxcc_data["cq"][0]
         return cqz
 
     # Infer a ITU zone from a callsign
@@ -293,13 +328,18 @@ class LookupHelper:
             hamqth_data = self.get_hamqth_data_for_callsign(call)
             if hamqth_data and "itu" in hamqth_data:
                 ituz = hamqth_data["itu"]
-        # Couldn't get anything from HamQTH database, Clublog doesn't provide this, so try QRZCQ data
+        # Couldn't get anything from HamQTH database, Clublog doesn't provide this, so try DXCC data
         if not ituz:
-            qrzcq_data = self.get_qrzcq_data_for_callsign(call)
+            dxcc_data = self.get_dxcc_data_for_callsign(call)
-            if qrzcq_data and "ituz" in qrzcq_data:
+            # Some DXCCs are in multiple zones, if so don't use the zone data as we can't be sure
-                ituz = qrzcq_data["ituz"]
+            if dxcc_data and "itu" in dxcc_data and len(dxcc_data["itu"]) == 1:
+                ituz = dxcc_data["itu"]
         return ituz
 
+    # Get an emoji flag for a given DXCC entity ID
+    def get_flag_for_dxcc(self, dxcc):
+        return self.DXCC_DATA[dxcc]["flag"] if dxcc in self.DXCC_DATA else None
+
     # Infer an operator name from a callsign (requires QRZ.com/HamQTH)
     def infer_name_from_callsign_online_lookup(self, call):
         data = self.get_qrz_data_for_callsign(call)
@@ -488,11 +528,10 @@ class LookupHelper:
         else:
             return None
 
-    # Utility method to get QRZCQ data from our constants table, if we can find it
+    # Utility method to get generic DXCC data from our lookup table, if we can find it
-    def get_qrzcq_data_for_callsign(self, call):
+    def get_dxcc_data_for_callsign(self, call):
-        # Iterate in reverse order - see comments on the data structure itself
+        for entry in self.DXCC_DATA.values():
-        for entry in reversed(QRZCQ_CALLSIGN_LOOKUP_DATA):
+            if re.match(entry["prefixRegex"], call):
-            if call.startswith(entry["prefix"]):
                 return entry
         return None
 

data/alert.py

@@ -6,7 +6,6 @@ from datetime import datetime, timedelta
 
 import pytz
 
-from core.constants import DXCC_FLAGS
 from core.lookup_helper import lookup_helper
 from core.sig_utils import get_icon_for_sig, get_sig_ref_info
 
@@ -95,8 +94,8 @@ class Alert:
             self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_calls[0])
         if self.dx_calls and self.dx_calls[0] and not self.dx_dxcc_id:
             self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_calls[0])
-        if self.dx_dxcc_id and self.dx_dxcc_id in DXCC_FLAGS and not self.dx_flag:
+        if self.dx_dxcc_id and not self.dx_flag:
-            self.dx_flag = DXCC_FLAGS[self.dx_dxcc_id]
+            self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id)
 
         # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference
         # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs

data/spot.py

@@ -9,7 +9,6 @@ from datetime import datetime
 import pytz
 from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 
-from core.constants import DXCC_FLAGS
 from core.lookup_helper import lookup_helper
 from core.sig_utils import get_icon_for_sig, get_sig_ref_info, ANY_SIG_REGEX, get_ref_regex_for_sig
 from data.sig_ref import SIGRef
@@ -174,8 +173,8 @@ class Spot:
             self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_call)
         if self.dx_call and not self.dx_dxcc_id:
             self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_call)
-        if self.dx_dxcc_id and self.dx_dxcc_id in DXCC_FLAGS and not self.dx_flag:
+        if self.dx_dxcc_id and not self.dx_flag:
-            self.dx_flag = DXCC_FLAGS[self.dx_dxcc_id]
+            self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id)
 
         # Clean up spotter call if it has an SSID or -# from RBN
         if self.de_call and "-" in self.de_call:
@@ -207,8 +206,8 @@ class Spot:
                 self.de_continent = lookup_helper.infer_continent_from_callsign(self.de_call)
             if not self.de_dxcc_id:
                 self.de_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.de_call)
-            if self.de_dxcc_id and self.de_dxcc_id in DXCC_FLAGS and not self.de_flag:
+            if self.de_dxcc_id and not self.de_flag:
-                self.de_flag = DXCC_FLAGS[self.de_dxcc_id]
+                self.de_flag = lookup_helper.get_flag_for_dxcc(self.de_dxcc_id)
 
         # Band from frequency
         if self.freq and not self.band: