diff --git a/config-example.yml b/config-example.yml
index 10ce78e..dd9aed7 100644
--- a/config-example.yml
+++ b/config-example.yml
@@ -166,6 +166,13 @@ alert-providers:
enabled: true
+# Static reference data providers to use. This allows Spothole to download data such as mapping between callsign
+# prefixes and DXCC entities.
+static-data-providers:
+ - class: "K0SWE"
+ enabled: true
+
+
# SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF
# park IDs to their name and location.
sig-ref-data-providers:
diff --git a/core/config.py b/core/config.py
index b3f6fe9..9852cb8 100644
--- a/core/config.py
+++ b/core/config.py
@@ -60,6 +60,14 @@ def get_solar_conditions_provider_from_config(config_providers_entry):
return provider_class(config_providers_entry)
+def get_static_data_provider_from_config(config_providers_entry):
+ """Utility method to get a static reference data provider based on the class specified in its config entry."""
+
+ module = importlib.import_module('staticdataproviders.' + config_providers_entry["class"].lower())
+ provider_class = getattr(module, config_providers_entry["class"])
+ return provider_class(config_providers_entry)
+
+
def get_sig_ref_data_provider_from_config(config_providers_entry):
"""Utility method to get a SIG reference data provider based on the class specified in its config entry."""
diff --git a/core/data_store.py b/core/data_store.py
index 0511dfe..dd9e94d 100644
--- a/core/data_store.py
+++ b/core/data_store.py
@@ -22,6 +22,7 @@ class DataStore:
self.alerts = None
self.spots = None
self.callsigns = None
+ self.dxcc_data = None
self.sigrefs = None
self.status_data = None
self._status = None
@@ -42,10 +43,13 @@ class DataStore:
self._status.add("status_data", {})
self.status_data = self._status.get("status_data")
- # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
- # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
- # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
- # so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
+ # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
+ # a regular basis but there's no need for a TTL since old data is better than no data.
+ self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data")
+
+ # For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers
+ # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
+ # syntax for keys to keep it a single level.
self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
@@ -73,6 +77,7 @@ class DataStore:
self.alerts.close()
self._solar.close()
self._status.close()
+ self.dxcc_data.close()
self.sigrefs.close()
self.callsigns.close()
diff --git a/core/lookup_helper.py b/core/lookup_helper.py
index 370a82a..0f17693 100644
--- a/core/lookup_helper.py
+++ b/core/lookup_helper.py
@@ -1,7 +1,5 @@
import gzip
-import json
import logging
-import re
import urllib.parse
from datetime import timedelta
@@ -9,14 +7,13 @@ import xmltodict
from diskcache import Cache
from pyhamtools import LookupLib, Callinfo, callinfo
from pyhamtools.exceptions import APIKeyMissingError
-from pyhamtools.frequency import freq_to_band
from pyhamtools.locator import latlong_to_locator
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests_cache import CachedSession
from core.config import config
-from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
- HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
+from core.constants import HTTP_HEADERS, HAMQTH_PRG
+from core.data_store import DATA_STORE
from core.url_data_cache import URLDataCache
# QRZ XML field names differ from pyhamtools' normalised names; map them here.
@@ -84,7 +81,6 @@ class LookupHelper:
self._lookup_lib_basic = None
self._country_files_cty_plist_download_location = None
self._dxcc_json_download_location = None
- self._dxcc_data = None
def start(self):
# Lookup helpers from pyhamtools. We use five (!) of these. The simplest is country-files.com, which downloads
@@ -118,22 +114,6 @@ class LookupHelper:
filename=self._clublog_xml_download_location)
self._clublog_callsign_data_cache = Cache('cache/clublog_callsign_lookup_cache')
- # We also get a lookup of DXCC data from K0SWE to use for additional lookups of e.g. flags.
- self._dxcc_json_download_location = "cache/dxcc.json"
- success = self._download_dxcc_json()
- if success:
- with open(self._dxcc_json_download_location) as f:
- tmp_dxcc_data = json.load(f)["dxcc"]
- # Reformat as a map for faster lookup
- self._dxcc_data = {}
- for dxcc in tmp_dxcc_data:
- self._dxcc_data[dxcc["entityCode"]] = dxcc
- else:
- logging.error("Could not download DXCC data, flags and similar data may be missing!")
-
- # Precompile regex matches for DXCCs to improve efficiency when iterating through them
- for dxcc in (self._dxcc_data.values() if self._dxcc_data else []):
- dxcc["_prefixRegexCompiled"] = re.compile(dxcc["prefixRegex"])
def _download_country_files_cty_plist(self):
"""Download the cty.plist file from country-files.com on first startup. The pyhamtools lib can actually download and use
@@ -163,31 +143,6 @@ class LookupHelper:
logging.error("Exception when downloading Clublog cty.xml", e)
return False
- def _download_dxcc_json(self):
- """Download the dxcc.json file on first startup."""
-
- try:
- logging.info("Downloading dxcc.json...")
- response = _URL_DATA_CACHE.get(
- "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
- headers=HTTP_HEADERS)
-
- if response.ok:
- with open(self._dxcc_json_download_location, "w") as f:
- f.write(response.text)
- f.flush()
- return True
- else:
- logging.warning(f"HTTP {response.status_code} when downloading dxcc.json.")
- return False
-
- except ConnectionError:
- logging.warning(f"Connection error when downloading dxcc.json.")
- except (ConnectTimeout, ReadTimeout):
- logging.warning(f"Timeout when downloading dxcc.json.")
- except Exception as e:
- logging.error("Exception when downloading dxcc.json", e)
- return False
def _download_clublog_ctyxml(self):
"""Download the cty.xml (gzipped) file from Clublog on first startup, so we can use it in preference to querying the
@@ -374,7 +329,8 @@ class LookupHelper:
def get_flag_for_dxcc(self, dxcc):
"""Get an emoji flag for a given DXCC entity ID"""
- return self._dxcc_data[dxcc]["flag"] if dxcc in self._dxcc_data else None
+ dxcc_data = DATA_STORE.dxcc_data[dxcc]
+ return dxcc_data["flag"] if dxcc_data else None
def infer_name_from_callsign_online_lookup(self, call, credentials=None):
"""Infer an operator name from a callsign (requires QRZ.com/HamQTH)"""
@@ -680,61 +636,3 @@ class LookupHelper:
# Singleton object
lookup_helper = LookupHelper()
-
-
-def infer_mode_from_comment(comment):
- """Infer a mode from the comment"""
-
- for mode in ALL_MODES:
- if mode in comment.upper():
- return mode
- for mode in MODE_ALIASES.keys():
- if mode in comment.upper():
- return MODE_ALIASES[mode]
- return None
-
-
-def infer_mode_type_from_mode(mode):
- """Infer a "mode family" from a mode."""
-
- if mode.upper() in CW_MODES:
- return "CW"
- elif mode.upper() in PHONE_MODES:
- return "PHONE"
- elif mode.upper() in DATA_MODES:
- return "DATA"
- else:
- if mode.upper() != "OTHER":
- logging.warning("Found an unrecognised mode: " + mode + ". Developer should categorise this.")
- return None
-
-
-def infer_band_from_freq(freq):
- """Infer a band from a frequency in Hz"""
-
- for b in BANDS:
- if b.start_freq <= freq <= b.end_freq:
- return b
- return UNKNOWN_BAND
-
-
-def infer_mode_from_frequency(freq):
- """Infer a mode from the frequency (in Hz) according to the band plan. Just a guess really."""
-
- try:
- khz = freq / 1000.0
- mode = freq_to_band(khz)["mode"]
- # Some additional common digimode ranges in addition to what the 3rd-party freq_to_band function returns.
- # This is mostly here just because freq_to_band is very specific about things like FT8 frequencies, and e.g.
- # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
- # might include other common digimode centres of activity here, but this achieves the main goal of keeping
- # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
- if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or (
- 21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077):
- mode = "FT8"
- if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or (
- 18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183):
- mode = "FT4"
- return mode
- except KeyError:
- return None
diff --git a/core/status_reporter.py b/core/status_reporter.py
index 7fc38b0..d21802d 100644
--- a/core/status_reporter.py
+++ b/core/status_reporter.py
@@ -15,7 +15,7 @@ class StatusReporter:
"""Provides a timed update of the application's status data."""
def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers,
- sig_ref_data_providers):
+ static_data_providers, sig_ref_data_providers):
"""Constructor"""
self._run_interval = run_interval
@@ -23,6 +23,7 @@ class StatusReporter:
self._spot_providers = spot_providers
self._alert_providers = alert_providers
self._solar_condition_providers = solar_condition_providers
+ self._static_data_providers = static_data_providers
self._sig_ref_data_providers = sig_ref_data_providers
self._thread = None
self._stop_event = Event()
@@ -74,6 +75,11 @@ class StatusReporter:
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
self._solar_condition_providers))
+ DATA_STORE.status_data["static_data_providers"] = list(
+ map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+ "last_updated": p.last_update_time.replace(
+ tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+ self._static_data_providers))
DATA_STORE.status_data["sig_ref_data_providers"] = list(
map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
diff --git a/core/utils.py b/core/utils.py
index 54a48ed..02d3f5a 100644
--- a/core/utils.py
+++ b/core/utils.py
@@ -1,8 +1,71 @@
+import logging
+
import simplejson
+from pyhamtools.frequency import freq_to_band
+
+from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES
def safe_json_dumps(obj):
"""Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
which are invalid in JSON."""
- return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
\ No newline at end of file
+ return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
+
+
+def infer_mode_from_comment(comment):
+ """Infer a mode from the comment"""
+
+ for mode in ALL_MODES:
+ if mode in comment.upper():
+ return mode
+ for mode in MODE_ALIASES.keys():
+ if mode in comment.upper():
+ return MODE_ALIASES[mode]
+ return None
+
+
+def infer_mode_type_from_mode(mode):
+ """Infer a "mode family" from a mode."""
+
+ if mode.upper() in CW_MODES:
+ return "CW"
+ elif mode.upper() in PHONE_MODES:
+ return "PHONE"
+ elif mode.upper() in DATA_MODES:
+ return "DATA"
+ else:
+ if mode.upper() != "OTHER":
+ logging.warning("Found an unrecognised mode: " + mode + ". Developer should categorise this.")
+ return None
+
+
+def infer_band_from_freq(freq):
+ """Infer a band from a frequency in Hz"""
+
+ for b in BANDS:
+ if b.start_freq <= freq <= b.end_freq:
+ return b
+ return UNKNOWN_BAND
+
+
+def infer_mode_from_frequency(freq):
+ """Infer a mode from the frequency (in Hz) according to the band plan. Just a guess really."""
+
+ try:
+ khz = freq / 1000.0
+ mode = freq_to_band(khz)["mode"]
+ # Some additional common digimode ranges in addition to what the 3rd-party freq_to_band function returns.
+ # This is mostly here just because freq_to_band is very specific about things like FT8 frequencies, and e.g.
+ # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
+ # might include other common digimode centres of activity here, but this achieves the main goal of keeping
+ # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
+ if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or (
+ 21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077):
+ mode = "FT8"
+ if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or (
+ 18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183):
+ mode = "FT4"
+ return mode
+ except KeyError:
+ return None
\ No newline at end of file
diff --git a/data/spot.py b/data/spot.py
index bf32145..886099c 100644
--- a/data/spot.py
+++ b/data/spot.py
@@ -12,9 +12,10 @@ from pyhamtools.locator import locator_to_latlong, latlong_to_locator
from core.config import MAX_SPOT_AGE
from core.constants import MODE_ALIASES, PROPAGATION_MODES
from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
-from core.lookup_helper import lookup_helper, infer_band_from_freq, infer_mode_from_comment, \
- infer_mode_from_frequency, infer_mode_type_from_mode
+from core.lookup_helper import lookup_helper
from core.sig_utils import populate_sig_ref_info, ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
+from core.utils import infer_band_from_freq, infer_mode_from_comment, \
+ infer_mode_from_frequency, infer_mode_type_from_mode
from data.sig_ref import SIGRef
diff --git a/server/handlers/api/addspot.py b/server/handlers/api/addspot.py
index e93e090..e93ec6b 100644
--- a/server/handlers/api/addspot.py
+++ b/server/handlers/api/addspot.py
@@ -10,7 +10,7 @@ from tornado.web import Application
from core.config import ALLOW_SPOTTING
from core.constants import UNKNOWN_BAND
-from core.lookup_helper import infer_band_from_freq
+from core.utils import infer_band_from_freq
from core.prometheus_metrics_handler import api_requests_counter
from core.sig_utils import get_ref_regex_for_sig
from core.utils import safe_json_dumps
diff --git a/sigrefdataproviders/file_download_sig_ref_data_provider.py b/sigrefdataproviders/file_download_sig_ref_data_provider.py
index d070cc7..f4bc2e3 100644
--- a/sigrefdataproviders/file_download_sig_ref_data_provider.py
+++ b/sigrefdataproviders/file_download_sig_ref_data_provider.py
@@ -21,7 +21,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
self._poll_interval = poll_interval
self._thread = None
self._stop_event = Event()
- self._url_data_cache = URLDataCache("sigrefdata-" + sig_name)
+ self._url_data_cache = URLDataCache("sigrefdata_" + sig_name)
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
@@ -50,7 +50,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
if http_response.ok:
# Pass off to the subclass for processing
new_data = self._http_response_to_data(http_response)
- # Submit the new spots for processing. There might not be any spots for the less popular programs.
+ # Add the new data to the SIG Ref data store
if new_data:
self._add_data(new_data)
@@ -62,8 +62,10 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
logging.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.")
except ConnectionError:
+ self.status = "Error"
logging.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.")
except (ConnectTimeout, ReadTimeout):
+ self.status = "Error"
logging.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.")
except Exception:
self.status = "Error"
diff --git a/sigrefdataproviders/local_file_sig_ref_data_provider.py b/sigrefdataproviders/local_file_sig_ref_data_provider.py
index 05a3152..aacadd9 100644
--- a/sigrefdataproviders/local_file_sig_ref_data_provider.py
+++ b/sigrefdataproviders/local_file_sig_ref_data_provider.py
@@ -22,10 +22,11 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
else:
- logging.info("No new SIG ref data found for " + self.sig_name)
+ self.status = "Error"
+ logging.info("Failed to load SIG ref data for " + self.sig_name)
except Exception as e:
self.status = "Error"
- logging.exception("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")")
+ logging.error("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")", e, exc_info=True)
def stop(self):
pass
diff --git a/sigrefdataproviders/sig_ref_data_provider.py b/sigrefdataproviders/sig_ref_data_provider.py
index efe1977..8393e49 100644
--- a/sigrefdataproviders/sig_ref_data_provider.py
+++ b/sigrefdataproviders/sig_ref_data_provider.py
@@ -15,7 +15,6 @@ class SIGRefDataProvider:
self.sig_name = sig_name
self.enabled = provider_config["enabled"]
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
- self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
self.status = "Not Started" if self.enabled else "Disabled"
self.reference_count = 0
diff --git a/spothole.py b/spothole.py
index 02c0ee5..2510149 100644
--- a/spothole.py
+++ b/spothole.py
@@ -5,7 +5,8 @@ import signal
import sys
from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_data_provider_from_config, \
- get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config
+ get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config, \
+ get_static_data_provider_from_config
from core.constants import SOFTWARE_VERSION
from core.data_store import DATA_STORE
from core.lookup_helper import lookup_helper
@@ -17,6 +18,7 @@ web_server = None
spot_providers = []
alert_providers = []
solar_condition_providers = []
+static_data_providers = []
sig_ref_data_providers = []
cleanup_timer = None
run = True
@@ -42,6 +44,9 @@ def shutdown(_signum=None, _frame=None):
for srdp in sig_ref_data_providers:
if srdp.enabled:
srdp.stop()
+ for srdp in static_data_providers:
+ if srdp.enabled:
+ srdp.stop()
DATA_STORE.close()
os._exit(0)
@@ -95,6 +100,13 @@ if __name__ == '__main__':
if p.enabled:
p.start()
+ # Fetch, set up and start static reference data providers
+ for entry in config.get("static-data-providers", []):
+ static_data_providers.append(get_static_data_provider_from_config(entry))
+ for p in static_data_providers:
+ if p.enabled:
+ p.start()
+
# Fetch, set up and start SIG reference data providers
for entry in config.get("sig-ref-data-providers", []):
sig_ref_data_providers.append(get_sig_ref_data_provider_from_config(entry))
@@ -104,7 +116,7 @@ if __name__ == '__main__':
# Set up status reporter
status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers,
- alert_providers=alert_providers,
+ alert_providers=alert_providers, static_data_providers=static_data_providers,
sig_ref_data_providers=sig_ref_data_providers,
solar_condition_providers=solar_condition_providers, run_interval=5)
status_reporter.start()
diff --git a/static/apidocs/openapi.yml b/static/apidocs/openapi.yml
index 4b852ad..f5ecf4f 100644
--- a/static/apidocs/openapi.yml
+++ b/static/apidocs/openapi.yml
@@ -23,7 +23,7 @@ info:
* Added `comment_names` to SIGs in the `/options`, to reflect how they might be referred to in spot comments where
it differs from their `name`.
* Added `propagation_mode` field to spots
- * Added `sig_ref_data_providers` to status and removed `cleanup`
+ * Added `sig_ref_data_providers` and `static_data_providers` to status and removed `cleanup`
### 1.3
@@ -1721,6 +1721,28 @@ components:
is zero, the provider has never updated.
example: 1759579508
+ StaticDataProviderStatus:
+ type: object
+ properties:
+ sig_name:
+ type: string
+ description: The name of the provider.
+ example: K0SWE
+ enabled:
+ type: boolean
+ description: Whether the provider is enabled or not.
+ example: true
+ status:
+ type: string
+ description: The status of the provider.
+ example: OK
+ last_updated:
+ type: number
+ description: >
+ The last time at which this provider received data, UTC seconds since UNIX epoch. If this
+ is zero, the provider has never updated.
+ example: 1759579508
+
SIGRefDataProviderStatus:
type: object
properties:
@@ -1844,6 +1866,11 @@ components:
description: An array of all the solar conditions providers.
items:
$ref: '#/components/schemas/SolarConditionsProviderStatus'
+ static_data_providers:
+ type: array
+ description: An array of all the static reference data providers.
+ items:
+ $ref: '#/components/schemas/StaticDataProviderStatus'
sig_ref_data_providers:
type: array
description: An array of all the SIG reference data providers.
diff --git a/static/js/status.js b/static/js/status.js
index c15c563..fd7e6ff 100644
--- a/static/js/status.js
+++ b/static/js/status.js
@@ -40,6 +40,15 @@ function loadStatus() {
`);
});
+ jsonData["static_data_providers"].forEach(p => {
+ $("#static-data-providers-status-container").append(`
+
+
${p["sig_name"]}
+
Status: ${p["status"]}
+
Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}
+
diff --git a/staticdataproviders/file_download_static_data_provider.py b/staticdataproviders/file_download_static_data_provider.py
new file mode 100644
index 0000000..94b2eb1
--- /dev/null
+++ b/staticdataproviders/file_download_static_data_provider.py
@@ -0,0 +1,77 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from staticdataproviders.static_data_provider import StaticDataProvider
+
+
+class FileDownloadStaticDataProvider(StaticDataProvider):
+ """Generic static reference data provider class for providers that fetch their data from the web by downloading a
+ file."""
+
+ def __init__(self, name, provider_config, url, poll_interval):
+ """ Set up the provider, note poll_interval is in *days*."""
+ super().__init__(name, provider_config)
+ self._url = url
+ self._poll_interval = poll_interval
+ self._thread = None
+ self._stop_event = Event()
+ self._url_data_cache = URLDataCache("staticdata_" + name)
+
+ def start(self):
+ # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+ # subsequent polls, so start() returns immediately and the application can continue starting.
+ logging.info(
+ "Set up query of " + self.name + " static reference data every " + str(self._poll_interval) + " days.")
+ self._thread = Thread(target=self._run, daemon=True)
+ self._thread.start()
+
+ def stop(self):
+ self._stop_event.set()
+
+ def _run(self):
+ while True:
+ self._poll()
+ if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
+ break
+
+ def _poll(self):
+ try:
+ # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+ # caching, but just so continual restarts of the software during testing don't hammer the servers.
+ logging.debug("Downloading " + self.name + " static reference data...")
+ http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
+ # Check response code was good
+ if http_response.ok:
+ # Pass off to the subclass for processing
+ ok = self._handle_http_response(http_response)
+ if ok:
+ self.status = "OK"
+ self.last_update_time = datetime.now(pytz.UTC)
+ logging.info("Updated static reference data for " + self.name)
+ else:
+ self.status = "Error"
+ logging.warning(f"HTTP {http_response.status_code} when downloading static reference data for {self.name}.")
+
+ except ConnectionError:
+ self.status = "Error"
+ logging.warning(f"Connection error when downloading static reference data for {self.name}.")
+ except (ConnectTimeout, ReadTimeout):
+ self.status = "Error"
+ logging.warning(f"Timeout when downloading static reference data for {self.name}.")
+ except Exception:
+ self.status = "Error"
+ logging.exception("Exception in HTTP static reference data provider (" + self.name + ")")
+ self._stop_event.wait(timeout=1)
+
+ def _handle_http_response(self, http_response):
+ """Handle an HTTP response returned by the server and load the data from it. Return true if successful,
+ false otherwise."""
+
+ raise NotImplementedError("Subclasses must implement this method")
diff --git a/staticdataproviders/k0swe.py b/staticdataproviders/k0swe.py
new file mode 100644
index 0000000..2beb609
--- /dev/null
+++ b/staticdataproviders/k0swe.py
@@ -0,0 +1,40 @@
+import logging
+import re
+
+from core.data_store import DATA_STORE
+from staticdataproviders.file_download_static_data_provider import FileDownloadStaticDataProvider
+
+
+class K0SWE(FileDownloadStaticDataProvider):
+ """Static data provider for K0SWE's dxcc.json, which provides callsign regex to DXCC entity mapping, plus DXCC to
+ continent, flag emoji etc."""
+
+ POLL_INTERVAL_DAYS = 7
+ DATA_URL = "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json"
+
+ def __init__(self, provider_config):
+ super().__init__("K0SWE DXCC JSON", provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+ def _handle_http_response(self, http_response):
+ try:
+ dxcc_list = http_response.json()["dxcc"]
+ # Reformat as a map for to place in the data store
+ dxcc_map = {}
+ for dxcc in dxcc_list:
+ dxcc_map[dxcc["entityCode"]] = dxcc
+
+ # Precompile regex matches for DXCCs to improve efficiency when iterating through them
+ for dxcc in dxcc_map.values():
+ dxcc["_prefixRegexCompiled"] = re.compile(dxcc["prefixRegex"])
+
+ # Add to data store
+ for k, v in dxcc_map.items():
+ DATA_STORE.dxcc_data[k] = v
+
+ return True
+
+ except Exception as e:
+ logging.error("Exception when loading K0SWE dxcc.json.", e, exc_info=True)
+ return False
+
+
diff --git a/staticdataproviders/local_file_static_data_provider.py b/staticdataproviders/local_file_static_data_provider.py
new file mode 100644
index 0000000..11b12d7
--- /dev/null
+++ b/staticdataproviders/local_file_static_data_provider.py
@@ -0,0 +1,36 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from staticdataproviders.static_data_provider import StaticDataProvider
+
+
+class LocalFileStaticDataProvider(StaticDataProvider):
+ """Generic static reference data provider class for providers that fetch their data from a local file on startup."""
+
+ def __init__(self, name, provider_config, path):
+ super().__init__(name, provider_config)
+ self._path = path
+
+ def start(self):
+ logging.debug("Loading " + self.name + " static reference data from file.")
+ try:
+ ok = self._load_data(self._path)
+ if ok:
+ self.status = "OK"
+ self.last_update_time = datetime.now(pytz.UTC)
+ else:
+ self.status = "Error"
+ logging.info("Failed to load data for " + self.name)
+ except Exception as e:
+ self.status = "Error"
+ logging.error("Exception in local file Static Data Provider (" + self.name + ")", e, exc_info=True)
+
+ def stop(self):
+ pass
+
+ def _load_data(self, path):
+ """Load data from the given file path. Return true if successful, false otherwise."""
+
+ raise NotImplementedError("Subclasses must implement this method")
diff --git a/staticdataproviders/static_data_provider.py b/staticdataproviders/static_data_provider.py
new file mode 100644
index 0000000..97730de
--- /dev/null
+++ b/staticdataproviders/static_data_provider.py
@@ -0,0 +1,31 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+
+class StaticDataProvider:
+ """Generic static reference data provider class. Subclasses of this query the individual URLs or files for data."""
+
+ def __init__(self, name, provider_config):
+ """Constructor"""
+
+ self.name = name
+ self.enabled = provider_config["enabled"]
+ self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+ self.status = "Not Started" if self.enabled else "Disabled"
+ self.reference_count = 0
+
+
+ def start(self):
+ """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+ raise NotImplementedError("Subclasses must implement this method")
+
+
+ def stop(self):
+ """Stop any threads and prepare for application shutdown"""
+
+ raise NotImplementedError("Subclasses must implement this method")
diff --git a/templates/status.html b/templates/status.html
index 30681dc..2fbee72 100644
--- a/templates/status.html
+++ b/templates/status.html
@@ -54,6 +54,15 @@
+
+