w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-09-20 22:37:44 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Use ruff linter to fix issues and provide consistent formatting

Browse Source
This commit is contained in:
Ian Renton
2026-08-15 08:25:54 +01:00
parent 7391c28cd0
commit af3f82c14d
121 changed files with 1989 additions and 996 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/spothole.iml → .idea/Spothole.iml
Generated
+2 -2
View File
@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4"> <module external.system.id="pyproject.toml" type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager"> <component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$"> <content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$" />
<excludeFolder url="file://$MODULE_DIR$/.venv" /> <excludeFolder url="file://$MODULE_DIR$/.venv" />
<excludeFolder url="file://$MODULE_DIR$/static/vendor" /> <excludeFolder url="file://$MODULE_DIR$/static/vendor" />
</content> </content>
.idea/codeStyles/codeStyleConfig.xml
Generated
+5
View File
@@ -0,0 +1,5 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
</state>
</component>
.idea/modules.xml
Generated
+1 -1
View File
@@ -2,7 +2,7 @@
<project version="4"> <project version="4">
<component name="ProjectModuleManager"> <component name="ProjectModuleManager">
<modules> <modules>
<module fileurl="file://$PROJECT_DIR$/.idea/spothole.iml" filepath="$PROJECT_DIR$/.idea/spothole.iml" /> <module fileurl="file://$PROJECT_DIR$/.idea/Spothole.iml" filepath="$PROJECT_DIR$/.idea/Spothole.iml" />
</modules> </modules>
</component> </component>
</project> </project>
.idea/runConfigurations/Run.xml
Generated
+3 -1
View File
@@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager"> <component name="ProjectRunConfigurationManager">
<configuration default="false" name="Run" type="PythonConfigurationType" factoryName="Python"> <configuration default="false" name="Run" type="PythonConfigurationType" factoryName="Python">
<module name="spothole" /> <module name="Spothole" />
<option name="ENV_FILES" value="" /> <option name="ENV_FILES" value="" />
<option name="INTERPRETER_OPTIONS" value="" /> <option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" /> <option name="PARENT_ENVS" value="true" />
@@ -12,6 +12,8 @@
<option name="IS_MODULE_SDK" value="true" /> <option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" /> <option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" /> <option name="ADD_SOURCE_ROOTS" value="true" />
<option name="DEBUG_JUST_MY_CODE" value="true" />
<option name="RUN_TOOL" value="" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/spothole.py" /> <option name="SCRIPT_NAME" value="$PROJECT_DIR$/spothole.py" />
<option name="PARAMETERS" value="" /> <option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" /> <option name="SHOW_COMMAND_LINE" value="false" />
core/config.py
+5 -2
View File
@@ -7,7 +7,8 @@ import yaml
# Check you have a config file # Check you have a config file
if not os.path.isfile("config.yml"): if not os.path.isfile("config.yml"):
logging.error( logging.error(
"Your config file is missing. Ensure you have copied config-example.yml to config.yml and updated it according to your needs.") "Your config file is missing. Ensure you have copied config-example.yml to config.yml and updated it according to your needs."
)
exit() exit()
# Load config # Load config
@@ -30,7 +31,9 @@ LOG_LEVEL = config.get("log_level", "INFO")
LOG_WEB_REQUESTS = config.get("log_web_requests", False) LOG_WEB_REQUESTS = config.get("log_web_requests", False)
WEB_UI_OPTIONS["qrz_enabled"] = any(p["class"] == "QRZ" and p["enabled"] for p in config["callsign_data_providers"]) WEB_UI_OPTIONS["qrz_enabled"] = any(p["class"] == "QRZ" and p["enabled"] for p in config["callsign_data_providers"])
WEB_UI_OPTIONS["hamqth_enabled"] = any(p["class"] == "HamQTH" and p["enabled"] for p in config["callsign_data_providers"]) WEB_UI_OPTIONS["hamqth_enabled"] = any(
p["class"] == "HamQTH" and p["enabled"] for p in config["callsign_data_providers"]
)
WEB_UI_OPTIONS["recaptcha_site_key"] = RECAPTCHA_SITE_KEY WEB_UI_OPTIONS["recaptcha_site_key"] = RECAPTCHA_SITE_KEY
WEB_UI_OPTIONS["allow_upstream_spotting"] = ALLOW_SPOTTING and ALLOW_UPSTREAM_SPOTTING WEB_UI_OPTIONS["allow_upstream_spotting"] = ALLOW_SPOTTING and ALLOW_UPSTREAM_SPOTTING
core/constants.py
+169 -28
View File
@@ -11,36 +11,176 @@ HAMQTH_PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}"
# Special Interest Groups # Special Interest Groups
SIGS = [ SIGS = [
SIG(name="POTA", comment_names=["POTA"], description="Parks on the Air", ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST"), SIG(
SIG(name="SOTA", comment_names=["SOTA"], description="Summits on the Air", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}"), name="POTA",
SIG(name="WWFF", comment_names=["WWFF"], description="World Wide Flora & Fauna", ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}"), comment_names=["POTA"],
SIG(name="GMA", comment_names=["GMA"], description="Global Mountain Activity", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}"), description="Parks on the Air",
SIG(name="WWBOTA", comment_names=["WWBOTA", "BOTA"], description="Worldwide Bunkers on the Air", ref_regex=r"B\/[A-Z0-9]{1,3}\-\d{3,4}"), ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST",
SIG(name="HEMA", comment_names=["HEMA"], description="HuMPs Excluding Marilyns Award", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}"), ),
SIG(name="IOTA", comment_names=["IOTA"], description="Islands on the Air", ref_regex=r"[A-Z]{2}\-\d{3}"), SIG(
SIG(name="MOTA", comment_names=["MOTA"], description="Mills on the Air", ref_regex=r"X\d{4,6}"), name="SOTA",
SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}[\- ]\d{3,4}"), comment_names=["SOTA"],
SIG(name="ILLW", comment_names=["ILLW"], description="International Lighthouse & Lightship Weekend", ref_regex=r"[A-Z]{2}\d{4}"), description="Summits on the Air",
SIG(name="SIOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"), ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
SIG(name="WCA", comment_names=["WCA"], description="World Castles Award", ref_regex=r"[A-Z0-9]{1,3}\-\d{5}"), ),
SIG(name="ZLOTA", comment_names=["ZLOTA"], description="New Zealand on the Air", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}"), SIG(
SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"), name="WWFF",
comment_names=["WWFF"],
description="World Wide Flora & Fauna",
ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}",
),
SIG(
name="GMA",
comment_names=["GMA"],
description="Global Mountain Activity",
ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
),
SIG(
name="WWBOTA",
comment_names=["WWBOTA", "BOTA"],
description="Worldwide Bunkers on the Air",
ref_regex=r"B\/[A-Z0-9]{1,3}\-\d{3,4}",
),
SIG(
name="HEMA",
comment_names=["HEMA"],
description="HuMPs Excluding Marilyns Award",
ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}",
),
SIG(
name="IOTA",
comment_names=["IOTA"],
description="Islands on the Air",
ref_regex=r"[A-Z]{2}\-\d{3}",
),
SIG(
name="MOTA",
comment_names=["MOTA"],
description="Mills on the Air",
ref_regex=r"X\d{4,6}",
),
SIG(
name="ARLHS",
comment_names=["ARLHS"],
description="Amateur Radio Lighthouse Society",
ref_regex=r"[A-Z]{3}[\- ]\d{3,4}",
),
SIG(
name="ILLW",
comment_names=["ILLW"],
description="International Lighthouse & Lightship Weekend",
ref_regex=r"[A-Z]{2}\d{4}",
),
SIG(
name="SIOTA",
comment_names=["SIOTA"],
description="Silos on the Air",
ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d",
),
SIG(
name="WCA",
comment_names=["WCA"],
description="World Castles Award",
ref_regex=r"[A-Z0-9]{1,3}\-\d{5}",
),
SIG(
name="ZLOTA",
comment_names=["ZLOTA"],
description="New Zealand on the Air",
ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}",
),
SIG(
name="WOTA",
comment_names=["WOTA"],
description="Wainwrights on the Air",
ref_regex=r"[A-Z]{3}-[0-9]{2}",
),
SIG(name="BOTA", comment_names=[], description="Beaches on the Air"), SIG(name="BOTA", comment_names=[], description="Beaches on the Air"),
SIG(name="KRMNPA", comment_names=["KRMNPA"], description="Keith Roget Memorial National Parks Award", ref_regex=r"VKFF\-\d{4}"), SIG(
SIG(name="SANPCPA", comment_names=["SANPCPA"], description="South Australian National Parks and Conservation Parks Award", ref_regex=r"VKFF\-\d{4}"), name="KRMNPA",
SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"LL[A-Z]{2}\-\d{4}"), comment_names=["KRMNPA"],
SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2,3}R\-\d{4}"), description="Keith Roget Memorial National Parks Award",
SIG(name="Tiles", comment_names=[], description="Tiles on the Air", ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}"), ref_regex=r"VKFF\-\d{4}",
SIG(name="WAB", comment_names=["WAB"], description="Worked All Britain", ref_regex=r"[A-Z]{1,2}[0-9]{2}"), ),
SIG(name="WAI", comment_names=["WAI"], description="Worked All Ireland", ref_regex=r"[A-Z][0-9]{2}"), SIG(
SIG(name="DME", comment_names=["DME"], description="Diplomas de Municipios Españoles", ref_regex=r"\d{4,5}"), name="SANPCPA",
SIG(name="Toilets", comment_names=[], description="Toilets on the Air", ref_regex=r"T\-[0-9]{2}") comment_names=["SANPCPA"],
description="South Australian National Parks and Conservation Parks Award",
ref_regex=r"VKFF\-\d{4}",
),
SIG(
name="LLOTA",
comment_names=["LLOTA"],
description="Lagos y Lagunas on the Air",
ref_regex=r"LL[A-Z]{2}\-\d{4}",
),
SIG(
name="Towers",
comment_names=["TOTA"],
description="Towers on the Air",
ref_regex=r"[A-Z]{2,3}R\-\d{4}",
),
SIG(
name="Tiles",
comment_names=[],
description="Tiles on the Air",
ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}",
),
SIG(
name="WAB",
comment_names=["WAB"],
description="Worked All Britain",
ref_regex=r"[A-Z]{1,2}[0-9]{2}",
),
SIG(
name="WAI",
comment_names=["WAI"],
description="Worked All Ireland",
ref_regex=r"[A-Z][0-9]{2}",
),
SIG(
name="DME",
comment_names=["DME"],
description="Diplomas de Municipios Españoles",
ref_regex=r"\d{4,5}",
),
SIG(
name="Toilets",
comment_names=[],
description="Toilets on the Air",
ref_regex=r"T\-[0-9]{2}",
),
] ]
# Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA". # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
CW_MODES = ["CW"] CW_MODES = ["CW"]
PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "FUSION", "M17"] PHONE_MODES = [
DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "PSK", "OLIVIA", "PKT", "MSK144"] "PHONE",
"SSB",
"USB",
"LSB",
"AM",
"FM",
"DV",
"DMR",
"DSTAR",
"C4FM",
"FUSION",
"M17",
]
DATA_MODES = [
"DATA",
"FT8",
"FT4",
"RTTY",
"SSTV",
"JS8",
"HELL",
"PSK",
"OLIVIA",
"PKT",
"MSK144",
]
ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
MODE_TYPES = ["CW", "PHONE", "DATA"] MODE_TYPES = ["CW", "PHONE", "DATA"]
SSB_SUB_MODES = ["USB", "LSB"] SSB_SUB_MODES = ["USB", "LSB"]
@@ -58,7 +198,7 @@ MODE_ALIASES = {
"MFSK": "FSK", "MFSK": "FSK",
"MFSK32": "FSK", "MFSK32": "FSK",
"DIGI": "DATA", "DIGI": "DATA",
"DIGITAL": "DATA" "DIGITAL": "DATA",
} }
# Band definitions # Band definitions
@@ -88,7 +228,8 @@ BANDS = [
Band(name="10GHz", start_freq=10000000000, end_freq=10500000000), Band(name="10GHz", start_freq=10000000000, end_freq=10500000000),
Band(name="24GHz", start_freq=24000000000, end_freq=24050000000), Band(name="24GHz", start_freq=24000000000, end_freq=24050000000),
Band(name="47GHz", start_freq=47000000000, end_freq=47200000000), Band(name="47GHz", start_freq=47000000000, end_freq=47200000000),
Band(name="76GHz", start_freq=75500000000, end_freq=81500000000)] Band(name="76GHz", start_freq=75500000000, end_freq=81500000000),
]
UNKNOWN_BAND = Band(name="Unknown", start_freq=0, end_freq=0) UNKNOWN_BAND = Band(name="Unknown", start_freq=0, end_freq=0)
# Continents # Continents
@@ -106,5 +247,5 @@ PROPAGATION_MODES = {
"MS": "Meteor scatter", "MS": "Meteor scatter",
"RS": "Rain scatter", "RS": "Rain scatter",
"AS": "Aircraft scatter", "AS": "Aircraft scatter",
"SAT": "Satellite" "SAT": "Satellite",
} }
core/data_providers.py
+14 -5
View File
@@ -15,7 +15,6 @@ class DataProviders:
self.sig_ref_data_providers = [] self.sig_ref_data_providers = []
self.callsign_data_providers = [] self.callsign_data_providers = []
def setup(self): def setup(self):
for entry in config["spot_providers"]: for entry in config["spot_providers"]:
self.spot_providers.append(create_provider_from_config("providers.spot", entry)) self.spot_providers.append(create_provider_from_config("providers.spot", entry))
@@ -34,7 +33,7 @@ class DataProviders:
def start_providers(providers, provider_type): def start_providers(providers, provider_type):
"""Helper method to activate enabled providers in the list.""" """Helper method to activate enabled providers in the list."""
logging.info(f"Starting %s providers...", provider_type) logging.info(f"Starting {provider_type} providers...")
for p in providers: for p in providers:
if p.enabled: if p.enabled:
p.start() p.start()
@@ -43,11 +42,20 @@ class DataProviders:
# Start data providers before spot/alert providers so the lookup data is there already for incoming spots. # Start data providers before spot/alert providers so the lookup data is there already for incoming spots.
# Each category is fired off after a small delay to give the rest of Spothole chance to start up. # Each category is fired off after a small delay to give the rest of Spothole chance to start up.
threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")).start() threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")).start()
threading.Timer(10.0, lambda: self.start_providers(self.callsign_data_providers, "callsign data")).start() threading.Timer(
10.0,
lambda: self.start_providers(self.callsign_data_providers, "callsign data"),
).start()
threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")).start() threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")).start()
threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")).start() threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")).start()
threading.Timer(25.0, lambda: self.start_providers(self.solar_condition_providers, "solar condition")).start() threading.Timer(
threading.Timer(30.0, lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data")).start() 25.0,
lambda: self.start_providers(self.solar_condition_providers, "solar condition"),
).start()
threading.Timer(
30.0,
lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data"),
).start()
def stop(self): def stop(self):
for sp in self.spot_providers: for sp in self.spot_providers:
@@ -69,5 +77,6 @@ class DataProviders:
if cdp.enabled: if cdp.enabled:
cdp.stop() cdp.stop()
# Global object # Global object
DATA_PROVIDERS = DataProviders() DATA_PROVIDERS = DataProviders()
core/data_store.py
+23 -11
View File
@@ -4,7 +4,7 @@ from pathlib import Path
import diskcache import diskcache
from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE from core.config import MAX_ALERT_AGE, MAX_SPOT_AGE
from core.live_data_cache import LiveDataCache from core.live_data_cache import LiveDataCache
from data.solar_conditions import SolarConditions from data.solar_conditions import SolarConditions
@@ -64,7 +64,7 @@ class DataStore:
# of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref" # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
# syntax for keys to keep it a single level. # syntax for keys to keep it a single level.
self.sigrefs = diskcache.Cache(f"{CACHE_DIR}sigrefs") self.sigrefs = diskcache.Cache(f"{CACHE_DIR}sigrefs")
logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs)) logging.info(f"Loaded data for {len(self.sigrefs)} SIG references.")
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup. # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
@@ -75,23 +75,34 @@ class DataStore:
self.callsign_data_qrz = diskcache.Cache(f"{CACHE_DIR}callsign_data_qrz") self.callsign_data_qrz = diskcache.Cache(f"{CACHE_DIR}callsign_data_qrz")
self.callsign_data_hamqth = diskcache.Cache(f"{CACHE_DIR}callsign_data_hamqth") self.callsign_data_hamqth = diskcache.Cache(f"{CACHE_DIR}callsign_data_hamqth")
unique_keys = set() unique_keys = set()
for c in [self.callsign_data_countryfiles, self.callsign_data_clublogxml, self.callsign_data_clublogapi, for c in [
self.callsign_data_qrz, self.callsign_data_hamqth]: self.callsign_data_countryfiles,
self.callsign_data_clublogxml,
self.callsign_data_clublogapi,
self.callsign_data_qrz,
self.callsign_data_hamqth,
]:
unique_keys.update(c) unique_keys.update(c)
logging.info(f"Loaded data for %d callsigns.", len(unique_keys)) logging.info(f"Loaded data for {len(unique_keys)} callsigns.")
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
# specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
# before the spots and alerts are live in the system. # before the spots and alerts are live in the system.
self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE, self.spots = LiveDataCache(
maxsize=self._MAX_SPOT_COUNT,
ttl=MAX_SPOT_AGE,
snapshot_dir=f"{CACHE_DIR}spots", snapshot_dir=f"{CACHE_DIR}spots",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC,
logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys())) )
logging.info(f"Loaded {len(self.spots.keys())} spots from a previous run.")
self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE, self.alerts = LiveDataCache(
maxsize=self._MAX_ALERT_COUNT,
ttl=MAX_ALERT_AGE,
snapshot_dir=f"{CACHE_DIR}alerts", snapshot_dir=f"{CACHE_DIR}alerts",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC,
logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys())) )
logging.info(f"Loaded {len(self.alerts.keys())} alerts from a previous run.")
def regenerate_call_regex_to_dxcc_entity_map(self): def regenerate_call_regex_to_dxcc_entity_map(self):
"""DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC """DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC
@@ -118,5 +129,6 @@ class DataStore:
self.callsign_data_qrz.close() self.callsign_data_qrz.close()
self.callsign_data_hamqth.close() self.callsign_data_hamqth.close()
# Global object # Global object
DATA_STORE = DataStore() DATA_STORE = DataStore()
core/geo_utils.py
+2 -2
View File
@@ -113,8 +113,8 @@ def lat_lon_for_grid_sw_corner_plus_size(grid):
while block * 2 < length: while block * 2 < length:
if block % 2 == 0: if block % 2 == 0:
# Letters in this block # Letters in this block
lon_cell_no = ord(grid[block * 2]) - ord('A') lon_cell_no = ord(grid[block * 2]) - ord("A")
lat_cell_no = ord(grid[block * 2 + 1]) - ord('A') lat_cell_no = ord(grid[block * 2 + 1]) - ord("A")
# Bail if the values aren't in range. Allowed values are A-R (0-17) for the first letter block, or # Bail if the values aren't in range. Allowed values are A-R (0-17) for the first letter block, or
# A-X (0-23) thereafter. # A-X (0-23) thereafter.
max_cell_no = 17 if block == 0 else 23 max_cell_no = 17 if block == 0 else 23
core/live_data_cache.py
+3 -4
View File
@@ -34,8 +34,7 @@ class LiveDataCache:
try: try:
callback(value) callback(value)
except Exception: except Exception:
logging.exception("Listener raised an exception for key %s", key) logging.exception(f"Listener raised an exception for key {key}")
def get(self, key, default=None): def get(self, key, default=None):
with self._lock: with self._lock:
@@ -71,7 +70,7 @@ class LiveDataCache:
try: try:
self._disk_cache.set("snapshot", data) self._disk_cache.set("snapshot", data)
except Exception as e: except Exception as e:
logging.exception("Failed to write snapshot to %s", self._snapshot_dir, e) logging.exception(f"Failed to write snapshot to {self._snapshot_dir}")
def _load_snapshot(self): def _load_snapshot(self):
data = self._disk_cache.get("snapshot") data = self._disk_cache.get("snapshot")
@@ -84,7 +83,7 @@ class LiveDataCache:
# Only restore entries that would still be within TTL # Only restore entries that would still be within TTL
if now - saved_at < self._ttl: if now - saved_at < self._ttl:
self._cache[key] = value self._cache[key] = value
logging.info("Loaded snapshot from %s", self._snapshot_dir) logging.info(f"Loaded snapshot from {self._snapshot_dir}")
def _start_periodic_snapshot(self, interval): def _start_periodic_snapshot(self, interval):
def loop(): def loop():
core/prometheus_metrics_handler.py
+11 -17
View File
@@ -1,4 +1,10 @@
from prometheus_client import CollectorRegistry, generate_latest, Counter, disable_created_metrics, Gauge from prometheus_client import (
CollectorRegistry,
Counter,
Gauge,
disable_created_metrics,
generate_latest,
)
disable_created_metrics() disable_created_metrics()
# Prometheus metrics registry # Prometheus metrics registry
@@ -9,25 +15,13 @@ page_requests_counter = Counter(
"Total number of page requests received", "Total number of page requests received",
registry=registry, registry=registry,
) )
api_requests_counter = Counter( api_requests_counter = Counter("spothole_api_requests", "Total number of API requests received", registry=registry)
"spothole_api_requests", spots_gauge = Gauge("spothole_spots", "Number of spots currently in the software", registry=registry)
"Total number of API requests received", alerts_gauge = Gauge("spothole_alerts", "Number of alerts currently in the software", registry=registry)
registry=registry
)
spots_gauge = Gauge(
"spothole_spots",
"Number of spots currently in the software",
registry=registry
)
alerts_gauge = Gauge(
"spothole_alerts",
"Number of alerts currently in the software",
registry=registry
)
memory_use_gauge = Gauge( memory_use_gauge = Gauge(
"spothole_memory_usage_bytes", "spothole_memory_usage_bytes",
"Current memory usage of the software in bytes", "Current memory usage of the software in bytes",
registry=registry registry=registry,
) )
core/sig_lookup_helper.py
+2 -2
View File
@@ -1,6 +1,6 @@
import logging import logging
from pyhamtools.locator import locator_to_latlong, latlong_to_locator from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from core.data_store import DATA_STORE from core.data_store import DATA_STORE
from core.geo_utils import wab_wai_square_to_lat_lon from core.geo_utils import wab_wai_square_to_lat_lon
@@ -83,7 +83,7 @@ def get_sig_ref_info(sig, ref_id):
else: else:
# Maybe a super new reference we don't know about yet, but more likely a typo or a test reference, # Maybe a super new reference we don't know about yet, but more likely a typo or a test reference,
# just silently ignore it. # just silently ignore it.
logging.debug("%s database did not contain data for ref %s", sig, ref_id) logging.debug(f"{sig} database did not contain data for ref {ref_id}")
except Exception: except Exception:
logging.exception(f"Exception when looking up sig_ref info for {sig} ref {ref_id}") logging.exception(f"Exception when looking up sig_ref info for {sig} ref {ref_id}")
core/sig_utils.py
+1 -1
View File
@@ -21,4 +21,4 @@ def get_sig_name_from_comment_name(sig):
# Regex matching any SIG's "comment name", i.e. how it may be referred to in spot comments # Regex matching any SIG's "comment name", i.e. how it may be referred to in spot comments
ANY_SIG_REGEX = rf"({'|'.join((n for s in SIGS for n in s.comment_names))})" ANY_SIG_REGEX = rf"({'|'.join(n for s in SIGS for n in s.comment_names)})"
core/status_reporter.py
+100 -45
View File
@@ -1,6 +1,6 @@
import os import os
from datetime import datetime from datetime import datetime
from threading import Thread, Event from threading import Event, Thread
import psutil import psutil
import pytz import pytz
@@ -10,7 +10,7 @@ from core.config import SERVER_OWNER_CALLSIGN
from core.constants import SOFTWARE_VERSION from core.constants import SOFTWARE_VERSION
from core.data_providers import DATA_PROVIDERS from core.data_providers import DATA_PROVIDERS
from core.data_store import DATA_STORE from core.data_store import DATA_STORE
from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge from core.prometheus_metrics_handler import alerts_gauge, memory_use_gauge, spots_gauge
from server.webserver import WEB_SERVER from server.webserver import WEB_SERVER
@@ -55,55 +55,110 @@ class StatusReporter:
DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values()) DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values()) DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
DATA_STORE.status_data["spot_providers"] = list( DATA_STORE.status_data["spot_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, map(
"enabled_by_default_in_web_ui": p.enabled_by_default_in_web_ui, "status": p.status, lambda p: {
"last_updated": p.last_update_time.replace( "name": p.name,
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0, "enabled": p.enabled,
"last_spot": p.last_spot_time.replace( "enabled_by_default_in_web_ui": p.enabled_by_default_in_web_ui,
tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0}, "status": p.status,
DATA_PROVIDERS.spot_providers)) "last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
"last_spot": p.last_spot_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_spot_time.year > 2000
else 0,
},
DATA_PROVIDERS.spot_providers,
)
)
DATA_STORE.status_data["alert_providers"] = list( DATA_STORE.status_data["alert_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, map(
"last_updated": p.last_update_time.replace( lambda p: {
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, "name": p.name,
DATA_PROVIDERS.alert_providers)) "enabled": p.enabled,
"status": p.status,
"last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
},
DATA_PROVIDERS.alert_providers,
)
)
DATA_STORE.status_data["solar_condition_providers"] = list( DATA_STORE.status_data["solar_condition_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, map(
"last_updated": p.last_update_time.replace( lambda p: {
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, "name": p.name,
DATA_PROVIDERS.solar_condition_providers)) "enabled": p.enabled,
"status": p.status,
"last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
},
DATA_PROVIDERS.solar_condition_providers,
)
)
DATA_STORE.status_data["static_data_providers"] = list( DATA_STORE.status_data["static_data_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, map(
"last_updated": p.last_update_time.replace( lambda p: {
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0}, "name": p.name,
DATA_PROVIDERS.static_data_providers)) "enabled": p.enabled,
"status": p.status,
"last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
},
DATA_PROVIDERS.static_data_providers,
)
)
DATA_STORE.status_data["sig_ref_data_providers"] = list( DATA_STORE.status_data["sig_ref_data_providers"] = list(
map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status, map(
"last_updated": p.last_update_time.replace( lambda p: {
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0, "sig_name": p.sig_name,
"reference_count": p.reference_count}, "enabled": p.enabled,
DATA_PROVIDERS.sig_ref_data_providers)) "status": p.status,
"last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
"reference_count": p.reference_count,
},
DATA_PROVIDERS.sig_ref_data_providers,
)
)
DATA_STORE.status_data["callsign_data_providers"] = list( DATA_STORE.status_data["callsign_data_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, map(
"last_updated": p.last_update_time.replace( lambda p: {
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0, "name": p.name,
"lookup_count": p.lookup_count}, "enabled": p.enabled,
DATA_PROVIDERS.callsign_data_providers)) "status": p.status,
DATA_STORE.status_data["cleanup"] = {"status": CLEANUP_TIMER.status, "last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
"last_ran": CLEANUP_TIMER.last_cleanup_time.replace( if p.last_update_time.year > 2000
tzinfo=pytz.UTC).timestamp() if CLEANUP_TIMER.last_cleanup_time else 0} else 0,
DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"], "lookup_count": p.lookup_count,
"last_api_access": WEB_SERVER.web_server_metrics[ },
"last_api_access_time"].replace( DATA_PROVIDERS.callsign_data_providers,
tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[ )
"last_api_access_time"] else 0, )
DATA_STORE.status_data["cleanup"] = {
"status": CLEANUP_TIMER.status,
"last_ran": CLEANUP_TIMER.last_cleanup_time.replace(tzinfo=pytz.UTC).timestamp()
if CLEANUP_TIMER.last_cleanup_time
else 0,
}
DATA_STORE.status_data["webserver"] = {
"status": WEB_SERVER.web_server_metrics["status"],
"last_api_access": WEB_SERVER.web_server_metrics["last_api_access_time"]
.replace(tzinfo=pytz.UTC)
.timestamp()
if WEB_SERVER.web_server_metrics["last_api_access_time"]
else 0,
"api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"], "api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"],
"last_page_access": WEB_SERVER.web_server_metrics[ "last_page_access": WEB_SERVER.web_server_metrics["last_page_access_time"]
"last_page_access_time"].replace( .replace(tzinfo=pytz.UTC)
tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[ .timestamp()
"last_page_access_time"] else 0, if WEB_SERVER.web_server_metrics["last_page_access_time"]
"page_access_count": WEB_SERVER.web_server_metrics[ else 0,
"page_access_counter"]} "page_access_count": WEB_SERVER.web_server_metrics["page_access_counter"],
}
# Update Prometheus metrics # Update Prometheus metrics
memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss) memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)
core/url_data_cache.py
+5 -2
View File
@@ -17,8 +17,11 @@ class URLDataCache(CachedSession):
_lock = threading.Lock() _lock = threading.Lock()
def __init__(self, name): def __init__(self, name):
super().__init__(f"{CACHE_DIR}urls/{name}", expire_after=timedelta(days=1), super().__init__(
allowable_codes=(200, 400, 401, 403, 404)) f"{CACHE_DIR}urls/{name}",
expire_after=timedelta(days=1),
allowable_codes=(200, 400, 401, 403, 404),
)
def get(self, *args, **kwargs): def get(self, *args, **kwargs):
with self._lock: with self._lock:
core/utils.py
+31 -7
View File
@@ -4,7 +4,15 @@ import simplejson
from pyhamtools.frequency import freq_to_band from pyhamtools.frequency import freq_to_band
from pyhamtools.locator import latlong_to_locator from pyhamtools.locator import latlong_to_locator
from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES from core.constants import (
ALL_MODES,
BANDS,
CW_MODES,
DATA_MODES,
MODE_ALIASES,
PHONE_MODES,
UNKNOWN_BAND,
)
from core.data_store import DATA_STORE from core.data_store import DATA_STORE
from data.callsign import Callsign from data.callsign import Callsign
@@ -63,11 +71,25 @@ def infer_mode_from_frequency(freq):
# a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
# might include other common digimode centres of activity here, but this achieves the main goal of keeping # might include other common digimode centres of activity here, but this achieves the main goal of keeping
# large numbers of clearly-FT* spots off the list of people filtering out digimodes. # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or ( if (
21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077): (7074 <= khz < 7077)
or (10136 <= khz < 10139)
or (14074 <= khz < 14077)
or (18100 <= khz < 18103)
or (21074 <= khz < 21077)
or (24915 <= khz < 24918)
or (28074 <= khz < 28077)
):
mode = "FT8" mode = "FT8"
if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or ( if (
18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183): (7047.5 <= khz < 7050.5)
or (10140 <= khz < 10143)
or (14080 <= khz < 14083)
or (18104 <= khz < 18107)
or (21140 <= khz < 21143)
or (24919 <= khz < 24922)
or (28180 <= khz < 28183)
):
mode = "FT4" mode = "FT4"
return mode return mode
except KeyError: except KeyError:
@@ -100,7 +122,8 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
if lat and lon: if lat and lon:
grid = latlong_to_locator(lat, lon) grid = latlong_to_locator(lat, lon)
return Callsign(call=callsign, return Callsign(
call=callsign,
home_call=home_call, home_call=home_call,
country=country, country=country,
dxcc_id=dxcc_id, dxcc_id=dxcc_id,
@@ -110,7 +133,8 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
latitude=lat, latitude=lat,
longitude=lon, longitude=lon,
grid=grid, grid=grid,
location_source="DXCC") location_source="DXCC",
)
except (KeyError, ValueError): except (KeyError, ValueError):
# Unknown callsign, can't look anything up, return a Callsign object with basic data so that gets cached # Unknown callsign, can't look anything up, return a Callsign object with basic data so that gets cached
data/alert.py
+8 -5
View File
@@ -120,12 +120,12 @@ class Alert:
self.id = hashlib.sha256(str({"s": self.source, "sid": self.source_id}).encode("utf-8")).hexdigest() self.id = hashlib.sha256(str({"s": self.source, "sid": self.source_id}).encode("utf-8")).hexdigest()
else: else:
self.id = hashlib.sha256( self.id = hashlib.sha256(
str({"s": self.source, "c": self.dx_calls, "t": self.start_time}).encode("utf-8")).hexdigest() str({"s": self.source, "c": self.dx_calls, "t": self.start_time}).encode("utf-8")
).hexdigest()
# DX operator name lookup, using QRZ.com/HamQTH. # DX operator name lookup, using QRZ.com/HamQTH.
if self.dx_calls and not self.dx_names: if self.dx_calls and not self.dx_names:
self.dx_names = list( self.dx_names = list(map(lambda c: get_call_info(c, credentials).name, self.dx_calls))
map(lambda c: get_call_info(c, credentials).name, self.dx_calls))
except Exception: except Exception:
logging.exception("Exception while inferring missing data from spot") logging.exception("Exception while inferring missing data from spot")
@@ -141,5 +141,8 @@ class Alert:
either having an end_time in the past, or if it only has a start_time, then that start time was more than 3 hours either having an end_time in the past, or if it only has a start_time, then that start time was more than 3 hours
ago. If it somehow doesn't have a start_time either, it is considered to be expired.""" ago. If it somehow doesn't have a start_time either, it is considered to be expired."""
return not self.start_time or (self.end_time and self.end_time < datetime.now(pytz.UTC).timestamp()) or ( return (
not self.end_time and self.start_time < (datetime.now(pytz.UTC) - timedelta(hours=3)).timestamp()) not self.start_time
or (self.end_time and self.end_time < datetime.now(pytz.UTC).timestamp())
or (not self.end_time and self.start_time < (datetime.now(pytz.UTC) - timedelta(hours=3)).timestamp())
)
data/callsign.py
+13 -3
View File
@@ -42,6 +42,16 @@ class Callsign:
"""Utility method to indicate that the callsign data is fully populated. Multiple providers can return data for """Utility method to indicate that the callsign data is fully populated. Multiple providers can return data for
a callsign, and we try them in sequence until we have all the data we can, in which case there's no point a callsign, and we try them in sequence until we have all the data we can, in which case there's no point
querying any other providers.""" querying any other providers."""
return self.home_call is not None and self.name is not None and self.qth is not None and self.grid is not None\ return (
and self.latitude is not None and self.longitude is not None and self.country is not None and self.continent\ self.home_call is not None
is not None and self.dxcc_id is not None and self.cq_zone is not None and self.itu_zone is not None and self.name is not None
and self.qth is not None
and self.grid is not None
and self.latitude is not None
and self.longitude is not None
and self.country is not None
and self.continent is not None
and self.dxcc_id is not None
and self.cq_zone is not None
and self.itu_zone is not None
)
data/lookup_credentials.py
+1
View File
@@ -4,6 +4,7 @@ from dataclasses import dataclass
@dataclass @dataclass
class LookupCredentials: class LookupCredentials:
"""Per-request credentials for QRZ.com and HamQTH online callsign lookups.""" """Per-request credentials for QRZ.com and HamQTH online callsign lookups."""
qrz_username: str = "" qrz_username: str = ""
qrz_password: str = "" qrz_password: str = ""
qrz_session_key: str = "" # alternative to username/password qrz_session_key: str = "" # alternative to username/password
data/solar_conditions.py
+2 -2
View File
@@ -82,9 +82,9 @@ def _xray_blackout_scale(xray):
number = float(xray[1:]) number = float(xray[1:])
except ValueError: except ValueError:
return 0 return 0
if letter == 'M': if letter == "M":
return 1 if number < 5 else 2 return 1 if number < 5 else 2
if letter == 'X': if letter == "X":
if number < 10: if number < 10:
return 3 return 3
if number < 20: if number < 20:
data/spot.py
+46 -24
View File
@@ -1,5 +1,4 @@
import hashlib import hashlib
import hashlib
import json import json
import logging import logging
import re import re
@@ -7,16 +6,25 @@ from dataclasses import dataclass
from datetime import datetime, timedelta from datetime import datetime, timedelta
import pytz import pytz
from pyhamtools.locator import locator_to_latlong, latlong_to_locator from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from core.call_lookup_helper import get_call_info from core.call_lookup_helper import get_call_info
from core.config import MAX_SPOT_AGE from core.config import MAX_SPOT_AGE
from core.constants import MODE_ALIASES, PROPAGATION_MODES from core.constants import MODE_ALIASES, PROPAGATION_MODES
from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
from core.sig_lookup_helper import populate_missing_sig_ref_info from core.sig_lookup_helper import populate_missing_sig_ref_info
from core.sig_utils import ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name from core.sig_utils import (
from core.utils import infer_band_from_freq, infer_mode_from_comment, \ ANY_SIG_REGEX,
infer_mode_from_frequency, infer_mode_type_from_mode, get_flag_for_dxcc get_ref_regex_for_sig,
get_sig_name_from_comment_name,
)
from core.utils import (
get_flag_for_dxcc,
infer_band_from_freq,
infer_mode_from_comment,
infer_mode_from_frequency,
infer_mode_type_from_mode,
)
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
@@ -141,10 +149,7 @@ class Spot:
objects such as the sig_refs list..""" objects such as the sig_refs list.."""
if self.sig_refs: if self.sig_refs:
self.sig_refs = [ self.sig_refs = [sig_ref if isinstance(sig_ref, SIGRef) else SIGRef(**sig_ref) for sig_ref in self.sig_refs]
sig_ref if isinstance(sig_ref, SIGRef) else SIGRef(**sig_ref)
for sig_ref in self.sig_refs
]
def infer_missing(self, credentials=None): def infer_missing(self, credentials=None):
"""Infer missing parameters where possible""" """Infer missing parameters where possible"""
@@ -207,8 +212,11 @@ class Spot:
# Spotter country, continent, zones etc. from callsign. # Spotter country, continent, zones etc. from callsign.
# DE call with no digits, or APRS servers starting "T2" are not things we can look up location for # DE call with no digits, or APRS servers starting "T2" are not things we can look up location for
de_call_info = get_call_info(self.de_call, credentials) de_call_info = get_call_info(self.de_call, credentials)
if self.de_call and any(char.isdigit() for char in self.de_call) and not ( if (
self.de_call.startswith("T2") and self.source == "APRS-IS"): self.de_call
and any(char.isdigit() for char in self.de_call)
and not (self.de_call.startswith("T2") and self.source == "APRS-IS")
):
if not self.de_country: if not self.de_country:
self.de_country = de_call_info.country self.de_country = de_call_info.country
if not self.de_continent: if not self.de_continent:
@@ -279,9 +287,11 @@ class Spot:
# If so, add that to the sig_refs list for this spot. # If so, add that to the sig_refs list for this spot.
ref_regex = get_ref_regex_for_sig(found_sig) ref_regex = get_ref_regex_for_sig(found_sig)
if ref_regex: if ref_regex:
ref_matches = re.finditer(r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)", ref_matches = re.finditer(
r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)",
self.comment, self.comment,
re.IGNORECASE) re.IGNORECASE,
)
for ref_match in ref_matches: for ref_match in ref_matches:
self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig)) self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig))
@@ -312,8 +322,9 @@ class Spot:
# being the only source we have for propagation mode. Brace for nightmare regex from hell. # being the only source we have for propagation mode. Brace for nightmare regex from hell.
if self.comment: if self.comment:
grid_mode_grid_match = re.search( grid_mode_grid_match = re.search(
r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b', r"\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b",
self.comment) self.comment,
)
if grid_mode_grid_match: if grid_mode_grid_match:
# regex matches, so extract grids: # regex matches, so extract grids:
if not self.de_grid: if not self.de_grid:
@@ -329,13 +340,14 @@ class Spot:
self.propagation_mode = PROPAGATION_MODES[mode_tag] self.propagation_mode = PROPAGATION_MODES[mode_tag]
else: else:
self.propagation_mode = mode_tag self.propagation_mode = mode_tag
logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag) logging.info(f"Seen a new propagation mode tag not yet in the system: {mode_tag}")
# Parse "de_grid -> dx_grid" structures from the comment # Parse "de_grid -> dx_grid" structures from the comment
if self.comment: if self.comment:
grid_mode_grid_match = re.search( grid_mode_grid_match = re.search(
r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b', r"\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b",
self.comment) self.comment,
)
if grid_mode_grid_match: if grid_mode_grid_match:
# regex matches, so extract grids: # regex matches, so extract grids:
if not self.dx_grid: if not self.dx_grid:
@@ -370,7 +382,8 @@ class Spot:
self.id = hashlib.sha256(str({"s": self.source, "sid": self.source_id}).encode("utf-8")).hexdigest() self.id = hashlib.sha256(str({"s": self.source, "sid": self.source_id}).encode("utf-8")).hexdigest()
else: else:
self.id = hashlib.sha256( self.id = hashlib.sha256(
str({"s": self.source, "c": self.dx_call, "t": self.time}).encode("utf-8")).hexdigest() str({"s": self.source, "c": self.dx_call, "t": self.time}).encode("utf-8")
).hexdigest()
# DX operator details lookup. This should be the last resort compared to taking the data from the actual # DX operator details lookup. This should be the last resort compared to taking the data from the actual
# spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon or DXCC lat/lon # spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon or DXCC lat/lon
@@ -407,14 +420,23 @@ class Spot:
# DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
# is likely at home. # is likely at home.
self.dx_location_good = bool(self.dx_latitude and self.dx_longitude and ( self.dx_location_good = bool(
self.dx_location_source == "SPOT" or self.dx_location_source == "SIG REF LOOKUP" self.dx_latitude
and self.dx_longitude
and (
self.dx_location_source == "SPOT"
or self.dx_location_source == "SIG REF LOOKUP"
or self.dx_location_source == "GRID" or self.dx_location_source == "GRID"
or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or "")))) or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or ""))
)
)
# DE with no digits and APRS servers starting "T2" are not things we can look up location for # DE with no digits and APRS servers starting "T2" are not things we can look up location for
if self.de_call and any(char.isdigit() for char in str(self.de_call)) and not ( if (
self.de_call.startswith("T2") and self.source == "APRS-IS"): self.de_call
and any(char.isdigit() for char in str(self.de_call))
and not (self.de_call.startswith("T2") and self.source == "APRS-IS")
):
# DE operator location lookup # DE operator location lookup
if not self.de_latitude: if not self.de_latitude:
self.de_latitude = de_call_info.latitude self.de_latitude = de_call_info.latitude
providers/alert/alert_provider.py
+1 -1
View File
@@ -29,7 +29,7 @@ class AlertProvider:
# Sort the batch so that earliest ones go in first. This helps keep the ordering correct when alerts are fired # Sort the batch so that earliest ones go in first. This helps keep the ordering correct when alerts are fired
# off to SSE listeners. # off to SSE listeners.
alerts = sorted(alerts, key=lambda a: (a.start_time if a and a.start_time else 0)) alerts = sorted(alerts, key=lambda a: a.start_time if a and a.start_time else 0)
for alert in alerts: for alert in alerts:
# Fill in any blanks and add to the list # Fill in any blanks and add to the list
alert.infer_missing() alert.infer_missing()
providers/alert/bota.py
+13 -11
View File
@@ -3,9 +3,9 @@ from datetime import datetime, timedelta
import pytz import pytz
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert from data.alert import Alert
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class BOTA(HTTPAlertProvider): class BOTA(HTTPAlertProvider):
@@ -23,17 +23,17 @@ class BOTA(HTTPAlertProvider):
bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml") bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
if not bs.body: if not bs.body:
return new_alerts return new_alerts
div = bs.body.find('div', attrs={'class': 'view-activations-public'}) div = bs.body.find("div", attrs={"class": "view-activations-public"})
if div: if div:
table = div.find('table', attrs={'class': 'views-table'}) table = div.find("table", attrs={"class": "views-table"})
if table: if table:
tbody = table.find('tbody') tbody = table.find("tbody")
if not tbody: if not tbody:
return new_alerts return new_alerts
for row in tbody.find_all('tr'): for row in tbody.find_all("tr"):
cells = row.find_all('td') cells = row.find_all("td")
first_cell_anchor = cells[0].find('a') if len(cells) > 0 else None first_cell_anchor = cells[0].find("a") if len(cells) > 0 else None
second_cell_anchor = cells[1].find('a') if len(cells) > 1 else None second_cell_anchor = cells[1].find("a") if len(cells) > 1 else None
if not first_cell_anchor or not second_cell_anchor: if not first_cell_anchor or not second_cell_anchor:
continue continue
first_cell_text = first_cell_anchor.get_text().strip() first_cell_text = first_cell_anchor.get_text().strip()
@@ -41,7 +41,7 @@ class BOTA(HTTPAlertProvider):
dx_call = second_cell_anchor.get_text().strip().upper() dx_call = second_cell_anchor.get_text().strip().upper()
# Get the date, dealing with the fact we get no year so have to figure out if it's last year or next year # Get the date, dealing with the fact we get no year so have to figure out if it's last year or next year
date_span = cells[2].find('span') if len(cells) > 2 else None date_span = cells[2].find("span") if len(cells) > 2 else None
if not date_span: if not date_span:
continue continue
date_text = date_span.get_text().strip() date_text = date_span.get_text().strip()
@@ -52,11 +52,13 @@ class BOTA(HTTPAlertProvider):
date_time = date_time.replace(year=datetime.now(pytz.UTC).year + 1) date_time = date_time.replace(year=datetime.now(pytz.UTC).year + 1)
# Convert to our alert format # Convert to our alert format
alert = Alert(source=self.name, alert = Alert(
source=self.name,
dx_calls=[dx_call], dx_calls=[dx_call],
sig_refs=[SIGRef(id=ref_name, sig="BOTA")], sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
start_time=date_time.timestamp(), start_time=date_time.timestamp(),
is_dxpedition=False) is_dxpedition=False,
)
new_alerts.append(alert) new_alerts.append(alert)
return new_alerts return new_alerts
providers/alert/http_alert_provider.py
+3 -3
View File
@@ -1,13 +1,13 @@
import logging import logging
from datetime import datetime from datetime import datetime
from threading import Thread, Event from threading import Event, Thread
import pytz import pytz
import requests import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from providers.alert.alert_provider import AlertProvider
from core.constants import HTTP_HEADERS from core.constants import HTTP_HEADERS
from providers.alert.alert_provider import AlertProvider
class HTTPAlertProvider(AlertProvider): class HTTPAlertProvider(AlertProvider):
providers/alert/ng3k.py
+15 -8
View File
@@ -6,8 +6,8 @@ import pytz
from rss_parser import Parser from rss_parser import Parser
from rss_parser.models.rss import RSS from rss_parser.models.rss import RSS
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert from data.alert import Alert
from providers.alert.http_alert_provider import HTTPAlertProvider
class NG3K(HTTPAlertProvider): class NG3K(HTTPAlertProvider):
@@ -49,11 +49,16 @@ class NG3K(HTTPAlertProvider):
end_day = end_string.split(", ")[0].strip() end_day = end_string.split(", ")[0].strip()
end_mon = start_mon end_mon = start_mon
start_timestamp = datetime.strptime(f"{start_year} {start_mon} {start_day}", "%Y %b %d").replace( start_timestamp = (
tzinfo=pytz.UTC).timestamp() datetime.strptime(f"{start_year} {start_mon} {start_day}", "%Y %b %d")
end_timestamp = datetime.strptime(f"{end_year} {end_mon} {end_day} 23:59", .replace(tzinfo=pytz.UTC)
"%Y %b %d %H:%M").replace( .timestamp()
tzinfo=pytz.UTC).timestamp() )
end_timestamp = (
datetime.strptime(f"{end_year} {end_mon} {end_day} 23:59", "%Y %b %d %H:%M")
.replace(tzinfo=pytz.UTC)
.timestamp()
)
# Sometimes the DX callsign is "real", sometimes you just get a prefix with the real working callsigns being # Sometimes the DX callsign is "real", sometimes you just get a prefix with the real working callsigns being
# provided in the "by" field. e.g. call="JW", by="By LA7XK as JW7XK, LA6VM as JW6VM, LA9DL as JW9DL". So # provided in the "by" field. e.g. call="JW", by="By LA7XK as JW7XK, LA6VM as JW6VM, LA9DL as JW9DL". So
@@ -75,14 +80,16 @@ class NG3K(HTTPAlertProvider):
comment = extra_parts[3] if len(extra_parts) > 3 else "" comment = extra_parts[3] if len(extra_parts) > 3 else ""
# Convert to our alert format # Convert to our alert format
alert = Alert(source=self.name, alert = Alert(
source=self.name,
dx_calls=dx_calls, dx_calls=dx_calls,
dx_country=dx_country, dx_country=dx_country,
freqs_modes=bands + (f"; {modes}" if modes != "" else ""), freqs_modes=bands + (f"; {modes}" if modes != "" else ""),
comment=f"{by}; {comment}; {qsl_info}", comment=f"{by}; {comment}; {qsl_info}",
start_time=start_timestamp, start_time=start_timestamp,
end_time=end_timestamp, end_time=end_timestamp,
is_dxpedition=True) is_dxpedition=True,
)
# Add to our list. # Add to our list.
new_alerts.append(alert) new_alerts.append(alert)
providers/alert/parksnpeaks.py
+19 -6
View File
@@ -3,9 +3,9 @@ from datetime import datetime
import pytz import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert from data.alert import Alert
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class ParksNPeaks(HTTPAlertProvider): class ParksNPeaks(HTTPAlertProvider):
@@ -30,8 +30,9 @@ class ParksNPeaks(HTTPAlertProvider):
else: else:
sig_ref = source_alert["WWFFID"] sig_ref = source_alert["WWFFID"]
sig_ref_name = source_alert["Location"] sig_ref_name = source_alert["Location"]
start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace( start_time = (
tzinfo=pytz.UTC).timestamp() datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp()
)
sigrefs = [] sigrefs = []
# PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list, # PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list,
@@ -40,17 +41,29 @@ class ParksNPeaks(HTTPAlertProvider):
sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)] sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)]
# Convert to our alert format # Convert to our alert format
alert = Alert(source=self.name, alert = Alert(
source=self.name,
source_id=source_alert["alID"], source_id=source_alert["alID"],
dx_calls=[source_alert["CallSign"].upper()], dx_calls=[source_alert["CallSign"].upper()],
freqs_modes=f"{source_alert['Freq']} {source_alert['MODE']}", freqs_modes=f"{source_alert['Freq']} {source_alert['MODE']}",
comment=source_alert["Comments"], comment=source_alert["Comments"],
sig_refs=sigrefs, sig_refs=sigrefs,
start_time=start_time, start_time=start_time,
is_dxpedition=False) is_dxpedition=False,
)
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig and sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "SANPCPA", "LLOTA", "QRP"]: if sig and sig not in [
"POTA",
"SOTA",
"WWFF",
"SIOTA",
"ZLOTA",
"KRMNPA",
"SANPCPA",
"LLOTA",
"QRP",
]:
logging.warning(f"PNP alert found with sig {sig}, developer needs to add support for this!") logging.warning(f"PNP alert found with sig {sig}, developer needs to add support for this!")
# If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
providers/alert/pota.py
+22 -9
View File
@@ -2,9 +2,9 @@ from datetime import datetime
import pytz import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert from data.alert import Alert
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class POTA(HTTPAlertProvider): class POTA(HTTPAlertProvider):
@@ -21,18 +21,31 @@ class POTA(HTTPAlertProvider):
# Iterate through source data # Iterate through source data
for source_alert in http_response.json(): for source_alert in http_response.json():
# Convert to our alert format # Convert to our alert format
alert = Alert(source=self.name, alert = Alert(
source=self.name,
source_id=source_alert["scheduledActivitiesId"], source_id=source_alert["scheduledActivitiesId"],
dx_calls=[source_alert["activator"].upper()], dx_calls=[source_alert["activator"].upper()],
freqs_modes=source_alert["frequencies"], freqs_modes=source_alert["frequencies"],
comment=source_alert["comments"], comment=source_alert["comments"],
sig_refs=[SIGRef(id=source_alert["reference"], sig="POTA", name=source_alert["name"], sig_refs=[
url=f"https://pota.app/#/park/{source_alert['reference']}")], SIGRef(
start_time=datetime.strptime(source_alert["startDate"] + source_alert["startTime"], id=source_alert["reference"],
"%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(), sig="POTA",
end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"], name=source_alert["name"],
"%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(), url=f"https://pota.app/#/park/{source_alert['reference']}",
is_dxpedition=False) )
],
start_time=datetime.strptime(
source_alert["startDate"] + source_alert["startTime"],
"%Y-%m-%d%H:%M",
)
.replace(tzinfo=pytz.UTC)
.timestamp(),
end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"], "%Y-%m-%d%H:%M")
.replace(tzinfo=pytz.UTC)
.timestamp(),
is_dxpedition=False,
)
# Add to our list, but exclude any old spots that POTA can sometimes give us where even the end time is # Add to our list, but exclude any old spots that POTA can sometimes give us where even the end time is
# in the past. Don't worry about de-duping, removing old alerts etc. at this point; other code will do # in the past. Don't worry about de-duping, removing old alerts etc. at this point; other code will do
providers/alert/sota.py
+15 -7
View File
@@ -2,9 +2,9 @@ from datetime import datetime
import pytz import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert from data.alert import Alert
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class SOTA(HTTPAlertProvider): class SOTA(HTTPAlertProvider):
@@ -26,18 +26,26 @@ class SOTA(HTTPAlertProvider):
summit_points = None summit_points = None
if len(details) > 2: if len(details) > 2:
summit_points = int(details[-1].split(" ")[0]) summit_points = int(details[-1].split(" ")[0])
alert = Alert(source=self.name, alert = Alert(
source=self.name,
source_id=source_alert["id"], source_id=source_alert["id"],
dx_calls=[source_alert["activatingCallsign"].upper()], dx_calls=[source_alert["activatingCallsign"].upper()],
dx_names=[source_alert["activatorName"].upper()], dx_names=[source_alert["activatorName"].upper()],
freqs_modes=source_alert["frequency"], freqs_modes=source_alert["frequency"],
comment=source_alert["comments"], comment=source_alert["comments"],
sig_refs=[ sig_refs=[
SIGRef(id=f"{source_alert['associationCode']}/{source_alert['summitCode']}", sig="SOTA", SIGRef(
name=summit_name, activation_score=summit_points)], id=f"{source_alert['associationCode']}/{source_alert['summitCode']}",
start_time=datetime.strptime(source_alert["dateActivated"], sig="SOTA",
"%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=pytz.UTC).timestamp(), name=summit_name,
is_dxpedition=False) activation_score=summit_points,
)
],
start_time=datetime.strptime(source_alert["dateActivated"], "%Y-%m-%dT%H:%M:%SZ")
.replace(tzinfo=pytz.UTC)
.timestamp(),
is_dxpedition=False,
)
# Add to our list # Add to our list
new_alerts.append(alert) new_alerts.append(alert)
providers/alert/wota.py
+10 -5
View File
@@ -5,9 +5,9 @@ import pytz
from rss_parser import Parser as RSSParser from rss_parser import Parser as RSSParser
from rss_parser.models.rss import RSS from rss_parser.models.rss import RSS
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert from data.alert import Alert
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class WOTA(HTTPAlertProvider): class WOTA(HTTPAlertProvider):
@@ -25,9 +25,12 @@ class WOTA(HTTPAlertProvider):
rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig"))) rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig")))
# Iterate through source data # Iterate through source data
for source_alert in rss.channel.items: for source_alert in rss.channel.items:
# Reject GUID missing or zero # Reject GUID missing or zero
if not source_alert.guid or not source_alert.guid.content or source_alert.guid.content == "http://www.wota.org.uk/alerts/0": if (
not source_alert.guid
or not source_alert.guid.content
or source_alert.guid.content == "http://www.wota.org.uk/alerts/0"
):
continue continue
# Pick apart the title # Pick apart the title
@@ -51,13 +54,15 @@ class WOTA(HTTPAlertProvider):
time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC) time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
# Convert to our alert format # Convert to our alert format
alert = Alert(source=self.name, alert = Alert(
source=self.name,
source_id=source_alert.guid.content, source_id=source_alert.guid.content,
dx_calls=[dx_call], dx_calls=[dx_call],
freqs_modes=freqs_modes, freqs_modes=freqs_modes,
comment=comment, comment=comment,
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [], sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
start_time=time.timestamp()) start_time=time.timestamp(),
)
# Add to our list. # Add to our list.
new_alerts.append(alert) new_alerts.append(alert)
providers/alert/wwff.py
+11 -7
View File
@@ -2,9 +2,9 @@ from datetime import datetime
import pytz import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert from data.alert import Alert
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class WWFF(HTTPAlertProvider): class WWFF(HTTPAlertProvider):
@@ -21,17 +21,21 @@ class WWFF(HTTPAlertProvider):
# Iterate through source data # Iterate through source data
for source_alert in http_response.json(): for source_alert in http_response.json():
# Convert to our alert format # Convert to our alert format
alert = Alert(source=self.name, alert = Alert(
source=self.name,
source_id=source_alert["id"], source_id=source_alert["id"],
dx_calls=[source_alert["activator_call"].upper()], dx_calls=[source_alert["activator_call"].upper()],
freqs_modes=f"{source_alert['band']} {source_alert['mode']}", freqs_modes=f"{source_alert['band']} {source_alert['mode']}",
comment=source_alert["remarks"], comment=source_alert["remarks"],
sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")], sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")],
start_time=datetime.strptime(source_alert["utc_start"], start_time=datetime.strptime(source_alert["utc_start"], "%Y-%m-%d %H:%M:%S")
"%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(), .replace(tzinfo=pytz.UTC)
end_time=datetime.strptime(source_alert["utc_end"], .timestamp(),
"%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(), end_time=datetime.strptime(source_alert["utc_end"], "%Y-%m-%d %H:%M:%S")
is_dxpedition=False) .replace(tzinfo=pytz.UTC)
.timestamp(),
is_dxpedition=False,
)
# Add to our list # Add to our list
new_alerts.append(alert) new_alerts.append(alert)
providers/callsigndata/callsign_data_provider.py
-1
View File
@@ -51,7 +51,6 @@ class CallsignDataProvider:
else: else:
return None return None
def _perform_new_lookup(self, callsign, lookup_credentials): def _perform_new_lookup(self, callsign, lookup_credentials):
"""Makes a new request to the data source for callsign data.""" """Makes a new request to the data source for callsign data."""
providers/callsigndata/clublogapi.py
+6 -4
View File
@@ -2,12 +2,14 @@ import logging
from datetime import datetime from datetime import datetime
import pytz import pytz
from pyhamtools import LookupLib, Callinfo from pyhamtools import Callinfo, LookupLib
from core.data_store import DATA_STORE from core.data_store import DATA_STORE
from core.utils import get_callsign_object_from_pyhamtools_callinfo from core.utils import get_callsign_object_from_pyhamtools_callinfo
from data.callsign import Callsign from data.callsign import Callsign
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider from providers.callsigndata.api_query_callsign_data_provider import (
APIQueryCallsignDataProvider,
)
class ClublogAPI(APIQueryCallsignDataProvider): class ClublogAPI(APIQueryCallsignDataProvider):
@@ -24,11 +26,11 @@ class ClublogAPI(APIQueryCallsignDataProvider):
else: else:
provider_config["enabled"] = False provider_config["enabled"] = False
logging.warning( logging.warning(
"Clublog XML callsign data provider configured but no api key was provided, this has been disabled.") "Clublog XML callsign data provider configured but no api key was provided, this has been disabled."
)
super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi) super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
def _perform_new_lookup(self, callsign, lookup_credentials): def _perform_new_lookup(self, callsign, lookup_credentials):
callsign_data = Callsign(call=callsign) callsign_data = Callsign(call=callsign)
providers/callsigndata/clublogxml.py
+14 -5
View File
@@ -1,12 +1,14 @@
import gzip import gzip
import logging import logging
from pyhamtools import LookupLib, Callinfo from pyhamtools import Callinfo, LookupLib
from core.data_store import DATA_STORE from core.data_store import DATA_STORE
from core.utils import get_callsign_object_from_pyhamtools_callinfo from core.utils import get_callsign_object_from_pyhamtools_callinfo
from data.callsign import Callsign from data.callsign import Callsign
from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider from providers.callsigndata.file_download_callsign_data_provider import (
FileDownloadCallsignDataProvider,
)
class ClublogXML(FileDownloadCallsignDataProvider): class ClublogXML(FileDownloadCallsignDataProvider):
@@ -24,10 +26,17 @@ class ClublogXML(FileDownloadCallsignDataProvider):
if self._api_key == "": if self._api_key == "":
provider_config["enabled"] = False provider_config["enabled"] = False
logging.warning( logging.warning(
"Clublog XML callsign data provider configured but no api key was provided, this has been disabled.") "Clublog XML callsign data provider configured but no api key was provided, this has been disabled."
)
super().__init__("Clublog XML", provider_config, f"{self.DATA_URL}?api={self._api_key}", super().__init__(
self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS, DATA_STORE.callsign_data_clublogxml) "Clublog XML",
provider_config,
f"{self.DATA_URL}?api={self._api_key}",
self.CACHE_PATH_ZIPPED,
self.POLL_INTERVAL_DAYS,
DATA_STORE.callsign_data_clublogxml,
)
def _handle_file(self, path): def _handle_file(self, path):
try: try:
providers/callsigndata/countryfiles.py
+12 -4
View File
@@ -1,11 +1,13 @@
import logging import logging
from pyhamtools import LookupLib, Callinfo from pyhamtools import Callinfo, LookupLib
from core.data_store import DATA_STORE from core.data_store import DATA_STORE
from core.utils import get_callsign_object_from_pyhamtools_callinfo from core.utils import get_callsign_object_from_pyhamtools_callinfo
from data.callsign import Callsign from data.callsign import Callsign
from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider from providers.callsigndata.file_download_callsign_data_provider import (
FileDownloadCallsignDataProvider,
)
class CountryFiles(FileDownloadCallsignDataProvider): class CountryFiles(FileDownloadCallsignDataProvider):
@@ -17,8 +19,14 @@ class CountryFiles(FileDownloadCallsignDataProvider):
_callinfo = None _callinfo = None
def __init__(self, provider_config): def __init__(self, provider_config):
super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS, super().__init__(
DATA_STORE.callsign_data_countryfiles) "CountryFiles.com",
provider_config,
self.DATA_URL,
self.CACHE_PATH,
self.POLL_INTERVAL_DAYS,
DATA_STORE.callsign_data_countryfiles,
)
def _handle_file(self, path): def _handle_file(self, path):
try: try:
providers/callsigndata/file_download_callsign_data_provider.py
+5 -4
View File
@@ -1,6 +1,6 @@
import logging import logging
from datetime import datetime from datetime import datetime
from threading import Thread, Event from threading import Event, Thread
import pytz import pytz
from requests import ReadTimeout from requests import ReadTimeout
@@ -30,8 +30,7 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
def start(self): def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting. # subsequent polls, so start() returns immediately and the application can continue starting.
logging.info( logging.info(f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}") self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}")
self._thread.start() self._thread.start()
@@ -68,7 +67,9 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
else: else:
self.status = "Error" self.status = "Error"
logging.warning(f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}.") logging.warning(
f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}."
)
except ConnectionError: except ConnectionError:
self.status = "Error" self.status = "Error"
providers/callsigndata/hamqth.py
+31 -20
View File
@@ -1,6 +1,6 @@
import logging import logging
import urllib.parse import urllib.parse
from datetime import timedelta, datetime from datetime import datetime, timedelta
import pytz import pytz
import xmltodict import xmltodict
@@ -10,10 +10,12 @@ from requests_cache import CachedSession
from core.config import SERVER_OWNER_CALLSIGN from core.config import SERVER_OWNER_CALLSIGN
from core.constants import HTTP_HEADERS, SOFTWARE_VERSION from core.constants import HTTP_HEADERS, SOFTWARE_VERSION
from core.data_store import DATA_STORE, CACHE_DIR from core.data_store import CACHE_DIR, DATA_STORE
from core.url_data_cache import URLDataCache from core.url_data_cache import URLDataCache
from data.callsign import Callsign from data.callsign import Callsign
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider from providers.callsigndata.api_query_callsign_data_provider import (
APIQueryCallsignDataProvider,
)
class HamQTH(APIQueryCallsignDataProvider): class HamQTH(APIQueryCallsignDataProvider):
@@ -26,14 +28,15 @@ class HamQTH(APIQueryCallsignDataProvider):
self._URL_DATA_CACHE = URLDataCache("hamqth") self._URL_DATA_CACHE = URLDataCache("hamqth")
# Separate URL cache for session key lookups. Once a session key is returned from logging in with a username # Separate URL cache for session key lookups. Once a session key is returned from logging in with a username
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes. # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/hamqth-creds", self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/hamqth-creds", expire_after=timedelta(minutes=55))
expire_after=timedelta(minutes=55))
def _perform_new_lookup(self, callsign, lookup_credentials): def _perform_new_lookup(self, callsign, lookup_credentials):
# If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because # If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because
# # someone might provide credentials next time around. # # someone might provide credentials next time around.
if not lookup_credentials or not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password) if not lookup_credentials or not (
or lookup_credentials.hamqth_session_id): (lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
or lookup_credentials.hamqth_session_id
):
return None return None
try: try:
@@ -45,7 +48,8 @@ class HamQTH(APIQueryCallsignDataProvider):
try: try:
session_data = self._CREDENTIALS_CACHE.get( session_data = self._CREDENTIALS_CACHE.get(
f"{self._HAMQTH_BASE_URL}?u={urllib.parse.quote_plus(lookup_credentials.hamqth_username)}&p={urllib.parse.quote_plus(lookup_credentials.hamqth_password)}", f"{self._HAMQTH_BASE_URL}?u={urllib.parse.quote_plus(lookup_credentials.hamqth_username)}&p={urllib.parse.quote_plus(lookup_credentials.hamqth_password)}",
headers=HTTP_HEADERS).content headers=HTTP_HEADERS,
).content
dict_data = xmltodict.parse(session_data) dict_data = xmltodict.parse(session_data)
if "session_id" in dict_data["HamQTH"]["session"]: if "session_id" in dict_data["HamQTH"]["session"]:
session_id = str(dict_data["HamQTH"]["session"]["session_id"]) session_id = str(dict_data["HamQTH"]["session"]["session_id"])
@@ -67,13 +71,16 @@ class HamQTH(APIQueryCallsignDataProvider):
if home_call != callsign: if home_call != callsign:
calls_to_try.append(home_call) calls_to_try.append(home_call)
except ValueError: except ValueError:
logging.debug("Could not look up home call for callsign %s", callsign) logging.debug(f"Could not look up home call for callsign {callsign}")
# Try looking up each call using the API # Try looking up each call using the API
for lookup_call in calls_to_try: for lookup_call in calls_to_try:
try: try:
response = self._URL_DATA_CACHE.get( response = self._URL_DATA_CACHE.get(
f"{self._HAMQTH_BASE_URL}?id={session_id}&callsign={urllib.parse.quote_plus(lookup_call)}&prg={self._PRG}", headers=HTTP_HEADERS, timeout=10) f"{self._HAMQTH_BASE_URL}?id={session_id}&callsign={urllib.parse.quote_plus(lookup_call)}&prg={self._PRG}",
headers=HTTP_HEADERS,
timeout=10,
)
if response.ok: if response.ok:
# Found data, convert it to our object and return it # Found data, convert it to our object and return it
data = xmltodict.parse(response.content)["HamQTH"]["search"] data = xmltodict.parse(response.content)["HamQTH"]["search"]
@@ -83,19 +90,18 @@ class HamQTH(APIQueryCallsignDataProvider):
return self.hamqth_response_to_callsign(callsign, data) return self.hamqth_response_to_callsign(callsign, data)
elif not response.from_cache: elif not response.from_cache:
logging.warning("HTTP %d looking up callsign %s using HamQTH", response.status_code, logging.warning(f"HTTP {response.status_code} looking up callsign {lookup_call} using HamQTH")
lookup_call)
except (KeyError, ValueError): except (KeyError, ValueError):
continue continue
except ConnectionError: except ConnectionError:
logging.warning(f"Connection error when looking up callsign %s using HamQTH", lookup_call) logging.warning(f"Connection error when looking up callsign {lookup_call} using HamQTH")
continue continue
except (ConnectTimeout, ReadTimeout): except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when looking up callsign %s using HamQTH", lookup_call) logging.warning(f"Timeout when looking up callsign {lookup_call} using HamQTH")
continue continue
except Exception: except Exception:
logging.exception("Exception when looking up callsign %s using HamQTH", lookup_call) logging.exception(f"Exception when looking up callsign {lookup_call} using HamQTH")
continue continue
# Not found in HamQTH; return a Callsign object with no data so we cache that and don't keep retrying # Not found in HamQTH; return a Callsign object with no data so we cache that and don't keep retrying
@@ -114,9 +120,12 @@ class HamQTH(APIQueryCallsignDataProvider):
# Check for sensible latitudes # Check for sensible latitudes
lat = None lat = None
lon = None lon = None
if "latitude" in data and "longitude" in data and ( if (
float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float( "latitude" in data
data["latitude"]) < 89.9: and "longitude" in data
and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
and -89.9 < float(data["latitude"]) < 89.9
):
lat = float(data["latitude"]) lat = float(data["latitude"])
lon = float(data["longitude"]) lon = float(data["longitude"])
@@ -125,7 +134,8 @@ class HamQTH(APIQueryCallsignDataProvider):
if "grid" in data and not data["grid"].startswith("AA00"): if "grid" in data and not data["grid"].startswith("AA00"):
grid = data["grid"] grid = data["grid"]
return Callsign(call=callsign, return Callsign(
call=callsign,
home_call=callinfo.Callinfo.get_homecall(callsign), home_call=callinfo.Callinfo.get_homecall(callsign),
name=data["nick"] if "nick" in data else None, name=data["nick"] if "nick" in data else None,
qth=data["qth"] if "qth" in data else None, qth=data["qth"] if "qth" in data else None,
@@ -137,4 +147,5 @@ class HamQTH(APIQueryCallsignDataProvider):
dxcc_id=int(data["adif"]) if "adif" in data else None, dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cq"]) if "cq" in data else None, cq_zone=int(data["cq"]) if "cq" in data else None,
itu_zone=int(data["itu"]) if "itu" in data else None, itu_zone=int(data["itu"]) if "itu" in data else None,
location_source="HOME QTH") location_source="HOME QTH",
)
providers/callsigndata/qrz.py
+34 -23
View File
@@ -1,6 +1,6 @@
import logging import logging
import urllib.parse import urllib.parse
from datetime import timedelta, datetime from datetime import datetime, timedelta
import pytz import pytz
import xmltodict import xmltodict
@@ -9,10 +9,12 @@ from requests import ConnectTimeout, ReadTimeout
from requests_cache import CachedSession from requests_cache import CachedSession
from core.constants import HTTP_HEADERS from core.constants import HTTP_HEADERS
from core.data_store import DATA_STORE, CACHE_DIR from core.data_store import CACHE_DIR, DATA_STORE
from core.url_data_cache import URLDataCache from core.url_data_cache import URLDataCache
from data.callsign import Callsign from data.callsign import Callsign
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider from providers.callsigndata.api_query_callsign_data_provider import (
APIQueryCallsignDataProvider,
)
class QRZ(APIQueryCallsignDataProvider): class QRZ(APIQueryCallsignDataProvider):
@@ -24,14 +26,14 @@ class QRZ(APIQueryCallsignDataProvider):
self._URL_DATA_CACHE = URLDataCache("qrz") self._URL_DATA_CACHE = URLDataCache("qrz")
# Separate URL cache for session key lookups. Once a session key is returned from logging in with a username # Separate URL cache for session key lookups. Once a session key is returned from logging in with a username
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes. # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/qrz-creds", self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/qrz-creds", expire_after=timedelta(minutes=55))
expire_after=timedelta(minutes=55))
def _perform_new_lookup(self, callsign, lookup_credentials): def _perform_new_lookup(self, callsign, lookup_credentials):
# If we don't have QRZ credentials, skip this lookup. Return None so we don't *cache* the lack of data, because # If we don't have QRZ credentials, skip this lookup. Return None so we don't *cache* the lack of data, because
# someone might provide credentials next time around. # someone might provide credentials next time around.
if not lookup_credentials or not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password) if not lookup_credentials or not (
or lookup_credentials.qrz_session_key): (lookup_credentials.qrz_username and lookup_credentials.qrz_password) or lookup_credentials.qrz_session_key
):
return None return None
try: try:
@@ -43,7 +45,8 @@ class QRZ(APIQueryCallsignDataProvider):
try: try:
login_response = self._CREDENTIALS_CACHE.get( login_response = self._CREDENTIALS_CACHE.get(
f"{self._QRZ_BASE_URL}?username={urllib.parse.quote_plus(lookup_credentials.qrz_username)}&password={urllib.parse.quote_plus(lookup_credentials.qrz_password)}&agent=spothole", f"{self._QRZ_BASE_URL}?username={urllib.parse.quote_plus(lookup_credentials.qrz_username)}&password={urllib.parse.quote_plus(lookup_credentials.qrz_password)}&agent=spothole",
headers=HTTP_HEADERS).content headers=HTTP_HEADERS,
).content
login_data = xmltodict.parse(login_response) login_data = xmltodict.parse(login_response)
session = login_data.get("QRZDatabase", {}).get("Session", {}) session = login_data.get("QRZDatabase", {}).get("Session", {})
if "Key" in session: if "Key" in session:
@@ -66,14 +69,16 @@ class QRZ(APIQueryCallsignDataProvider):
if home_call != callsign: if home_call != callsign:
calls_to_try.append(home_call) calls_to_try.append(home_call)
except ValueError: except ValueError:
logging.debug("Could not look up home call for callsign %s", callsign) logging.debug(f"Could not look up home call for callsign {callsign}")
# Try looking up each call using the API # Try looking up each call using the API
for lookup_call in calls_to_try: for lookup_call in calls_to_try:
try: try:
response = self._URL_DATA_CACHE.get( response = self._URL_DATA_CACHE.get(
f"{self._QRZ_BASE_URL}?s={session_key}&callsign={urllib.parse.quote_plus(lookup_call)}", f"{self._QRZ_BASE_URL}?s={session_key}&callsign={urllib.parse.quote_plus(lookup_call)}",
headers=HTTP_HEADERS, timeout=10) headers=HTTP_HEADERS,
timeout=10,
)
if response.ok: if response.ok:
qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {}) qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
if qrz_response: if qrz_response:
@@ -88,24 +93,25 @@ class QRZ(APIQueryCallsignDataProvider):
elif "Session" in qrz_response and "Error" in qrz_response.get("Session"): elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
# Errors here are normally just "callsign not in database", no need to log that ourselves # Errors here are normally just "callsign not in database", no need to log that ourselves
# above debug level. # above debug level.
logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call, logging.debug(
qrz_response.get("Session").get("Error")) f"QRZ returned an error looking up callsign {lookup_call}: {qrz_response.get('Session').get('Error')}"
)
elif not response.from_cache: elif not response.from_cache:
logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call) logging.warning(f"QRZ returned a malformed response looking up callsign {lookup_call}")
elif not response.from_cache: elif not response.from_cache:
logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call) logging.warning(f"HTTP {response.status_code} looking up callsign {lookup_call} using QRZ")
except (KeyError, ValueError): except (KeyError, ValueError):
continue continue
except ConnectionError: except ConnectionError:
logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call) logging.warning(f"Connection error when looking up callsign {lookup_call} using QRZ")
continue continue
except (ConnectTimeout, ReadTimeout): except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call) logging.warning(f"Timeout when looking up callsign {lookup_call} using QRZ.")
continue continue
except Exception: except Exception:
logging.exception("Exception when looking up callsign %s using QRZ", lookup_call) logging.exception(f"Exception when looking up callsign {lookup_call} using QRZ")
continue continue
# Not found in QRZ; return a Callsign object with no data so we cache that and don't keep retrying # Not found in QRZ; return a Callsign object with no data so we cache that and don't keep retrying
@@ -128,16 +134,19 @@ class QRZ(APIQueryCallsignDataProvider):
if "fname" in data: if "fname" in data:
name = data["fname"] name = data["fname"]
if "nick" in data: if "nick" in data:
name = f"{name} \"{data['nick']}\"" name = f'{name} "{data["nick"]}"'
if "name" in data: if "name" in data:
name = f"{name} {data['name']}" name = f"{name} {data['name']}"
# Check for sensible latitudes # Check for sensible latitudes
lat = None lat = None
lon = None lon = None
if "latitude" in data and "longitude" in data and ( if (
float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float( "latitude" in data
data["latitude"]) < 89.9: and "longitude" in data
and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
and -89.9 < float(data["latitude"]) < 89.9
):
lat = float(data["latitude"]) lat = float(data["latitude"])
lon = float(data["longitude"]) lon = float(data["longitude"])
@@ -146,7 +155,8 @@ class QRZ(APIQueryCallsignDataProvider):
if "grid" in data and not data["grid"].startswith("AA00"): if "grid" in data and not data["grid"].startswith("AA00"):
grid = data["grid"] grid = data["grid"]
return Callsign(call=callsign, return Callsign(
call=callsign,
home_call=callinfo.Callinfo.get_homecall(callsign), home_call=callinfo.Callinfo.get_homecall(callsign),
name=name, name=name,
qth=data["addr2"] if "addr2" in data else None, qth=data["addr2"] if "addr2" in data else None,
@@ -158,4 +168,5 @@ class QRZ(APIQueryCallsignDataProvider):
dxcc_id=int(data["adif"]) if "adif" in data else None, dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cqzone"]) if "cqzone" in data else None, cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
itu_zone=int(data["ituzone"]) if "ituzone" in data else None, itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
location_source="HOME QTH") location_source="HOME QTH",
)
providers/sigrefdata/arlhs.py
+13 -7
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep from time import sleep
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ARLHS(FileDownloadSIGRefDataProvider): class ARLHS(FileDownloadSIGRefDataProvider):
@@ -20,14 +22,18 @@ class ARLHS(FileDownloadSIGRefDataProvider):
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
if "ARLHS" in row and row["ARLHS"] != "": if "ARLHS" in row and row["ARLHS"] != "":
ref_id = row["ARLHS"] ref_id = row["ARLHS"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Lighthouse", ref_type="Lighthouse",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}", url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[ latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
"Latitude"] != "" else None, longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[ grid=row["Maidenhead Locator"],
"Longitude"] != "" else None, )
grid=row["Maidenhead Locator"])) )
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/dme.py
+18 -7
View File
@@ -4,7 +4,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider from providers.sigrefdata.local_file_sig_ref_data_provider import (
LocalFileSIGRefDataProvider,
)
class DME(LocalFileSIGRefDataProvider): class DME(LocalFileSIGRefDataProvider):
@@ -21,16 +23,25 @@ class DME(LocalFileSIGRefDataProvider):
with open(path, encoding="latin-1") as _f: with open(path, encoding="latin-1") as _f:
for row in csv.DictReader(_f, delimiter=";"): for row in csv.DictReader(_f, delimiter=";"):
ref_id = row["COD_INE"][:5] ref_id = row["COD_INE"][:5]
latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get( latitude = (
"LATITUD_ETRS89_REGCAN95") else None float(row["LATITUD_ETRS89_REGCAN95"].replace(",", "."))
longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get( if row.get("LATITUD_ETRS89_REGCAN95")
"LONGITUD_ETRS89_REGCAN95") else None else None
)
longitude = (
float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", "."))
if row.get("LONGITUD_ETRS89_REGCAN95")
else None
)
ref = SIGRef(sig=self.SIG, id=ref_id, ref = SIGRef(
sig=self.SIG,
id=ref_id,
ref_type="Town", ref_type="Town",
name=f"{row['NOMBRE_ACTUAL']}, {row['PROVINCIA']}", name=f"{row['NOMBRE_ACTUAL']}, {row['PROVINCIA']}",
latitude=latitude, latitude=latitude,
longitude=longitude) longitude=longitude,
)
if latitude and longitude: if latitude and longitude:
ref.grid = latlong_to_locator(latitude, longitude, 6) ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref) new_data.append(ref)
providers/sigrefdata/file_download_sig_ref_data_provider.py
+2 -3
View File
@@ -1,6 +1,6 @@
import logging import logging
from datetime import datetime from datetime import datetime
from threading import Thread, Event from threading import Event, Thread
import pytz import pytz
from requests import ReadTimeout from requests import ReadTimeout
@@ -26,8 +26,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
def start(self): def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting. # subsequent polls, so start() returns immediately and the application can continue starting.
logging.info( logging.info(f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}") self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}")
self._thread.start() self._thread.start()
providers/sigrefdata/gma.py
+16 -9
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep from time import sleep
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class GMA(FileDownloadSIGRefDataProvider): class GMA(FileDownloadSIGRefDataProvider):
@@ -19,16 +21,21 @@ class GMA(FileDownloadSIGRefDataProvider):
new_data = [] new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"] ref_id = row["Reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Summit", ref_type="Summit",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}", url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[ latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
"Latitude"] != "" else None, longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[ altitude=float(row["Height (m)"].replace("m", ""))
"Longitude"] != "" else None, if "Height (m)" in row and row["Height (m)"] != ""
altitude=float(row["Height (m)"].replace("m", "")) if "Height (m)" in row and row[ else None,
"Height (m)"] != "" else None, grid=row["Maidenhead Locator"],
grid=row["Maidenhead Locator"])) )
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/illw.py
+13 -7
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep from time import sleep
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ILLW(FileDownloadSIGRefDataProvider): class ILLW(FileDownloadSIGRefDataProvider):
@@ -20,14 +22,18 @@ class ILLW(FileDownloadSIGRefDataProvider):
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
if "ILLW" in row and row["ILLW"] != "": if "ILLW" in row and row["ILLW"] != "":
ref_id = row["ILLW"] ref_id = row["ILLW"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Lighthouse", ref_type="Lighthouse",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}", url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[ latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
"Latitude"] != "" else None, longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[ grid=row["Maidenhead Locator"],
"Longitude"] != "" else None, )
grid=row["Maidenhead Locator"])) )
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/iota.py
+19 -4
View File
@@ -4,7 +4,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class IOTA(FileDownloadSIGRefDataProvider): class IOTA(FileDownloadSIGRefDataProvider):
@@ -29,10 +31,23 @@ class IOTA(FileDownloadSIGRefDataProvider):
try: try:
grid = latlong_to_locator(latitude, longitude, 6) grid = latlong_to_locator(latitude, longitude, 6)
except ValueError: except ValueError:
logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude) logging.debug(
"Error converting lat/lon to locator for an IOTA reference %f %f",
latitude,
longitude,
)
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"], new_data.append(
ref_type="Island", grid=grid, latitude=latitude, longitude=longitude)) SIGRef(
sig=self.SIG,
id=ref_id,
name=ref["name"],
ref_type="Island",
grid=grid,
latitude=latitude,
longitude=longitude,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
# of the data in this case # of the data in this case
providers/sigrefdata/krmnpa.py
+3 -1
View File
@@ -1,4 +1,6 @@
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import ParksNPeaksKMLSIGRefDataProvider from providers.sigrefdata.pnp_kml_sig_ref_data_provider import (
ParksNPeaksKMLSIGRefDataProvider,
)
class KRMNPA(ParksNPeaksKMLSIGRefDataProvider): class KRMNPA(ParksNPeaksKMLSIGRefDataProvider):
providers/sigrefdata/llota.py
+11 -3
View File
@@ -3,7 +3,9 @@ from time import sleep
from pyhamtools.locator import locator_to_latlong from pyhamtools.locator import locator_to_latlong
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class LLOTA(FileDownloadSIGRefDataProvider): class LLOTA(FileDownloadSIGRefDataProvider):
@@ -25,12 +27,18 @@ class LLOTA(FileDownloadSIGRefDataProvider):
grid = str(ref["grid_locator"]) grid = str(ref["grid_locator"])
ll = locator_to_latlong(grid) ll = locator_to_latlong(grid)
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]), new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=str(ref["name"]),
ref_type="Lake", ref_type="Lake",
url=f"https://llota.app/list/ref/{ref_id}", url=f"https://llota.app/list/ref/{ref_id}",
grid=grid, grid=grid,
latitude=ll[0], latitude=ll[0],
longitude=ll[1])) longitude=ll[1],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
# of the data in this case # of the data in this case
providers/sigrefdata/mota.py
+13 -7
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep from time import sleep
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class MOTA(FileDownloadSIGRefDataProvider): class MOTA(FileDownloadSIGRefDataProvider):
@@ -19,14 +21,18 @@ class MOTA(FileDownloadSIGRefDataProvider):
new_data = [] new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"] ref_id = row["Reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Mill", ref_type="Mill",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}", url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[ latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
"Latitude"] != "" else None, longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[ grid=row["Maidenhead Locator"],
"Longitude"] != "" else None, )
grid=row["Maidenhead Locator"])) )
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/pnp_kml_sig_ref_data_provider.py
+9 -3
View File
@@ -5,7 +5,9 @@ from fastkml import kml
from pyhamtools.locator import latlong_to_locator from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider): class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
@@ -37,11 +39,15 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
longitude, latitude = placemark.geometry.x, placemark.geometry.y longitude, latitude = placemark.geometry.x, placemark.geometry.y
ref = SIGRef(sig=self.sig_name, id=ref_id, name=placemark.name, ref = SIGRef(
sig=self.sig_name,
id=ref_id,
name=placemark.name,
ref_type="Park", ref_type="Park",
url=f"https://parksnpeaks.org/getPark.php?actPark={ref_id}", url=f"https://parksnpeaks.org/getPark.php?actPark={ref_id}",
latitude=latitude, latitude=latitude,
longitude=longitude) longitude=longitude,
)
if latitude and longitude: if latitude and longitude:
ref.grid = latlong_to_locator(latitude, longitude, 6) ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref) new_data.append(ref)
providers/sigrefdata/pota.py
+12 -6
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep from time import sleep
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class POTA(FileDownloadSIGRefDataProvider): class POTA(FileDownloadSIGRefDataProvider):
@@ -19,14 +21,18 @@ class POTA(FileDownloadSIGRefDataProvider):
new_data = [] new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"] ref_id = row["reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["name"] if "name" in row else None,
ref_type="Park", ref_type="Park",
url=f"https://pota.app/#/park/{ref_id}", url=f"https://pota.app/#/park/{ref_id}",
grid=row["grid"] if "grid" in row else None, grid=row["grid"] if "grid" in row else None,
latitude=float(row["latitude"]) if "latitude" in row and row[ latitude=float(row["latitude"]) if "latitude" in row and row["latitude"] != "" else None,
"latitude"] != "" else None, longitude=float(row["longitude"]) if "longitude" in row and row["longitude"] != "" else None,
longitude=float(row["longitude"]) if "longitude" in row and row[ )
"longitude"] != "" else None)) )
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/sanpcpa.py
+3 -1
View File
@@ -1,4 +1,6 @@
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import ParksNPeaksKMLSIGRefDataProvider from providers.sigrefdata.pnp_kml_sig_ref_data_provider import (
ParksNPeaksKMLSIGRefDataProvider,
)
class SANPCPA(ParksNPeaksKMLSIGRefDataProvider): class SANPCPA(ParksNPeaksKMLSIGRefDataProvider):
providers/sigrefdata/sig_ref_data_provider.py
+1 -3
View File
@@ -19,13 +19,11 @@ class SIGRefDataProvider:
self.reference_count = 0 self.reference_count = 0
self._stop = False self._stop = False
def start(self): def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources""" """Start the provider. This should return immediately after spawning threads to access the remote resources"""
raise NotImplementedError("Subclasses must implement this method") raise NotImplementedError("Subclasses must implement this method")
def stop(self): def stop(self):
"""Stop any threads and prepare for application shutdown. Subclasses should implement this method and call """Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
super().""" super()."""
@@ -47,4 +45,4 @@ class SIGRefDataProvider:
break break
self.reference_count = len(new_data) self.reference_count = len(new_data)
logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name) logging.info(f"Loaded {self.reference_count} references for {self.sig_name} into the data store.")
providers/sigrefdata/siota.py
+11 -3
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep from time import sleep
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class SIOTA(FileDownloadSIGRefDataProvider): class SIOTA(FileDownloadSIGRefDataProvider):
@@ -19,11 +21,17 @@ class SIOTA(FileDownloadSIGRefDataProvider):
new_data = [] new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["SILO_CODE"] ref_id = row["SILO_CODE"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["NAME"] if "NAME" in row else None,
ref_type="Silo", ref_type="Silo",
grid=row["LOCATOR"] if "LOCATOR" in row else None, grid=row["LOCATOR"] if "LOCATOR" in row else None,
latitude=float(row["LAT"]) if "LAT" in row else None, latitude=float(row["LAT"]) if "LAT" in row else None,
longitude=float(row["LNG"]) if "LNG" in row else None)) longitude=float(row["LNG"]) if "LNG" in row else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/sota.py
+9 -3
View File
@@ -4,7 +4,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class SOTA(FileDownloadSIGRefDataProvider): class SOTA(FileDownloadSIGRefDataProvider):
@@ -24,13 +26,17 @@ class SOTA(FileDownloadSIGRefDataProvider):
latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
altitude = float(row["AltM"]) if "AltM" in row and row["AltM"] != "" else None altitude = float(row["AltM"]) if "AltM" in row and row["AltM"] != "" else None
ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None, ref = SIGRef(
sig=self.SIG,
id=ref_id,
name=row["SummitName"] if "SummitName" in row else None,
ref_type="Summit", ref_type="Summit",
url=f"https://www.sotadata.org.uk/en/summit/{ref_id}", url=f"https://www.sotadata.org.uk/en/summit/{ref_id}",
latitude=latitude, latitude=latitude,
longitude=longitude, longitude=longitude,
altitude=altitude, altitude=altitude,
activation_score=int(row["Points"]) if "Points" in row else None) activation_score=int(row["Points"]) if "Points" in row else None,
)
if latitude and longitude: if latitude and longitude:
ref.grid = latlong_to_locator(latitude, longitude, 6) ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref) new_data.append(ref)
providers/sigrefdata/toilets.py
+13 -3
View File
@@ -1,7 +1,9 @@
import csv import csv
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider from providers.sigrefdata.local_file_sig_ref_data_provider import (
LocalFileSIGRefDataProvider,
)
class Toilets(LocalFileSIGRefDataProvider): class Toilets(LocalFileSIGRefDataProvider):
@@ -19,8 +21,16 @@ class Toilets(LocalFileSIGRefDataProvider):
csv_data = _f.read() csv_data = _f.read()
dr = csv.DictReader(csv_data.splitlines()) dr = csv.DictReader(csv_data.splitlines())
for row in dr: for row in dr:
new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], ref_type="Toilet", new_data.append(
latitude=float(row["lat"]), longitude=float(row["lon"]))) SIGRef(
sig=self.SIG,
id=row["ref"],
name=row["ref"],
ref_type="Toilet",
latitude=float(row["lat"]),
longitude=float(row["lon"]),
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
# of the data in this case # of the data in this case
providers/sigrefdata/towers.py
+11 -3
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep from time import sleep
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class Towers(FileDownloadSIGRefDataProvider): class Towers(FileDownloadSIGRefDataProvider):
@@ -19,12 +21,18 @@ class Towers(FileDownloadSIGRefDataProvider):
new_data = [] new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"): for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
ref_id = row["Ref"] ref_id = row["Ref"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Nazev"] if "Nazev" in row else None,
ref_type="Tower", ref_type="Tower",
url=f"https://wwtota.com/seznam/karta_rozhledny.php?ref={ref_id}", url=f"https://wwtota.com/seznam/karta_rozhledny.php?ref={ref_id}",
grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None, grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None, latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None)) longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/wca.py
+12 -4
View File
@@ -5,7 +5,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WCA(FileDownloadSIGRefDataProvider): class WCA(FileDownloadSIGRefDataProvider):
@@ -34,14 +36,20 @@ class WCA(FileDownloadSIGRefDataProvider):
longitude = float(split[1]) longitude = float(split[1])
grid = latlong_to_locator(latitude, longitude) grid = latlong_to_locator(latitude, longitude)
except ValueError: except ValueError:
logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id) logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for {ref_id}")
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
ref_type="Castle", ref_type="Castle",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}", url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=latitude, latitude=latitude,
longitude=longitude, longitude=longitude,
grid=grid)) grid=grid,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/wota.py
+12 -3
View File
@@ -1,7 +1,9 @@
from time import sleep from time import sleep
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WOTA(FileDownloadSIGRefDataProvider): class WOTA(FileDownloadSIGRefDataProvider):
@@ -25,12 +27,19 @@ class WOTA(FileDownloadSIGRefDataProvider):
number = int(ref_id.upper().replace("LDO-", "")) number = int(ref_id.upper().replace("LDO-", ""))
url = f"https://www.wota.org.uk/MM_LDO-{number + 214!s}" url = f"https://www.wota.org.uk/MM_LDO-{number + 214!s}"
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=feature["properties"]["title"],
url=url,
ref_type="Summit", ref_type="Summit",
grid=feature["properties"]["qthLocator"], grid=feature["properties"]["qthLocator"],
latitude=feature["geometry"]["coordinates"][1], latitude=feature["geometry"]["coordinates"][1],
longitude=feature["geometry"]["coordinates"][0], longitude=feature["geometry"]["coordinates"][0],
altitude=feature["properties"]["height"])) altitude=feature["properties"]["height"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/wwbota.py
+11 -3
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep from time import sleep
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WWBOTA(FileDownloadSIGRefDataProvider): class WWBOTA(FileDownloadSIGRefDataProvider):
@@ -19,12 +21,18 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
new_data = [] new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["Reference"] ref_id = row["Reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Bunker", ref_type="Bunker",
url=f"https://bunkerwiki.org/?s={ref_id}" if ref_id.startswith("B/G") else None, url=f"https://bunkerwiki.org/?s={ref_id}" if ref_id.startswith("B/G") else None,
grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None, grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None, latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None)) longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/wwff.py
+17 -8
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep from time import sleep
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WWFF(FileDownloadSIGRefDataProvider): class WWFF(FileDownloadSIGRefDataProvider):
@@ -19,15 +21,22 @@ class WWFF(FileDownloadSIGRefDataProvider):
new_data = [] new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"] ref_id = row["reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None, new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["name"] if "name" in row else None,
ref_type="Park", ref_type="Park",
url=f"https://wwff.co/directory/?showRef={ref_id}", url=f"https://wwff.co/directory/?showRef={ref_id}",
grid=row["iaruLocator"] if "iaruLocator" in row and row[ grid=row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None,
"iaruLocator"] != "-" else None, latitude=float(row["latitude"])
latitude=float(row["latitude"]) if "latitude" in row and row[ if "latitude" in row and row["latitude"] != "" and row["latitude"] != "-"
"latitude"] != "" and row["latitude"] != "-" else None, else None,
longitude=float(row["longitude"]) if "longitude" in row and row[ longitude=float(row["longitude"])
"longitude"] != "" and row["longitude"] != "-" else None)) if "longitude" in row and row["longitude"] != "" and row["longitude"] != "-"
else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case # the data in this case
providers/sigrefdata/zlota.py
+9 -3
View File
@@ -3,7 +3,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ZLOTA(FileDownloadSIGRefDataProvider): class ZLOTA(FileDownloadSIGRefDataProvider):
@@ -25,11 +27,15 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
latitude = ref["latitude"] latitude = ref["latitude"]
longitude = ref["longitude"] longitude = ref["longitude"]
new_ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"], new_ref = SIGRef(
sig=self.SIG,
id=ref_id,
name=ref["name"],
ref_type=ref["asset_type"].title(), ref_type=ref["asset_type"].title(),
url=f"https://ontheair.nz/assets/{ref_id.replace('/', '_')}", url=f"https://ontheair.nz/assets/{ref_id.replace('/', '_')}",
latitude=latitude, latitude=latitude,
longitude=longitude) longitude=longitude,
)
# Check lat/lon validity and update grid accordingly # Check lat/lon validity and update grid accordingly
if latitude and longitude: if latitude and longitude:
providers/solarconditions/giroionosonde.py
+27 -13
View File
@@ -1,11 +1,11 @@
import csv import csv
import logging import logging
from datetime import datetime, timezone, timedelta from datetime import datetime, timedelta, timezone
from threading import Thread, Event from threading import Event, Thread
import pytz import pytz
import requests import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS from core.constants import HTTP_HEADERS
from providers.solarconditions.ionosonde_utils import compute_band_states from providers.solarconditions.ionosonde_utils import compute_band_states
@@ -40,9 +40,16 @@ class GIROIonosonde(SolarConditionsProvider):
# entries so KC2G cache data is preserved. # entries so KC2G cache data is preserved.
existing = self._solar_conditions.ionosonde_data or {} existing = self._solar_conditions.ionosonde_data or {}
new_entries = { new_entries = {
s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None, s["ursi"]: {
"luf": None, "band_states": None} "ursi": s["ursi"],
for s in self._stations if s["ursi"] not in existing "name": s["name"],
"fof2": None,
"muf": None,
"luf": None,
"band_states": None,
}
for s in self._stations
if s["ursi"] not in existing
} }
if new_entries: if new_entries:
self.update_data({"ionosonde_data": {**existing, **new_entries}}) self.update_data({"ionosonde_data": {**existing, **new_entries}})
@@ -50,7 +57,7 @@ class GIROIonosonde(SolarConditionsProvider):
@staticmethod @staticmethod
def _load_stations(): def _load_stations():
stations = [] stations = []
with open(STATIONS_INDEX, newline='') as f: with open(STATIONS_INDEX, newline="") as f:
for row in csv.reader(f): for row in csv.reader(f):
if len(row) >= 2: if len(row) >= 2:
stations.append({"ursi": row[0].strip(), "name": row[1].strip()}) stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
@@ -94,8 +101,14 @@ class GIROIonosonde(SolarConditionsProvider):
# Merge GIRO's readings into any existing data for this station. # Merge GIRO's readings into any existing data for this station.
existing = ionosonde_data.get(ursi, {}) existing = ionosonde_data.get(ursi, {})
merged_fof2 = {**{float(t): v for t, v in (existing.get("fof2") or {}).items()}, **fof2} merged_fof2 = {
merged_muf = {**{float(t): v for t, v in (existing.get("muf") or {}).items()}, **muf} **{float(t): v for t, v in (existing.get("fof2") or {}).items()},
**fof2,
}
merged_muf = {
**{float(t): v for t, v in (existing.get("muf") or {}).items()},
**muf,
}
merged_luf = dict(luf) if luf else {} merged_luf = dict(luf) if luf else {}
merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts} merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts}
@@ -104,7 +117,8 @@ class GIROIonosonde(SolarConditionsProvider):
band_states = compute_band_states(merged_fof2, merged_muf, merged_luf) band_states = compute_band_states(merged_fof2, merged_muf, merged_luf)
ionosonde_data[ursi] = { ionosonde_data[ursi] = {
"ursi": ursi, "name": name, "ursi": ursi,
"name": name,
"fof2": merged_fof2 or None, "fof2": merged_fof2 or None,
"muf": merged_muf or None, "muf": merged_muf or None,
"luf": merged_luf or None, "luf": merged_luf or None,
@@ -132,7 +146,7 @@ class GIROIonosonde(SolarConditionsProvider):
return None, None, None return None, None, None
return self._parse_all(http_response.text) return self._parse_all(http_response.text)
except (ConnectTimeout, ReadTimeout): except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing Giro ionosonde API.") logging.warning("Timeout when accessing Giro ionosonde API.")
return None, None, None return None, None, None
except ConnectionError: except ConnectionError:
logging.warning("Connection error when accessing Giro ionosonde API.") logging.warning("Connection error when accessing Giro ionosonde API.")
@@ -147,14 +161,14 @@ class GIROIonosonde(SolarConditionsProvider):
luf_data = {} luf_data = {}
for line in text.splitlines(): for line in text.splitlines():
line = line.strip() line = line.strip()
if not line or line.startswith('#'): if not line or line.startswith("#"):
continue continue
# Data rows have the following format: timestamp CS foF2 QD MUFD QD fmin QD # Data rows have the following format: timestamp CS foF2 QD MUFD QD fmin QD
parts = line.split() parts = line.split()
if len(parts) >= 5: if len(parts) >= 5:
try: try:
# Python 3.8 TZ parsing fudge # Python 3.8 TZ parsing fudge
ts = datetime.fromisoformat(parts[0].replace('Z', '+00:00')).timestamp() ts = datetime.fromisoformat(parts[0].replace("Z", "+00:00")).timestamp()
except ValueError: except ValueError:
continue continue
try: try:
providers/solarconditions/hamqsl.py
+11 -5
View File
@@ -2,9 +2,12 @@ import logging
from xml.etree import ElementTree from xml.etree import ElementTree
import pytz import pytz
from dateutil import parser as dateutil_parser, tz as dateutil_tz from dateutil import parser as dateutil_parser
from dateutil import tz as dateutil_tz
from providers.solarconditions.http_solar_conditions_provider import HTTPSolarConditionsProvider from providers.solarconditions.http_solar_conditions_provider import (
HTTPSolarConditionsProvider,
)
POLL_INTERVAL = 3600 # 1 hour POLL_INTERVAL = 3600 # 1 hour
URL = "https://www.hamqsl.com/solarxml.php" URL = "https://www.hamqsl.com/solarxml.php"
@@ -92,7 +95,8 @@ class HamQSL(HTTPSolarConditionsProvider):
"aurora_latitude": float_val("latdegree"), "aurora_latitude": float_val("latdegree"),
"solar_wind": float_val("solarwind"), "solar_wind": float_val("solarwind"),
"magnetic_field": float_val("magneticfield"), "magnetic_field": float_val("magneticfield"),
"geomag_field": text("geomagfield").title() "geomag_field": text("geomagfield")
.title()
.replace("Vr Quiet", "Very Quiet") .replace("Vr Quiet", "Very Quiet")
.replace("Unsettld", "Unsettled") .replace("Unsettld", "Unsettled")
.replace("Min Strm", "Minor Storm") .replace("Min Strm", "Minor Storm")
@@ -102,8 +106,10 @@ class HamQSL(HTTPSolarConditionsProvider):
"geomag_noise": text("signalnoise"), "geomag_noise": text("signalnoise"),
"hf_conditions": hf_conditions, "hf_conditions": hf_conditions,
"vhf_conditions": { "vhf_conditions": {
"vhf_aurora_northern_hemi": (vhf_map.get(("vhf-aurora", "northern_hemi")) or "").title().replace( "vhf_aurora_northern_hemi": (vhf_map.get(("vhf-aurora", "northern_hemi")) or "")
"Lat Aur", "Latitude") or None, .title()
.replace("Lat Aur", "Latitude")
or None,
"es_2m_europe": vhf_map.get(("E-Skip", "europe")), "es_2m_europe": vhf_map.get(("E-Skip", "europe")),
"es_4m_europe": vhf_map.get(("E-Skip", "europe_4m")), "es_4m_europe": vhf_map.get(("E-Skip", "europe_4m")),
"es_6m_europe": vhf_map.get(("E-Skip", "europe_6m")), "es_6m_europe": vhf_map.get(("E-Skip", "europe_6m")),
providers/solarconditions/http_solar_conditions_provider.py
+3 -4
View File
@@ -1,10 +1,10 @@
import logging import logging
from datetime import datetime from datetime import datetime
from threading import Thread, Event from threading import Event, Thread
import pytz import pytz
import requests import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS from core.constants import HTTP_HEADERS
from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider
@@ -22,8 +22,7 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
self._stop_event = Event() self._stop_event = Event()
def start(self): def start(self):
logging.info( logging.info(f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}") self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}")
self._thread.start() self._thread.start()
providers/solarconditions/kc2gprop.py
+4 -4
View File
@@ -1,10 +1,10 @@
import logging import logging
from datetime import datetime, timezone, timedelta from datetime import datetime, timedelta, timezone
from threading import Thread, Event from threading import Event, Thread
import pytz import pytz
import requests import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS from core.constants import HTTP_HEADERS
from providers.solarconditions.ionosonde_utils import compute_band_states from providers.solarconditions.ionosonde_utils import compute_band_states
@@ -119,7 +119,7 @@ class KC2GProp(SolarConditionsProvider):
except ConnectionError: except ConnectionError:
logging.warning("Connection error when accessing KC2G ionosonde API.") logging.warning("Connection error when accessing KC2G ionosonde API.")
except (ConnectTimeout, ReadTimeout): except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing KC2G ionosonde API.") logging.warning("Timeout when accessing KC2G ionosonde API.")
except Exception: except Exception:
self.status = "Error" self.status = "Error"
logging.exception("Exception in KC2G ionosonde data provider") logging.exception("Exception in KC2G ionosonde data provider")
providers/solarconditions/noaa3dayforecast.py
+19 -9
View File
@@ -2,7 +2,9 @@ import logging
import re import re
from datetime import datetime, timezone from datetime import datetime, timezone
from providers.solarconditions.http_solar_conditions_provider import HTTPSolarConditionsProvider from providers.solarconditions.http_solar_conditions_provider import (
HTTPSolarConditionsProvider,
)
POLL_INTERVAL = 10800 # Every 3 hours POLL_INTERVAL = 10800 # Every 3 hours
URL = "https://services.swpc.noaa.gov/text/3-day-forecast.txt" URL = "https://services.swpc.noaa.gov/text/3-day-forecast.txt"
@@ -32,13 +34,13 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Find the date header line by scanning the next few lines for month & day patterns # Find the date header line by scanning the next few lines for month & day patterns
date_header_idx = None date_header_idx = None
for j in range(start_idx + 1, min(start_idx + 6, len(lines))): for j in range(start_idx + 1, min(start_idx + 6, len(lines))):
if re.search(r'[A-Za-z]{3}\s+\d{2}', lines[j]): if re.search(r"[A-Za-z]{3}\s+\d{2}", lines[j]):
date_header_idx = j date_header_idx = j
break break
if date_header_idx is None: if date_header_idx is None:
logging.warning(f"NOAA 3-day forecast: could not find date header after '{section_header}'") logging.warning(f"NOAA 3-day forecast: could not find date header after '{section_header}'")
return None return None
date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', lines[date_header_idx]) date_matches = re.findall(r"([A-Za-z]{3})\s+(\d{2})", lines[date_header_idx])
if not date_matches: if not date_matches:
logging.warning(f"NOAA 3-day forecast: no dates in header: {lines[date_header_idx]}") logging.warning(f"NOAA 3-day forecast: no dates in header: {lines[date_header_idx]}")
return None return None
@@ -61,7 +63,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
if result: if result:
break break
continue continue
pct_matches = list(re.finditer(r'\b(\d+)%', line_stripped)) pct_matches = list(re.finditer(r"\b(\d+)%", line_stripped))
if not pct_matches: if not pct_matches:
if result: if result:
break break
@@ -94,7 +96,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Extract the year from the header line, e.g. "NOAA Kp index breakdown Apr 2-Apr 4, 2026" # Extract the year from the header line, e.g. "NOAA Kp index breakdown Apr 2-Apr 4, 2026"
header_line = lines[start_idx] header_line = lines[start_idx]
year_match = re.search(r'\b(\d{4})\b', header_line) year_match = re.search(r"\b(\d{4})\b", header_line)
if not year_match: if not year_match:
logging.warning(f"NOAA K-index forecast: could not extract year from: {header_line}") logging.warning(f"NOAA K-index forecast: could not extract year from: {header_line}")
return None return None
@@ -106,7 +108,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
return None return None
date_header_line = lines[start_idx + 2] date_header_line = lines[start_idx + 2]
date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', date_header_line) date_matches = re.findall(r"([A-Za-z]{3})\s+(\d{2})", date_header_line)
if not date_matches: if not date_matches:
logging.warning(f"NOAA K-index forecast: could not parse date headers from: {date_header_line}") logging.warning(f"NOAA K-index forecast: could not parse date headers from: {date_header_line}")
return None return None
@@ -122,7 +124,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Parse each data row, e.g. "00-03UT 2.00 3.00 2.00" # Parse each data row, e.g. "00-03UT 2.00 3.00 2.00"
k_index_forecast = {} k_index_forecast = {}
for line in lines[start_idx + 3 :]: for line in lines[start_idx + 3 :]:
time_match = re.match(r'^(\d{2})-(\d{2})UT\s+(.*)', line.strip()) time_match = re.match(r"^(\d{2})-(\d{2})UT\s+(.*)", line.strip())
if not time_match: if not time_match:
if k_index_forecast: if k_index_forecast:
break break
@@ -130,7 +132,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
start_hour = int(time_match.group(1)) start_hour = int(time_match.group(1))
# Split on 2 or more spaces so that e.g. "5.67 (G2)" stays as one token per column # Split on 2 or more spaces so that e.g. "5.67 (G2)" stays as one token per column
raw_values = re.split(r' {2,}', time_match.group(3).strip()) raw_values = re.split(r" {2,}", time_match.group(3).strip())
for i, val in enumerate(raw_values): for i, val in enumerate(raw_values):
if i >= len(column_dates): if i >= len(column_dates):
@@ -142,7 +144,15 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
continue continue
date = column_dates[i] date = column_dates[i]
start_dt = datetime(date.year, date.month, date.day, start_hour, 0, 0, tzinfo=timezone.utc) start_dt = datetime(
date.year,
date.month,
date.day,
start_hour,
0,
0,
tzinfo=timezone.utc,
)
# Key the data dict by start time # Key the data dict by start time
key = start_dt.timestamp() key = start_dt.timestamp()
providers/spot/aprsis.py
+4 -3
View File
@@ -43,7 +43,8 @@ class APRSIS(SpotProvider):
via_parts = str(data["via"]).split("-") via_parts = str(data["via"]).split("-")
de_call = via_parts[0].upper() de_call = via_parts[0].upper()
de_ssid = via_parts[1].upper() if len(via_parts) > 1 else None de_ssid = via_parts[1].upper() if len(via_parts) > 1 else None
spot = Spot(source="APRS-IS", spot = Spot(
source="APRS-IS",
dx_call=dx_call, dx_call=dx_call,
dx_ssid=dx_ssid, dx_ssid=dx_ssid,
de_call=de_call, de_call=de_call,
@@ -51,8 +52,8 @@ class APRSIS(SpotProvider):
comment=str(data["comment"]) if "comment" in data else None, comment=str(data["comment"]) if "comment" in data else None,
dx_latitude=float(data["latitude"]) if "latitude" in data else None, dx_latitude=float(data["latitude"]) if "latitude" in data else None,
dx_longitude=float(data["longitude"]) if "longitude" in data else None, dx_longitude=float(data["longitude"]) if "longitude" in data else None,
time=datetime.now( time=datetime.now(pytz.UTC).timestamp(),
pytz.UTC).timestamp()) # APRS-IS spots are live so we can assume spot time is "now" ) # APRS-IS spots are live so we can assume spot time is "now"
# Add to our list # Add to our list
self._submit(spot) self._submit(spot)
providers/spot/dxcluster.py
+19 -9
View File
@@ -18,10 +18,12 @@ class DXCluster(SpotProvider):
_LINE_PATTERN_EXCLUDE_RBN = re.compile( _LINE_PATTERN_EXCLUDE_RBN = re.compile(
r"^DX de ([a-z0-9/]+):\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)", r"^DX de ([a-z0-9/]+):\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE) re.IGNORECASE,
)
_LINE_PATTERN_ALLOW_RBN = re.compile( _LINE_PATTERN_ALLOW_RBN = re.compile(
r"^DX de ([a-z0-9/]+)-?#?:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)", r"^DX de ([a-z0-9/]+)-?#?:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE) re.IGNORECASE,
)
def __init__(self, provider_config): def __init__(self, provider_config):
"""Constructor requires hostname and port""" """Constructor requires hostname and port"""
@@ -31,10 +33,13 @@ class DXCluster(SpotProvider):
self._hostname = provider_config["host"] self._hostname = provider_config["host"]
self._port = provider_config["port"] self._port = provider_config["port"]
self._login_prompt = provider_config["login_prompt"] if "login_prompt" in provider_config else "login:" self._login_prompt = provider_config["login_prompt"] if "login_prompt" in provider_config else "login:"
self._login_callsign = provider_config[ self._login_callsign = (
"login_callsign"] if "login_callsign" in provider_config else SERVER_OWNER_CALLSIGN provider_config["login_callsign"] if "login_callsign" in provider_config else SERVER_OWNER_CALLSIGN
)
self._allow_rbn_spots = provider_config["allow_rbn_spots"] if "allow_rbn_spots" in provider_config else False self._allow_rbn_spots = provider_config["allow_rbn_spots"] if "allow_rbn_spots" in provider_config else False
self._spot_line_pattern = self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN self._spot_line_pattern = (
self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
)
self._telnet = None self._telnet = None
self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}") self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}")
self._thread.daemon = True self._thread.daemon = True
@@ -77,14 +82,19 @@ class DXCluster(SpotProvider):
match = self._spot_line_pattern.match(telnet_output.decode("latin-1")) match = self._spot_line_pattern.match(telnet_output.decode("latin-1"))
if match: if match:
spot_time = datetime.strptime(match.group(5), "%H%MZ") spot_time = datetime.strptime(match.group(5), "%H%MZ")
spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(), spot_datetime = datetime.combine(
tzinfo=pytz.UTC) datetime.now(pytz.UTC).date(),
spot = Spot(source=self.name, spot_time.time(),
tzinfo=pytz.UTC,
)
spot = Spot(
source=self.name,
dx_call=match.group(3), dx_call=match.group(3),
de_call=match.group(1), de_call=match.group(1),
freq=float(match.group(2)) * 1000, freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(), comment=match.group(4).strip(),
time=spot_datetime.timestamp()) time=spot_datetime.timestamp(),
)
# Add to our list # Add to our list
self._submit(spot) self._submit(spot)
providers/spot/gma.py
+55 -20
View File
@@ -27,7 +27,12 @@ class GMA(HTTPSpotProvider):
logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.") logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
self._url_data_cache = URLDataCache("GMA") self._url_data_cache = URLDataCache("GMA")
super().__init__("GMA", provider_config, f"{self.SPOTS_URL}?key={self._api_key}", self.POLL_INTERVAL_SEC) super().__init__(
"GMA",
provider_config,
f"{self.SPOTS_URL}?key={self._api_key}",
self.POLL_INTERVAL_SEC,
)
def _http_response_to_spots(self, http_response): def _http_response_to_spots(self, http_response):
new_spots = [] new_spots = []
@@ -41,43 +46,67 @@ class GMA(HTTPSpotProvider):
# Seen some real janky times from GMA, if we don't understand it just ignore this spot # Seen some real janky times from GMA, if we don't understand it just ignore this spot
try: try:
time = datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace( time = (
tzinfo=pytz.UTC).timestamp() datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M")
.replace(tzinfo=pytz.UTC)
.timestamp()
)
except ValueError: except ValueError:
continue continue
spot = Spot(source=self.name, spot = Spot(
source=self.name,
dx_call=source_spot["ACTIVATOR"].upper(), dx_call=source_spot["ACTIVATOR"].upper(),
de_call=source_spot["SPOTTER"].upper(), de_call=source_spot["SPOTTER"].upper(),
# Seen GMA spots with no frequency or with "QRT" in this field # Seen GMA spots with no frequency or with "QRT" in this field
freq=float(source_spot["QRG"]) * 1000 if ( freq=float(source_spot["QRG"]) * 1000
source_spot["QRG"] != "" and source_spot["QRG"] != "QRT") else None, if (source_spot["QRG"] != "" and source_spot["QRG"] != "QRT")
else None,
# Filter out some weird mode strings # Filter out some weird mode strings
mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None, mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
comment=source_spot["TEXT"], comment=source_spot["TEXT"],
sig_refs=[SIGRef(id=source_spot["REF"], sig="", name=source_spot["NAME"], latitude=lat, sig_refs=[
longitude=lon)], SIGRef(
id=source_spot["REF"],
sig="",
name=source_spot["NAME"],
latitude=lat,
longitude=lon,
)
],
time=time, time=time,
dx_latitude=lat, dx_latitude=lat,
dx_longitude=lon, dx_longitude=lon,
qrt=source_spot["QRG"] == "QRT") qrt=source_spot["QRG"] == "QRT",
)
# GMA doesn't give what programme (SIG) the reference is for until we separately look it up. # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
if "REF" in source_spot: if "REF" in source_spot:
try: try:
ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"], ref_response = self._url_data_cache.get(
headers=HTTP_HEADERS) self.REF_INFO_URL_ROOT + source_spot["REF"],
headers=HTTP_HEADERS,
)
# Sometimes this is blank even if it's a 200 response, so handle that # Sometimes this is blank even if it's a 200 response, so handle that
if ref_response.ok and ref_response.text is not None and ref_response.text != "" and ref_response.text != "\n": if (
ref_response.ok
and ref_response.text is not None
and ref_response.text != ""
and ref_response.text != "\n"
):
ref_info = ref_response.json() ref_info = ref_response.json()
# If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
# spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
# and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry # and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry
# to determine if it's a SOTA summit. # to determine if it's a SOTA summit.
if spot.sig_refs and "reftype" in ref_info and ref_info["reftype"] not in ["POTA", if (
"WWFF"] and ( spot.sig_refs
ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info[ and "reftype" in ref_info
"sota"] == ""): and ref_info["reftype"] not in ["POTA", "WWFF"]
and (
ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info["sota"] == ""
)
):
match ref_info["reftype"]: match ref_info["reftype"]:
case "Summit": case "Summit":
spot.sig_refs[0].sig = "GMA" spot.sig_refs[0].sig = "GMA"
@@ -98,7 +127,9 @@ class GMA(HTTPSpotProvider):
spot.sig_refs[0].sig = "MOTA" spot.sig_refs[0].sig = "MOTA"
spot.sig = "MOTA" spot.sig = "MOTA"
case _: case _:
logging.warning(f"GMA spot found with ref type {ref_info['reftype']}, developer needs to add support for this!") logging.warning(
f"GMA spot found with ref type {ref_info['reftype']}, developer needs to add support for this!"
)
spot.sig_refs[0].sig = ref_info["reftype"] spot.sig_refs[0].sig = ref_info["reftype"]
spot.sig = ref_info["reftype"] spot.sig = ref_info["reftype"]
@@ -109,12 +140,16 @@ class GMA(HTTPSpotProvider):
elif not ref_response.from_cache: elif not ref_response.from_cache:
if not ref_response.ok: if not ref_response.ok:
logging.warning( logging.warning(
f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot['REF']}") f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot['REF']}"
)
else: else:
logging.warning( logging.warning(
f"GMA API returned a malformed response when looking up ref {source_spot['REF']}") f"GMA API returned a malformed response when looking up ref {source_spot['REF']}"
)
except: except:
logging.exception(f"Exception when looking up {self.REF_INFO_URL_ROOT}{source_spot['REF']}, ignoring this spot for now") logging.exception(
f"Exception when looking up {self.REF_INFO_URL_ROOT}{source_spot['REF']}, ignoring this spot for now"
)
else: else:
logging.warning(f"The GMA API returned an unexpected response (HTTP {http_response.status_code}).") logging.warning(f"The GMA API returned an unexpected response (HTTP {http_response.status_code}).")
providers/spot/hema.py
+18 -8
View File
@@ -4,7 +4,7 @@ from datetime import datetime
import pytz import pytz
import requests import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS from core.constants import HTTP_HEADERS
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
@@ -52,25 +52,35 @@ class HEMA(HTTPSpotProvider):
continue continue
# Convert to our spot format # Convert to our spot format
spot = Spot(source=self.name, spot = Spot(
source=self.name,
dx_call=spot_items[2].upper(), dx_call=spot_items[2].upper(),
de_call=spotter_comment_match.group(1).upper(), de_call=spotter_comment_match.group(1).upper(),
freq=float(freq_mode_match.group(1)) * 1000000, freq=float(freq_mode_match.group(1)) * 1000000,
mode=freq_mode_match.group(2).upper(), mode=freq_mode_match.group(2).upper(),
comment=spotter_comment_match.group(2), comment=spotter_comment_match.group(2),
sig="HEMA", sig="HEMA",
sig_refs=[SIGRef(id=spot_items[3].upper(), sig="HEMA", name=spot_items[4], sig_refs=[
latitude=float(spot_items[7]), longitude=float(spot_items[8]))], SIGRef(
time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace( id=spot_items[3].upper(),
tzinfo=pytz.UTC).timestamp(), sig="HEMA",
name=spot_items[4],
latitude=float(spot_items[7]),
longitude=float(spot_items[8]),
)
],
time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M")
.replace(tzinfo=pytz.UTC)
.timestamp(),
dx_latitude=float(spot_items[7]), dx_latitude=float(spot_items[7]),
dx_longitude=float(spot_items[8])) dx_longitude=float(spot_items[8]),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other
# code will do that for us. # code will do that for us.
new_spots.append(spot) new_spots.append(spot)
except (ConnectTimeout, ReadTimeout): except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing HEMA spots API.") logging.warning("Timeout when accessing HEMA spots API.")
except ConnectionError: except ConnectionError:
logging.warning("Connection error when accessing HEMA spots API.") logging.warning("Connection error when accessing HEMA spots API.")
return new_spots return new_spots
providers/spot/http_spot_provider.py
+1 -1
View File
@@ -1,6 +1,6 @@
import logging import logging
from datetime import datetime from datetime import datetime
from threading import Thread, Event from threading import Event, Thread
import pytz import pytz
import requests import requests
providers/spot/llota.py
+11 -3
View File
@@ -25,7 +25,8 @@ class LLOTA(HTTPSpotProvider):
comment = str(source_spot["history"][-1]["comment"]) comment = str(source_spot["history"][-1]["comment"])
spotter = str(source_spot["history"][-1]["spotter_callsign"]) spotter = str(source_spot["history"][-1]["spotter_callsign"])
# Convert to our spot format # Convert to our spot format
spot = Spot(source=self.name, spot = Spot(
source=self.name,
source_id=source_spot["id"], source_id=source_spot["id"],
dx_call=source_spot["callsign"].upper(), dx_call=source_spot["callsign"].upper(),
de_call=spotter.upper() if spotter else None, de_call=spotter.upper() if spotter else None,
@@ -33,8 +34,15 @@ class LLOTA(HTTPSpotProvider):
mode=source_spot["mode"].upper(), mode=source_spot["mode"].upper(),
comment=comment, comment=comment,
sig="LLOTA", sig="LLOTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="LLOTA", name=source_spot["reference_name"])], sig_refs=[
time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp()) SIGRef(
id=source_spot["reference"],
sig="LLOTA",
name=source_spot["reference_name"],
)
],
time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp(),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us. # that for us.
providers/spot/parksnpeaks.py
+39 -9
View File
@@ -18,7 +18,17 @@ class ParksNPeaks(HTTPSpotProvider):
SPOTS_URL = "https://www.parksnpeaks.org/api/ALL" SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
SUBMIT_URL = "https://www.parksnpeaks.org/api/SPOT/" SUBMIT_URL = "https://www.parksnpeaks.org/api/SPOT/"
SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv" SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
SUBMITTABLE_SIGS = ["POTA", "SOTA", "WWFF", "HEMA", "WOTA", "ZLOTA", "SIOTA", "KRMNPA", "SANPCPA"] SUBMITTABLE_SIGS = [
"POTA",
"SOTA",
"WWFF",
"HEMA",
"WOTA",
"ZLOTA",
"SIOTA",
"KRMNPA",
"SANPCPA",
]
def __init__(self, provider_config): def __init__(self, provider_config):
super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC) super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -29,18 +39,23 @@ class ParksNPeaks(HTTPSpotProvider):
if http_response and http_response != "": if http_response and http_response != "":
for source_spot in http_response.json(): for source_spot in http_response.json():
# Convert to our spot format # Convert to our spot format
spot = Spot(source=self.name, spot = Spot(
source=self.name,
source_id=source_spot["actID"], source_id=source_spot["actID"],
dx_call=source_spot["actCallsign"].upper(), dx_call=source_spot["actCallsign"].upper(),
de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None, de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
# typo exists in API # typo exists in API
freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "") freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "").replace("+/-", "").strip())
.replace("+/-", "").strip()) * 1000000 if (source_spot["actFreq"] != "") else None, * 1000000
if (source_spot["actFreq"] != "")
else None,
# Seen PNP spots with empty frequency, and with comma-separated thousands digits # Seen PNP spots with empty frequency, and with comma-separated thousands digits
mode=source_spot["actMode"].upper(), mode=source_spot["actMode"].upper(),
comment=source_spot["actComments"], comment=source_spot["actComments"],
time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S").replace( time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S")
tzinfo=pytz.UTC).timestamp()) .replace(tzinfo=pytz.UTC)
.timestamp(),
)
# Extract a de_call if it's in the comment but not in the "actSpoter" field # Extract a de_call if it's in the comment but not in the "actSpoter" field
m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment or "") m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment or "")
@@ -53,7 +68,12 @@ class ParksNPeaks(HTTPSpotProvider):
sig_ref = source_spot["actSiteID"] sig_ref = source_spot["actSiteID"]
if sig and sig != "" and sig != "QRP" and sig_ref and sig_ref != "": if sig and sig != "" and sig != "QRP" and sig_ref and sig_ref != "":
spot.sig = sig spot.sig = sig
sig_refs = [SIGRef(id=source_spot["actSiteID"], sig=source_spot["actClass"].upper())] sig_refs = [
SIGRef(
id=source_spot["actSiteID"],
sig=source_spot["actClass"].upper(),
)
]
spot.sig_refs = sig_refs spot.sig_refs = sig_refs
# Free text location is not present in all spots, so only add it if it's set # Free text location is not present in all spots, so only add it if it's set
@@ -61,7 +81,16 @@ class ParksNPeaks(HTTPSpotProvider):
sig_refs[0].name = source_spot["actLocation"] sig_refs[0].name = source_spot["actLocation"]
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "SANPCPA", "LLOTA"]: if sig not in [
"POTA",
"SOTA",
"WWFF",
"SIOTA",
"ZLOTA",
"KRMNPA",
"SANPCPA",
"LLOTA",
]:
logging.warning(f"PNP spot found with sig {sig}, developer needs to add support for this!") logging.warning(f"PNP spot found with sig {sig}, developer needs to add support for this!")
# Add new spot to the list # Add new spot to the list
@@ -77,7 +106,8 @@ class ParksNPeaks(HTTPSpotProvider):
api_key = credentials.get("api_key", "") api_key = credentials.get("api_key", "")
if not user_id or not api_key: if not user_id or not api_key:
raise ValueError( raise ValueError(
"Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account.") "Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account."
)
sig_ref = spot.sig_refs[0].id if spot.sig_refs else "" sig_ref = spot.sig_refs[0].id if spot.sig_refs else ""
body = { body = {
"actClass": spot.sig or "", "actClass": spot.sig or "",
providers/spot/pota.py
+16 -6
View File
@@ -24,7 +24,8 @@ class POTA(HTTPSpotProvider):
# Iterate through source data # Iterate through source data
for source_spot in http_response.json(): for source_spot in http_response.json():
# Convert to our spot format # Convert to our spot format
spot = Spot(source=self.name, spot = Spot(
source=self.name,
source_id=source_spot["spotId"], source_id=source_spot["spotId"],
dx_call=source_spot["activator"].upper(), dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(), de_call=source_spot["spotter"].upper(),
@@ -32,13 +33,22 @@ class POTA(HTTPSpotProvider):
mode=source_spot["mode"].upper(), mode=source_spot["mode"].upper(),
comment=source_spot["comments"], comment=source_spot["comments"],
sig="POTA", sig="POTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="POTA", name=source_spot["name"], sig_refs=[
latitude=source_spot["latitude"], longitude=source_spot["longitude"])], SIGRef(
time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace( id=source_spot["reference"],
tzinfo=pytz.UTC).timestamp(), sig="POTA",
name=source_spot["name"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
)
],
time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
dx_grid=source_spot["grid6"], dx_grid=source_spot["grid6"],
dx_latitude=source_spot["latitude"], dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"]) dx_longitude=source_spot["longitude"],
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us. # that for us.
providers/spot/rbn.py
+11 -5
View File
@@ -18,7 +18,8 @@ class RBN(SpotProvider):
_LINE_PATTERN = re.compile( _LINE_PATTERN = re.compile(
r"^DX de ([a-z0-9/]+)-.*:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)", r"^DX de ([a-z0-9/]+)-.*:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE) re.IGNORECASE,
)
def __init__(self, provider_config): def __init__(self, provider_config):
"""Constructor requires port number.""" """Constructor requires port number."""
@@ -64,14 +65,19 @@ class RBN(SpotProvider):
match = self._LINE_PATTERN.match(telnet_output.decode("latin-1")) match = self._LINE_PATTERN.match(telnet_output.decode("latin-1"))
if match: if match:
spot_time = datetime.strptime(match.group(5), "%H%MZ") spot_time = datetime.strptime(match.group(5), "%H%MZ")
spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(), spot_datetime = datetime.combine(
tzinfo=pytz.UTC) datetime.now(pytz.UTC).date(),
spot = Spot(source=self.name, spot_time.time(),
tzinfo=pytz.UTC,
)
spot = Spot(
source=self.name,
dx_call=match.group(3), dx_call=match.group(3),
de_call=match.group(1), de_call=match.group(1),
freq=float(match.group(2)) * 1000, freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(), comment=match.group(4).strip(),
time=spot_datetime.timestamp()) time=spot_datetime.timestamp(),
)
# Add to our list # Add to our list
self._submit(spot) self._submit(spot)
providers/spot/sota.py
+26 -13
View File
@@ -2,9 +2,9 @@ import logging
from datetime import datetime from datetime import datetime
import requests import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS, SSB_SUB_MODES, DV_SUB_MODES from core.constants import DV_SUB_MODES, HTTP_HEADERS, SSB_SUB_MODES
from data.sig_ref import SIGRef from data.sig_ref import SIGRef
from data.spot import Spot from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -41,24 +41,33 @@ class SOTA(HTTPSpotProvider):
# Iterate through source data # Iterate through source data
for source_spot in source_data: for source_spot in source_data:
# Convert to our spot format # Convert to our spot format
spot = Spot(source=self.name, spot = Spot(
source=self.name,
source_id=source_spot["id"], source_id=source_spot["id"],
dx_call=source_spot["activatorCallsign"].upper(), dx_call=source_spot["activatorCallsign"].upper(),
dx_name=source_spot["activatorName"], dx_name=source_spot["activatorName"],
de_call=source_spot["callsign"].upper(), de_call=source_spot["callsign"].upper(),
freq=(float(source_spot["frequency"]) * 1000000) if ( freq=(float(source_spot["frequency"]) * 1000000)
source_spot["frequency"] is not None) else None, if (source_spot["frequency"] is not None)
else None,
# Seen SOTA spots with no frequency! # Seen SOTA spots with no frequency!
mode=source_spot["mode"].upper(), mode=source_spot["mode"].upper(),
comment=source_spot["comments"], comment=source_spot["comments"],
sig="SOTA", sig="SOTA",
sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA", sig_refs=[
name=source_spot["summitName"], latitude=source_spot["latitude"], SIGRef(
id=source_spot["summitCode"],
sig="SOTA",
name=source_spot["summitName"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"], longitude=source_spot["longitude"],
activation_score=source_spot["points"])], activation_score=source_spot["points"],
)
],
dx_latitude=source_spot["latitude"], dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"], dx_longitude=source_spot["longitude"],
time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp()) time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp(),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us. # that for us.
@@ -66,7 +75,7 @@ class SOTA(HTTPSpotProvider):
except ConnectionError: except ConnectionError:
logging.warning("Connection error when accessing SOTA spots API") logging.warning("Connection error when accessing SOTA spots API")
except (ConnectTimeout, ReadTimeout): except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing SOTA spots API.") logging.warning("Timeout when accessing SOTA spots API.")
return new_spots return new_spots
def can_submit_spot(self, sig): def can_submit_spot(self, sig):
@@ -102,10 +111,14 @@ class SOTA(HTTPSpotProvider):
"mode": mode or "", "mode": mode or "",
"callsign": spot.de_call, "callsign": spot.de_call,
"comments": spot.comment or "", "comments": spot.comment or "",
"type": "TEST" # todo replatce with NORMAL/QRT once testing complete "type": "TEST", # todo replatce with NORMAL/QRT once testing complete
}
headers = {
**HTTP_HEADERS,
"Authorization": f"bearer {access_token}",
"id_token": id_token,
"Content-Type": "application/json",
} }
headers = {**HTTP_HEADERS, "Authorization": f"bearer {access_token}", "id_token": id_token,
"Content-Type": "application/json"}
response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30)) response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
if not response.ok: if not response.ok:
raise RuntimeError(f"SOTA API returned {response.status_code!s}: {response.text}") raise RuntimeError(f"SOTA API returned {response.status_code!s}: {response.text}")
providers/spot/spot_provider.py
+1 -1
View File
@@ -32,7 +32,7 @@ class SpotProvider:
# Sort the batch so that earliest ones go in first. This helps keep the ordering correct when spots are fired # Sort the batch so that earliest ones go in first. This helps keep the ordering correct when spots are fired
# off to SSE listeners. # off to SSE listeners.
spots = sorted(spots, key=lambda s: (s.time if s and s.time else 0)) spots = sorted(spots, key=lambda s: s.time if s and s.time else 0)
for spot in spots: for spot in spots:
if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time: if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time:
# Fill in any blanks and add to the list # Fill in any blanks and add to the list
providers/spot/sse_spot_provider.py
+12 -6
View File
@@ -37,8 +37,7 @@ class SSESpotProvider(SpotProvider):
try: try:
event_source.close() event_source.close()
except Exception: except Exception:
logging.exception( logging.exception(f"Exception closing SSE connection for {self.name} during stop()")
f"Exception closing SSE connection for {self.name} during stop()")
if self._thread: if self._thread:
self._thread.join(timeout=15) self._thread.join(timeout=15)
@@ -60,14 +59,20 @@ class SSESpotProvider(SpotProvider):
try: try:
logging.debug(f"Connecting to {self.name} spot API...") logging.debug(f"Connecting to {self.name} spot API...")
self.status = "Connecting" self.status = "Connecting"
with EventSource(self._url, headers=HTTP_HEADERS, latest_event_id=self._last_event_id, timeout=10, with EventSource(
on_open=self._on_open, on_error=self._on_error) as event_source: self._url,
headers=HTTP_HEADERS,
latest_event_id=self._last_event_id,
timeout=10,
on_open=self._on_open,
on_error=self._on_error,
) as event_source:
self._set_event_source(event_source) self._set_event_source(event_source)
try: try:
for event in event_source: for event in event_source:
if self._stop_event.is_set(): if self._stop_event.is_set():
break break
if event.type == 'message': if event.type == "message":
try: try:
self._last_event_id = event.last_event_id self._last_event_id = event.last_event_id
new_spot = self._sse_message_to_spot(event.data) new_spot = self._sse_message_to_spot(event.data)
@@ -80,7 +85,8 @@ class SSESpotProvider(SpotProvider):
except Exception: except Exception:
logging.exception( logging.exception(
f"Exception processing message from SSE Spot Provider ({self.name})") f"Exception processing message from SSE Spot Provider ({self.name})"
)
finally: finally:
self._set_event_source(None) self._set_event_source(None)
providers/spot/tiles.py
+34 -13
View File
@@ -14,8 +14,22 @@ class Tiles(HTTPSpotProvider):
POLL_INTERVAL_SEC = 120 POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/spots?active_hours=24" SPOTS_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/spots?active_hours=24"
SUBMIT_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/self-spot" SUBMIT_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/self-spot"
VALID_MODES = ["SSB", "CW", "FT8", "FT4", "FM", "DMR", "D-STAR", "M17", "AX.25", "JS8Call", "PSK31", "Olivia", VALID_MODES = [
"VarAC", "Other"] "SSB",
"CW",
"FT8",
"FT4",
"FM",
"DMR",
"D-STAR",
"M17",
"AX.25",
"JS8Call",
"PSK31",
"Olivia",
"VarAC",
"Other",
]
def __init__(self, provider_config): def __init__(self, provider_config):
super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC) super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -25,7 +39,8 @@ class Tiles(HTTPSpotProvider):
# Iterate through source data # Iterate through source data
for source_spot in http_response.json()["spots"]: for source_spot in http_response.json()["spots"]:
# Convert to our spot format # Convert to our spot format
spot = Spot(source=self.name, spot = Spot(
source=self.name,
source_id=source_spot["id"], source_id=source_spot["id"],
dx_call=source_spot["call_sign"].upper(), dx_call=source_spot["call_sign"].upper(),
# No separate spotter callsign, assume all spots are self-spots # No separate spotter callsign, assume all spots are self-spots
@@ -36,14 +51,21 @@ class Tiles(HTTPSpotProvider):
sig="Tiles", sig="Tiles",
# Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway. # Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
# Just take the grid reference itself as the single Tiles SIG reference. # Just take the grid reference itself as the single Tiles SIG reference.
sig_refs=[SIGRef(id=source_spot["maidenhead_grid"], sig="Tiles", sig_refs=[
name=source_spot["maidenhead_grid"], latitude=source_spot["latitude"], SIGRef(
longitude=source_spot["longitude"])], id=source_spot["maidenhead_grid"],
sig="Tiles",
name=source_spot["maidenhead_grid"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
)
],
time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(), time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
dx_grid=source_spot["maidenhead_grid"], dx_grid=source_spot["maidenhead_grid"],
dx_latitude=source_spot["latitude"], dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"], dx_longitude=source_spot["longitude"],
dx_location_source="GRID") dx_location_source="GRID",
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us. # that for us.
@@ -56,7 +78,6 @@ class Tiles(HTTPSpotProvider):
def submit_spot(self, spot, credentials): def submit_spot(self, spot, credentials):
# Tiles on the air currently only supports *self* spots # Tiles on the air currently only supports *self* spots
if spot.dx_call == spot.de_call: if spot.dx_call == spot.de_call:
# Figure out a valid mode. Borrowed this from PoLo :) # Figure out a valid mode. Borrowed this from PoLo :)
# https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js # https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js
if spot.mode: if spot.mode:
@@ -80,24 +101,24 @@ class Tiles(HTTPSpotProvider):
"lat": spot.dx_latitude or None, "lat": spot.dx_latitude or None,
"lon": spot.dx_longitude or None, "lon": spot.dx_longitude or None,
"qrt": spot.qrt or False, "qrt": spot.qrt or False,
"pin": credentials.get("offline_spot_gateway_pin", "") "pin": credentials.get("offline_spot_gateway_pin", ""),
} }
headers = {**HTTP_HEADERS, "Content-Type": "application/json"} headers = {**HTTP_HEADERS, "Content-Type": "application/json"}
response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30)) response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
if not response.ok: if not response.ok:
raise RuntimeError( raise RuntimeError(f"Tiles on the Air API returned {response.status_code!s}: {response.text}")
f"Tiles on the Air API returned {response.status_code!s}: {response.text}")
else: else:
raise RuntimeError("The Tiles on the Air API requires a mode to be set.") raise RuntimeError("The Tiles on the Air API requires a mode to be set.")
else: else:
raise RuntimeError( raise RuntimeError(
"The Tiles on the Air API only supports self-spots, the DX call and spotter call must match.") "The Tiles on the Air API only supports self-spots, the DX call and spotter call must match."
)
# Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles' # Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles'
# strange frequency format where we can sometimes have e.g. "14.123.5". # strange frequency format where we can sometimes have e.g. "14.123.5".
def strip_extra_decimal_points(s): def strip_extra_decimal_points(s):
parts = s.split('.', 1) parts = s.split(".", 1)
if len(parts) == 1: if len(parts) == 1:
return s return s
return f"{parts[0]}.{parts[1].replace('.', '')}" return f"{parts[0]}.{parts[1].replace('.', '')}"
providers/spot/towers.py
+6 -3
View File
@@ -26,14 +26,17 @@ class Towers(HTTPSpotProvider):
likely_freq = float(source_spot["freq"]) * 1000 likely_freq = float(source_spot["freq"]) * 1000
if likely_freq < 1000000: if likely_freq < 1000000:
likely_freq = likely_freq * 1000 likely_freq = likely_freq * 1000
spot = Spot(source=self.name, spot = Spot(
source=self.name,
dx_call=source_spot["call"].upper(), dx_call=source_spot["call"].upper(),
freq=likely_freq, freq=likely_freq,
comment=source_spot["comment"], comment=source_spot["comment"],
sig="Towers", sig="Towers",
sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")], sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
time=datetime.strptime(response_json["updated"][:10] + source_spot["time"], time=datetime.strptime(
"%Y-%m-%d%H:%M").timestamp()) response_json["updated"][:10] + source_spot["time"], "%Y-%m-%d%H:%M"
).timestamp(),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us. # that for us.
providers/spot/ukpacketnet.py
+32 -17
View File
@@ -36,42 +36,56 @@ class UKPacketNet(HTTPSpotProvider):
# First build a "full" comment combining some of the extra info # First build a "full" comment combining some of the extra info
comment = listed_port["comment"] if "comment" in listed_port else "" comment = listed_port["comment"] if "comment" in listed_port else ""
comment = f"{comment} {listed_port['mode']}" if "mode" in listed_port else comment comment = f"{comment} {listed_port['mode']}" if "mode" in listed_port else comment
comment = f"{comment} {listed_port['modulation']}" if "modulation" in listed_port else comment comment = (
comment = f"{comment} {listed_port['baud']!s} baud" if "baud" in listed_port and listed_port[ f"{comment} {listed_port['modulation']}" if "modulation" in listed_port else comment
"baud"] > 0 else comment )
comment = (
f"{comment} {listed_port['baud']!s} baud"
if "baud" in listed_port and listed_port["baud"] > 0
else comment
)
# Get frequency from the comment if it's not set properly in the data structure. This is # Get frequency from the comment if it's not set properly in the data structure. This is
# very hacky but a lot of node comments contain their frequency as the first or second # very hacky but a lot of node comments contain their frequency as the first or second
# word of their comment, but not in the proper data structure field. # word of their comment, but not in the proper data structure field.
freq = listed_port["freq"] if "freq" in listed_port and listed_port[ freq = (
"freq"] > 0 else None listed_port["freq"] if "freq" in listed_port and listed_port["freq"] > 0 else None
)
if not freq and comment: if not freq and comment:
possible_freq = comment.split(" ")[0].upper().replace("MHZ", "") possible_freq = comment.split(" ")[0].upper().replace("MHZ", "")
if re.match(r"^[0-9.]+$", if (
possible_freq) and possible_freq != "1200" and possible_freq != "9600": re.match(r"^[0-9.]+$", possible_freq)
and possible_freq != "1200"
and possible_freq != "9600"
):
freq = float(possible_freq) * 1000000 freq = float(possible_freq) * 1000000
if not freq and len(comment.split(" ")) > 1: if not freq and len(comment.split(" ")) > 1:
possible_freq = comment.split(" ")[1].upper().replace("MHZ", "") possible_freq = comment.split(" ")[1].upper().replace("MHZ", "")
if re.match(r"^[0-9.]+$", if (
possible_freq) and possible_freq != "1200" and possible_freq != "9600": re.match(r"^[0-9.]+$", possible_freq)
and possible_freq != "1200"
and possible_freq != "9600"
):
freq = float(possible_freq) * 1000000 freq = float(possible_freq) * 1000000
# Check for a found frequency likely having been in kHz, sorry to all GHz packet folks # Check for a found frequency likely having been in kHz, sorry to all GHz packet folks
if freq and freq > 1000000000: if freq and freq > 1000000000:
freq = freq / 1000 freq = freq / 1000
# Now build the spot object # Now build the spot object
spot = Spot(source=self.name, spot = Spot(
source=self.name,
dx_call=heard["callsign"].upper(), dx_call=heard["callsign"].upper(),
de_call=node["callsign"].upper(), de_call=node["callsign"].upper(),
freq=freq, freq=freq,
mode="PKT", mode="PKT",
comment=comment, comment=comment,
time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S").replace( time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S")
tzinfo=pytz.UTC).timestamp(), .replace(tzinfo=pytz.UTC)
de_grid=node["location"]["locator"] if "locator" in node[ .timestamp(),
"location"] else None, de_grid=node["location"]["locator"] if "locator" in node["location"] else None,
de_latitude=node["location"]["coords"]["lat"], de_latitude=node["location"]["coords"]["lat"],
de_longitude=node["location"]["coords"]["lon"]) de_longitude=node["location"]["coords"]["lon"],
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us. # that for us.
@@ -84,8 +98,9 @@ class UKPacketNet(HTTPSpotProvider):
# data, and we can use that to look these up. # data, and we can use that to look these up.
for spot in new_spots: for spot in new_spots:
if spot.dx_call in nodes: if spot.dx_call in nodes:
spot.dx_grid = nodes[spot.dx_call]["location"]["locator"] if "locator" in nodes[spot.dx_call][ spot.dx_grid = (
"location"] else None nodes[spot.dx_call]["location"]["locator"] if "locator" in nodes[spot.dx_call]["location"] else None
)
spot.dx_latitude = nodes[spot.dx_call]["location"]["coords"]["lat"] spot.dx_latitude = nodes[spot.dx_call]["location"]["coords"]["lat"]
spot.dx_longitude = nodes[spot.dx_call]["location"]["coords"]["lon"] spot.dx_longitude = nodes[spot.dx_call]["location"]["coords"]["lon"]
providers/spot/websocket_spot_provider.py
+1 -2
View File
@@ -60,8 +60,7 @@ class WebsocketSpotProvider(SpotProvider):
logging.debug(f"Received data from {self.name} spot API.") logging.debug(f"Received data from {self.name} spot API.")
except Exception: except Exception:
logging.exception( logging.exception(f"Exception processing message from Websocket Spot Provider ({self.name})")
f"Exception processing message from Websocket Spot Provider ({self.name})")
except Exception as e: except Exception as e:
self.status = "Error" self.status = "Error"
providers/spot/wota.py
+10 -5
View File
@@ -28,10 +28,13 @@ class WOTA(HTTPSpotProvider):
rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig"))) rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
# Iterate through source data # Iterate through source data
for source_spot in rss.channel.items: for source_spot in rss.channel.items:
try: try:
# Reject GUID missing or zero # Reject GUID missing or zero
if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0": if (
not source_spot.guid
or not source_spot.guid.content
or source_spot.guid.content == "http://www.wota.org.uk/spots/0"
):
continue continue
# Pick apart the title # Pick apart the title
@@ -48,7 +51,7 @@ class WOTA(HTTPSpotProvider):
# Pick apart the description # Pick apart the description
desc_split = source_spot.description.split(". ") desc_split = source_spot.description.split(". ")
freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip() freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
freq_mode_split = re.split(r'[\-\s]+', freq_mode) freq_mode_split = re.split(r"[\-\s]+", freq_mode)
freq_hz = float(freq_mode_split[0].replace("'", ".")) * 1000000 freq_hz = float(freq_mode_split[0].replace("'", ".")) * 1000000
mode = None mode = None
if len(freq_mode_split) > 1: if len(freq_mode_split) > 1:
@@ -64,7 +67,8 @@ class WOTA(HTTPSpotProvider):
time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC) time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
# Convert to our spot format # Convert to our spot format
spot = Spot(source=self.name, spot = Spot(
source=self.name,
source_id=source_spot.guid.content, source_id=source_spot.guid.content,
dx_call=dx_call, dx_call=dx_call,
de_call=spotter, de_call=spotter,
@@ -73,7 +77,8 @@ class WOTA(HTTPSpotProvider):
comment=comment, comment=comment,
sig="WOTA", sig="WOTA",
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [], sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
time=time.timestamp()) time=time.timestamp(),
)
new_spots.append(spot) new_spots.append(spot)
except Exception as e: except Exception as e:
providers/spot/wwbota.py
+11 -4
View File
@@ -20,11 +20,17 @@ class WWBOTA(SSESpotProvider):
# n-fer activations. # n-fer activations.
refs = [] refs = []
for ref in source_spot["references"]: for ref in source_spot["references"]:
sigref = SIGRef(id=ref["reference"], sig="WWBOTA", name=ref["name"], latitude=ref["lat"], sigref = SIGRef(
longitude=ref["long"]) id=ref["reference"],
sig="WWBOTA",
name=ref["name"],
latitude=ref["lat"],
longitude=ref["long"],
)
refs.append(sigref) refs.append(sigref)
spot = Spot(source=self.name, spot = Spot(
source=self.name,
dx_call=source_spot["call"].upper(), dx_call=source_spot["call"].upper(),
de_call=source_spot["spotter"].upper(), de_call=source_spot["spotter"].upper(),
freq=float(source_spot["freq"]) * 1000000, freq=float(source_spot["freq"]) * 1000000,
@@ -38,7 +44,8 @@ class WWBOTA(SSESpotProvider):
dx_grid=source_spot["references"][0]["locator"], dx_grid=source_spot["references"][0]["locator"],
dx_latitude=source_spot["references"][0]["lat"], dx_latitude=source_spot["references"][0]["lat"],
dx_longitude=source_spot["references"][0]["long"], dx_longitude=source_spot["references"][0]["long"],
qrt=source_spot["type"] == "QRT") qrt=source_spot["type"] == "QRT",
)
# WWBOTA does support a special "Test" spot type, we need to avoid adding that. # WWBOTA does support a special "Test" spot type, we need to avoid adding that.
return spot if source_spot["type"] != "Test" else None return spot if source_spot["type"] != "Test" else None
providers/spot/wwff.py
+13 -4
View File
@@ -21,7 +21,8 @@ class WWFF(HTTPSpotProvider):
# Iterate through source data # Iterate through source data
for source_spot in http_response.json(): for source_spot in http_response.json():
# Convert to our spot format # Convert to our spot format
spot = Spot(source=self.name, spot = Spot(
source=self.name,
source_id=source_spot["id"], source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(), dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(), de_call=source_spot["spotter"].upper(),
@@ -29,11 +30,19 @@ class WWFF(HTTPSpotProvider):
mode=source_spot["mode"].upper(), mode=source_spot["mode"].upper(),
comment=source_spot["remarks"], comment=source_spot["remarks"],
sig="WWFF", sig="WWFF",
sig_refs=[SIGRef(id=source_spot["reference"], sig="WWFF", name=source_spot["reference_name"], sig_refs=[
latitude=source_spot["latitude"], longitude=source_spot["longitude"])], SIGRef(
id=source_spot["reference"],
sig="WWFF",
name=source_spot["reference_name"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
)
],
time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(), time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
dx_latitude=source_spot["latitude"], dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"]) dx_longitude=source_spot["longitude"],
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us. # that for us.
providers/spot/xota.py
+10 -3
View File
@@ -29,14 +29,21 @@ class XOTA(WebsocketSpotProvider):
string = b.decode("utf-8") string = b.decode("utf-8")
source_spot = json.loads(string) source_spot = json.loads(string)
ref_id = f"{self._sig_ref_prefix} {source_spot['reference']['title']}" ref_id = f"{self._sig_ref_prefix} {source_spot['reference']['title']}"
spot = Spot(source=self.name, spot = Spot(
source=self.name,
source_id=source_spot["id"], source_id=source_spot["id"],
dx_call=source_spot["stationCallSign"].upper(), dx_call=source_spot["stationCallSign"].upper(),
freq=float(source_spot["freq"]) * 1000, freq=float(source_spot["freq"]) * 1000,
mode=source_spot["mode"].upper(), mode=source_spot["mode"].upper(),
sig=self.SIG, sig=self.SIG,
sig_refs=[ sig_refs=[
SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"])], SIGRef(
id=ref_id,
sig=self.SIG or "",
url=source_spot["reference"]["website"],
)
],
time=datetime.now(pytz.UTC).timestamp(), time=datetime.now(pytz.UTC).timestamp(),
qrt=source_spot["state"] != "active") qrt=source_spot["state"] != "active",
)
return spot return spot
providers/spot/zlota.py
+13 -4
View File
@@ -26,7 +26,8 @@ class ZLOTA(HTTPSpotProvider):
freq_hz = freq_hz * 1000 freq_hz = freq_hz * 1000
# Convert to our spot format # Convert to our spot format
spot = Spot(source=self.name, spot = Spot(
source=self.name,
source_id=source_spot["id"], source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(), dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(), de_call=source_spot["spotter"].upper(),
@@ -34,9 +35,17 @@ class ZLOTA(HTTPSpotProvider):
mode=source_spot["mode"].upper().strip(), mode=source_spot["mode"].upper().strip(),
comment=source_spot["comments"], comment=source_spot["comments"],
sig="ZLOTA", sig="ZLOTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="ZLOTA", name=source_spot["name"])], sig_refs=[
time=datetime.fromisoformat(source_spot["referenced_time"].replace("Z", "+00:00")).astimezone( SIGRef(
pytz.UTC).timestamp()) id=source_spot["reference"],
sig="ZLOTA",
name=source_spot["name"],
)
],
time=datetime.fromisoformat(source_spot["referenced_time"].replace("Z", "+00:00"))
.astimezone(pytz.UTC)
.timestamp(),
)
new_spots.append(spot) new_spots.append(spot)
return new_spots return new_spots
providers/staticdata/cqzonedata.py
+4 -2
View File
@@ -5,7 +5,9 @@ import geopandas
from shapely import prepare from shapely import prepare
from core.data_store import DATA_STORE from core.data_store import DATA_STORE
from providers.staticdata.local_file_static_data_provider import LocalFileStaticDataProvider from providers.staticdata.local_file_static_data_provider import (
LocalFileStaticDataProvider,
)
class CQZoneData(LocalFileStaticDataProvider): class CQZoneData(LocalFileStaticDataProvider):
@@ -21,7 +23,7 @@ class CQZoneData(LocalFileStaticDataProvider):
with open(path) as f: with open(path) as f:
cq_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"]) cq_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
for idx in cq_zone_data.index: for idx in cq_zone_data.index:
prepare(cq_zone_data.at[idx, 'geometry']) prepare(cq_zone_data.at[idx, "geometry"])
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to prepare the rest # Bail out if a stop has been requested, i.e. the program is shutting down - no need to prepare the rest
# of the data in this case # of the data in this case
providers/staticdata/file_download_static_data_provider.py
+5 -4
View File
@@ -1,6 +1,6 @@
import logging import logging
from datetime import datetime from datetime import datetime
from threading import Thread, Event from threading import Event, Thread
import pytz import pytz
from requests import ReadTimeout from requests import ReadTimeout
@@ -27,8 +27,7 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
def start(self): def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting. # subsequent polls, so start() returns immediately and the application can continue starting.
logging.info( logging.info(f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.")
f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}") self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}")
self._thread.start() self._thread.start()
@@ -57,7 +56,9 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
logging.info(f"Updated static reference data for {self.name}") logging.info(f"Updated static reference data for {self.name}")
else: else:
self.status = "Error" self.status = "Error"
logging.warning(f"HTTP {http_response.status_code} when downloading static reference data for {self.name}.") logging.warning(
f"HTTP {http_response.status_code} when downloading static reference data for {self.name}."
)
except ConnectionError: except ConnectionError:
self.status = "Error" self.status = "Error"
providers/staticdata/ituzonedata.py
+4 -2
View File
@@ -5,7 +5,9 @@ import geopandas
from shapely import prepare from shapely import prepare
from core.data_store import DATA_STORE from core.data_store import DATA_STORE
from providers.staticdata.local_file_static_data_provider import LocalFileStaticDataProvider from providers.staticdata.local_file_static_data_provider import (
LocalFileStaticDataProvider,
)
class ITUZoneData(LocalFileStaticDataProvider): class ITUZoneData(LocalFileStaticDataProvider):
@@ -21,7 +23,7 @@ class ITUZoneData(LocalFileStaticDataProvider):
with open(path) as f: with open(path) as f:
itu_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"]) itu_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
for idx in itu_zone_data.index: for idx in itu_zone_data.index:
prepare(itu_zone_data.at[idx, 'geometry']) prepare(itu_zone_data.at[idx, "geometry"])
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to prepare the rest # Bail out if a stop has been requested, i.e. the program is shutting down - no need to prepare the rest
# of the data in this case # of the data in this case
providers/staticdata/k0swe.py
+3 -3
View File
@@ -1,7 +1,9 @@
import logging import logging
from core.data_store import DATA_STORE from core.data_store import DATA_STORE
from providers.staticdata.file_download_static_data_provider import FileDownloadStaticDataProvider from providers.staticdata.file_download_static_data_provider import (
FileDownloadStaticDataProvider,
)
class K0SWE(FileDownloadStaticDataProvider): class K0SWE(FileDownloadStaticDataProvider):
@@ -44,5 +46,3 @@ class K0SWE(FileDownloadStaticDataProvider):
except Exception: except Exception:
logging.exception("Exception when loading K0SWE dxcc.json.") logging.exception("Exception when loading K0SWE dxcc.json.")
return False return False
providers/staticdata/static_data_provider.py
-2
View File
@@ -15,13 +15,11 @@ class StaticDataProvider:
self.status = "Not Started" if self.enabled else "Disabled" self.status = "Not Started" if self.enabled else "Disabled"
self.reference_count = 0 self.reference_count = 0
def start(self): def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources""" """Start the provider. This should return immediately after spawning threads to access the remote resources"""
raise NotImplementedError("Subclasses must implement this method") raise NotImplementedError("Subclasses must implement this method")
def stop(self): def stop(self):
"""Stop any threads and prepare for application shutdown""" """Stop any threads and prepare for application shutdown"""
requirements.txt
+1
View File
@@ -20,3 +20,4 @@ geopandas~=0.13.2
simplejson~=4.1.1 simplejson~=4.1.1
cachetools~=7.1.6 cachetools~=7.1.6
fastkml~=1.4.0 fastkml~=1.4.0
ruff~=0.16.3
server/handlers/api/addspot.py
+37 -22
View File
@@ -14,8 +14,7 @@ from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRE
from core.constants import UNKNOWN_BAND from core.constants import UNKNOWN_BAND
from core.prometheus_metrics_handler import api_requests_counter from core.prometheus_metrics_handler import api_requests_counter
from core.sig_utils import get_ref_regex_for_sig from core.sig_utils import get_ref_regex_for_sig
from core.utils import infer_band_from_freq from core.utils import infer_band_from_freq, safe_json_dumps
from core.utils import safe_json_dumps
from data.spot import Spot from data.spot import Spot
from providers.spot.spot_provider import SpotProvider from providers.spot.spot_provider import SpotProvider
@@ -25,7 +24,12 @@ RECAPTCHA_VERIFY_URL = "https://www.google.com/recaptcha/api/siteverify"
class APISpotHandler(tornado.web.RequestHandler): class APISpotHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/spot (POST)""" """API request handler for /api/v2/spot (POST)"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._spots = None self._spots = None
self._web_server_metrics = None self._web_server_metrics = None
self._spot_providers = None self._spot_providers = None
@@ -53,7 +57,7 @@ class APISpotHandler(tornado.web.RequestHandler):
return return
# Reject if format not json # Reject if format not json
if not self.request.headers.get('Content-Type', '').startswith("application/json"): if not self.request.headers.get("Content-Type", "").startswith("application/json"):
self.set_status(415) self.set_status(415)
self.write(safe_json_dumps("Error - request Content-Type must be application/json")) self.write(safe_json_dumps("Error - request Content-Type must be application/json"))
self.set_header("Cache-Control", "no-store") self.set_header("Cache-Control", "no-store")
@@ -82,13 +86,10 @@ class APISpotHandler(tornado.web.RequestHandler):
upstream_credentials = handling.get("upstream_credentials", {}) upstream_credentials = handling.get("upstream_credentials", {})
captcha_token = handling.get("captcha_token", None) captcha_token = handling.get("captcha_token", None)
# Spothole v2.0 release only: deny upstream spotting. Spothole API breaking changes were in v2.0 but # Spothole v2.0 release only: deny upstream spotting. Spothole API breaking changes were in v2.0 but
# functionality is not ready yet. TODO # functionality is not ready yet. TODO
submit_upstream = False submit_upstream = False
# Verify CAPTCHA if required # Verify CAPTCHA if required
if RECAPTCHA_SECRET_KEY: if RECAPTCHA_SECRET_KEY:
if not captcha_token: if not captcha_token:
@@ -111,7 +112,8 @@ class APISpotHandler(tornado.web.RequestHandler):
if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call: if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
self.set_status(422) self.set_status(422)
self.write( self.write(
safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.")) safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.")
)
self.set_header("Cache-Control", "no-store") self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json") self.set_header("Content-Type", "application/json")
return return
@@ -133,8 +135,7 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject if frequency not in a known band # Reject if frequency not in a known band
if infer_band_from_freq(spot.freq) == UNKNOWN_BAND: if infer_band_from_freq(spot.freq) == UNKNOWN_BAND:
self.set_status(422) self.set_status(422)
self.write( self.write(safe_json_dumps(f"Error - Frequency of {spot.freq / 1000.0!s}kHz is not in a known band."))
safe_json_dumps(f"Error - Frequency of {spot.freq / 1000.0!s}kHz is not in a known band."))
self.set_header("Cache-Control", "no-store") self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json") self.set_header("Content-Type", "application/json")
return return
@@ -142,20 +143,29 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject if grid formatting incorrect # Reject if grid formatting incorrect
if spot.dx_grid and not re.match( if spot.dx_grid and not re.match(
r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$", r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
spot.dx_grid.upper()): spot.dx_grid.upper(),
):
self.set_status(422) self.set_status(422)
self.write( self.write(safe_json_dumps(f"Error - '{spot.dx_grid}' does not look like a valid Maidenhead grid."))
safe_json_dumps(f"Error - '{spot.dx_grid}' does not look like a valid Maidenhead grid."))
self.set_header("Cache-Control", "no-store") self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json") self.set_header("Content-Type", "application/json")
return return
# Reject if sig_ref format incorrect for sig # Reject if sig_ref format incorrect for sig
if spot.sig and spot.sig_refs and len(spot.sig_refs) > 0 and spot.sig_refs[0].id and get_ref_regex_for_sig( if (
spot.sig) and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id): spot.sig
and spot.sig_refs
and len(spot.sig_refs) > 0
and spot.sig_refs[0].id
and get_ref_regex_for_sig(spot.sig)
and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id)
):
self.set_status(422) self.set_status(422)
self.write(safe_json_dumps( self.write(
f"Error - '{spot.sig_refs[0].id}' does not look like a valid reference for {spot.sig}.")) safe_json_dumps(
f"Error - '{spot.sig_refs[0].id}' does not look like a valid reference for {spot.sig}."
)
)
self.set_header("Cache-Control", "no-store") self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json") self.set_header("Content-Type", "application/json")
return return
@@ -185,7 +195,8 @@ class APISpotHandler(tornado.web.RequestHandler):
if not spot.dx_grid and upstream_provider_name == "Tiles": if not spot.dx_grid and upstream_provider_name == "Tiles":
self.set_status(422) self.set_status(422)
self.write( self.write(
safe_json_dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.")) safe_json_dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.")
)
self.set_header("Cache-Control", "no-store") self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json") self.set_header("Content-Type", "application/json")
return return
@@ -210,7 +221,9 @@ class APISpotHandler(tornado.web.RequestHandler):
upstream_warning = str(e) upstream_warning = str(e)
except Exception: except Exception:
logging.exception(f"Failed to submit spot upstream to {upstream_provider_name}") logging.exception(f"Failed to submit spot upstream to {upstream_provider_name}")
upstream_warning = f"Spot was saved locally but upstream submission to {upstream_provider_name} failed." upstream_warning = (
f"Spot was saved locally but upstream submission to {upstream_provider_name} failed."
)
else: else:
upstream_warning = f"No enabled provider named '{upstream_provider_name}' supports upstream submission for {spot.sig if spot.sig else ''} spots." upstream_warning = f"No enabled provider named '{upstream_provider_name}' supports upstream submission for {spot.sig if spot.sig else ''} spots."
@@ -250,10 +263,12 @@ class APISpotHandler(tornado.web.RequestHandler):
"""Verify a Google reCAPTCHA v2 token. Returns True if valid.""" """Verify a Google reCAPTCHA v2 token. Returns True if valid."""
try: try:
response = requests.post(RECAPTCHA_VERIFY_URL, response = requests.post(
RECAPTCHA_VERIFY_URL,
data={"secret": RECAPTCHA_SECRET_KEY, "response": token}, data={"secret": RECAPTCHA_SECRET_KEY, "response": token},
timeout=(5, 10)) timeout=(5, 10),
)
return response.ok and response.json().get("success", False) return response.ok and response.json().get("success", False)
except Exception: except Exception:
logging.exception(f"reCAPTCHA verification request failed") logging.exception("reCAPTCHA verification request failed")
return False return False
server/handlers/api/alerts.py
+18 -9
View File
@@ -17,7 +17,12 @@ from data.lookup_credentials import extract_credentials
class APIAlertsHandler(tornado.web.RequestHandler): class APIAlertsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/alerts""" """API request handler for /api/v2/alerts"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._alerts = None self._alerts = None
self._web_server_metrics = None self._web_server_metrics = None
super().__init__(application, request, **kwargs) super().__init__(application, request, **kwargs)
@@ -82,8 +87,7 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
def custom_headers(self): def custom_headers(self):
"""Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data""" """Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data"""
return {"Cache-Control": "no-store", return {"Cache-Control": "no-store", "X-Accel-Buffering": "no"}
"X-Accel-Buffering": "no"}
def open(self): def open(self):
try: try:
@@ -142,7 +146,7 @@ def get_alert_list_with_filters(all_alerts, query):
a = all_alerts.get(k) a = all_alerts.get(k)
if a is not None: if a is not None:
alerts.append(a) alerts.append(a)
alerts = sorted(alerts, key=lambda alert: (alert.start_time if alert and alert.start_time else 0)) alerts = sorted(alerts, key=lambda alert: alert.start_time if alert and alert.start_time else 0)
alerts = list(filter(lambda alert: alert_allowed_by_query(alert, query), alerts)) alerts = list(filter(lambda alert: alert_allowed_by_query(alert, query), alerts))
if "limit" in query.keys(): if "limit" in query.keys():
alerts = alerts[: int(query.get("limit"))] alerts = alerts[: int(query.get("limit"))]
@@ -164,8 +168,11 @@ def alert_allowed_by_query(alert, query):
# Check the duration if end_time is provided. If end_time is not provided, assume the activation is # Check the duration if end_time is provided. If end_time is not provided, assume the activation is
# "short", i.e. it always passes this check. If dxpeditions_skip_max_duration_check is true and # "short", i.e. it always passes this check. If dxpeditions_skip_max_duration_check is true and
# the alert is a dxpedition, it also always passes the check. # the alert is a dxpedition, it also always passes the check.
if alert.is_dxpedition and (query.get( if alert.is_dxpedition and (
"dxpeditions_skip_max_duration_check").upper() == "TRUE" if "dxpeditions_skip_max_duration_check" in query.keys() else False): query.get("dxpeditions_skip_max_duration_check").upper() == "TRUE"
if "dxpeditions_skip_max_duration_check" in query.keys()
else False
):
continue continue
if alert.end_time and alert.start_time and alert.end_time - alert.start_time > max_duration: if alert.end_time and alert.start_time and alert.end_time - alert.start_time > max_duration:
return False return False
@@ -192,8 +199,10 @@ def alert_allowed_by_query(alert, query):
return False return False
case "text_includes": case "text_includes":
text_includes = query.get(k).strip() text_includes = query.get(k).strip()
if (not alert.dx_call or text_includes.upper() not in alert.dx_call.upper()) \ if (
and (not alert.comment or text_includes.upper() not in alert.comment.upper()) \ (not alert.dx_call or text_includes.upper() not in alert.dx_call.upper())
and (not alert.freqs_modes or text_includes.upper() not in alert.freqs_modes.upper()): and (not alert.comment or text_includes.upper() not in alert.comment.upper())
and (not alert.freqs_modes or text_includes.upper() not in alert.freqs_modes.upper())
):
return False return False
return True return True
server/handlers/api/dxstats.py
+12 -4
View File
@@ -21,7 +21,12 @@ BANDS_SET = frozenset(BANDS)
class APIDxStatsHandler(tornado.web.RequestHandler): class APIDxStatsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/dxstats""" """API request handler for /api/v2/dxstats"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._spots = None self._spots = None
self._web_server_metrics = None self._web_server_metrics = None
super().__init__(application, request, **kwargs) super().__init__(application, request, **kwargs)
@@ -46,12 +51,15 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
continue continue
if not spot.time or spot.time < one_hour_ago: if not spot.time or spot.time < one_hour_ago:
continue continue
if spot.de_continent in CONTINENTS_SET and spot.dx_continent in CONTINENTS_SET and spot.band in BANDS_SET: if (
spot.de_continent in CONTINENTS_SET
and spot.dx_continent in CONTINENTS_SET
and spot.band in BANDS_SET
):
counts[spot.de_continent, spot.dx_continent, spot.band] += 1 counts[spot.de_continent, spot.dx_continent, spot.band] += 1
result = { result = {
de: {dx: {band: counts[de, dx, band] for band in BANDS} for dx in CONTINENTS} de: {dx: {band: counts[de, dx, band] for band in BANDS} for dx in CONTINENTS} for de in CONTINENTS
for de in CONTINENTS
} }
self.write(json.dumps(result)) self.write(json.dumps(result))
server/handlers/api/lookups.py
+32 -11
View File
@@ -10,7 +10,11 @@ from tornado.web import Application
from core.call_lookup_helper import get_call_info from core.call_lookup_helper import get_call_info
from core.constants import SIGS from core.constants import SIGS
from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone from core.geo_utils import (
lat_lon_for_grid_sw_corner_plus_size,
lat_lon_to_cq_zone,
lat_lon_to_itu_zone,
)
from core.prometheus_metrics_handler import api_requests_counter from core.prometheus_metrics_handler import api_requests_counter
from core.sig_lookup_helper import populate_missing_sig_ref_info from core.sig_lookup_helper import populate_missing_sig_ref_info
from core.sig_utils import get_ref_regex_for_sig from core.sig_utils import get_ref_regex_for_sig
@@ -22,7 +26,12 @@ from data.sig_ref import SIGRef
class APILookupCallHandler(tornado.web.RequestHandler): class APILookupCallHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/lookup/call""" """API request handler for /api/v2/lookup/call"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._web_server_metrics = None self._web_server_metrics = None
super().__init__(application, request, **kwargs) super().__init__(application, request, **kwargs)
@@ -42,7 +51,7 @@ class APILookupCallHandler(tornado.web.RequestHandler):
query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()} query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
# The "call" query param must exist and look like a callsign # The "call" query param must exist and look like a callsign
if "call" in query_params.keys(): if "call" in query_params:
call = str(query_params.get("call")).upper() call = str(query_params.get("call")).upper()
if re.match(r"^[A-Z0-9/\-]*$", call): if re.match(r"^[A-Z0-9/\-]*$", call):
credentials = extract_credentials(self.request.headers) credentials = extract_credentials(self.request.headers)
@@ -68,7 +77,12 @@ class APILookupCallHandler(tornado.web.RequestHandler):
class APILookupSIGRefHandler(tornado.web.RequestHandler): class APILookupSIGRefHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/lookup/sigref""" """API request handler for /api/v2/lookup/sigref"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._web_server_metrics = None self._web_server_metrics = None
super().__init__(application, request, **kwargs) super().__init__(application, request, **kwargs)
@@ -89,7 +103,7 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
# "sig" and "id" query params must exist, SIG must be known, and if we have a reference regex for that SIG, # "sig" and "id" query params must exist, SIG must be known, and if we have a reference regex for that SIG,
# the provided id must match it. # the provided id must match it.
if "sig" in query_params.keys() and "id" in query_params.keys(): if "sig" in query_params and "id" in query_params:
sig = str(query_params.get("sig")).upper() sig = str(query_params.get("sig")).upper()
ref_id = str(query_params.get("id")).upper() ref_id = str(query_params.get("id")).upper()
if sig in list(map(lambda p: p.name.upper(), SIGS)): if sig in list(map(lambda p: p.name.upper(), SIGS)):
@@ -98,8 +112,9 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
self.write(safe_json_dumps(data)) self.write(safe_json_dumps(data))
else: else:
self.write(safe_json_dumps( self.write(
f"Error - '{ref_id}' does not look like a valid reference ID for {sig}.")) safe_json_dumps(f"Error - '{ref_id}' does not look like a valid reference ID for {sig}.")
)
self.set_status(422) self.set_status(422)
else: else:
self.write(safe_json_dumps(f"Error - sig '{sig}' is not known.")) self.write(safe_json_dumps(f"Error - sig '{sig}' is not known."))
@@ -120,7 +135,12 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
class APILookupGridHandler(tornado.web.RequestHandler): class APILookupGridHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/lookup/grid""" """API request handler for /api/v2/lookup/grid"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._web_server_metrics = None self._web_server_metrics = None
super().__init__(application, request, **kwargs) super().__init__(application, request, **kwargs)
@@ -140,7 +160,7 @@ class APILookupGridHandler(tornado.web.RequestHandler):
query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()} query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
# "grid" query param must exist. # "grid" query param must exist.
if "grid" in query_params.keys(): if "grid" in query_params:
grid = str(query_params.get("grid")).upper() grid = str(query_params.get("grid")).upper()
lat, lon, lat_cell_size, lon_cell_size = lat_lon_for_grid_sw_corner_plus_size(grid) lat, lon, lat_cell_size, lon_cell_size = lat_lon_for_grid_sw_corner_plus_size(grid)
if lat is not None and lon is not None and lat_cell_size is not None and lon_cell_size is not None: if lat is not None and lon is not None and lat_cell_size is not None and lon_cell_size is not None:
@@ -154,7 +174,7 @@ class APILookupGridHandler(tornado.web.RequestHandler):
"latitude": center_lat, "latitude": center_lat,
"longitude": center_lon, "longitude": center_lon,
"cq_zone": center_cq_zone, "cq_zone": center_cq_zone,
"itu_zone": center_itu_zone "itu_zone": center_itu_zone,
}, },
"southwest": { "southwest": {
"latitude": lat, "latitude": lat,
@@ -163,7 +183,8 @@ class APILookupGridHandler(tornado.web.RequestHandler):
"northeast": { "northeast": {
"latitude": lat + lat_cell_size, "latitude": lat + lat_cell_size,
"longitude": lon + lon_cell_size, "longitude": lon + lon_cell_size,
}} },
}
self.write(safe_json_dumps(response)) self.write(safe_json_dumps(response))
else: else:
server/handlers/api/options.py
+45 -9
View File
@@ -7,8 +7,15 @@ import tornado
from tornado import httputil from tornado import httputil
from tornado.web import Application from tornado.web import Application
from core.config import MAX_SPOT_AGE, ALLOW_SPOTTING from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
from core.constants import BANDS, ALL_MODES, MODE_TYPES, SIGS, CONTINENTS, PROPAGATION_MODES from core.constants import (
ALL_MODES,
BANDS,
CONTINENTS,
MODE_TYPES,
PROPAGATION_MODES,
SIGS,
)
from core.prometheus_metrics_handler import api_requests_counter from core.prometheus_metrics_handler import api_requests_counter
from core.utils import safe_json_dumps from core.utils import safe_json_dumps
@@ -16,7 +23,12 @@ from core.utils import safe_json_dumps
class APIOptionsHandler(tornado.web.RequestHandler): class APIOptionsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/options""" """API request handler for /api/v2/options"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._status_data = None self._status_data = None
self._web_server_metrics = None self._web_server_metrics = None
self._spot_providers = None self._spot_providers = None
@@ -49,13 +61,35 @@ class APIOptionsHandler(tornado.web.RequestHandler):
# Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle
# things that aren't even available. # things that aren't even available.
spot_providers: list = list( spot_providers: list = list(
map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["spot_providers"]))) map(
lambda p: p["name"],
filter(lambda p: p["enabled"], self._status_data["spot_providers"]),
)
)
alert_providers = list( alert_providers = list(
map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"]))) map(
lambda p: p["name"],
filter(lambda p: p["enabled"], self._status_data["alert_providers"]),
)
)
callsign_data_providers = list( callsign_data_providers = list(
map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["callsign_data_providers"]))) map(
lambda p: p["name"],
filter(
lambda p: p["enabled"],
self._status_data["callsign_data_providers"],
),
)
)
spot_providers_enabled_by_default = list( spot_providers_enabled_by_default = list(
map(lambda p: p["name"], filter(lambda p: p["enabled"] and p["enabled_by_default_in_web_ui"], self._status_data["spot_providers"]))) map(
lambda p: p["name"],
filter(
lambda p: p["enabled"] and p["enabled_by_default_in_web_ui"],
self._status_data["spot_providers"],
),
)
)
# If spotting to this server is enabled, "API" is another valid spot source even though it does not come from # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
# one of our providers. # one of our providers.
@@ -63,7 +97,8 @@ class APIOptionsHandler(tornado.web.RequestHandler):
spot_providers.append("API") spot_providers.append("API")
spot_providers_enabled_by_default.append("API") spot_providers_enabled_by_default.append("API")
options = {"bands": BANDS, options = {
"bands": BANDS,
"modes": ALL_MODES, "modes": ALL_MODES,
"mode_types": MODE_TYPES, "mode_types": MODE_TYPES,
"sigs": SIGS, "sigs": SIGS,
@@ -75,7 +110,8 @@ class APIOptionsHandler(tornado.web.RequestHandler):
"propagation_modes": list(PROPAGATION_MODES.values()), "propagation_modes": list(PROPAGATION_MODES.values()),
"max_spot_age": MAX_SPOT_AGE, "max_spot_age": MAX_SPOT_AGE,
"spot_allowed": ALLOW_SPOTTING, "spot_allowed": ALLOW_SPOTTING,
"spot_submit_providers": spot_submit_providers} "spot_submit_providers": spot_submit_providers,
}
self.write(safe_json_dumps(options)) self.write(safe_json_dumps(options))
self.set_status(200) self.set_status(200)
server/handlers/api/solar_conditions.py
+6 -1
View File
@@ -14,7 +14,12 @@ from core.utils import safe_json_dumps
class APISolarConditionsHandler(tornado.web.RequestHandler): class APISolarConditionsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/solar""" """API request handler for /api/v2/solar"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._solar_conditions = None self._solar_conditions = None
self._web_server_metrics = None self._web_server_metrics = None
super().__init__(application, request, **kwargs) super().__init__(application, request, **kwargs)
server/handlers/api/spots.py
+11 -6
View File
@@ -17,7 +17,12 @@ from data.lookup_credentials import extract_credentials
class APISpotsHandler(tornado.web.RequestHandler): class APISpotsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/spots""" """API request handler for /api/v2/spots"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._spots = None self._spots = None
self._web_server_metrics = None self._web_server_metrics = None
super().__init__(application, request, **kwargs) super().__init__(application, request, **kwargs)
@@ -82,8 +87,7 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
def custom_headers(self): def custom_headers(self):
"""Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data""" """Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data"""
return {"Cache-Control": "no-store", return {"Cache-Control": "no-store", "X-Accel-Buffering": "no"}
"X-Accel-Buffering": "no"}
def open(self): def open(self):
"""Called once on the client opening a connection, set things up""" """Called once on the client opening a connection, set things up"""
@@ -145,7 +149,7 @@ def get_spot_list_with_filters(all_spots, query):
s = all_spots.get(k) s = all_spots.get(k)
if s is not None: if s is not None:
spots.append(s) spots.append(s)
spots = sorted(spots, key=lambda spot: (spot.time if spot and spot.time else 0), reverse=True) spots = sorted(spots, key=lambda spot: spot.time if spot and spot.time else 0, reverse=True)
spots = list(filter(lambda spot: spot_allowed_by_query(spot, query), spots)) spots = list(filter(lambda spot: spot_allowed_by_query(spot, query), spots))
if "limit" in query.keys(): if "limit" in query.keys():
spots = spots[: int(query.get("limit"))] spots = spots[: int(query.get("limit"))]
@@ -245,8 +249,9 @@ def spot_allowed_by_query(spot, query):
return False return False
case "text_includes": case "text_includes":
text_includes = query.get(k).strip() text_includes = query.get(k).strip()
if (not spot.dx_call or text_includes.upper() not in spot.dx_call.upper()) \ if (not spot.dx_call or text_includes.upper() not in spot.dx_call.upper()) and (
and (not spot.comment or text_includes.upper() not in spot.comment.upper()): not spot.comment or text_includes.upper() not in spot.comment.upper()
):
return False return False
case "allow_qrt": case "allow_qrt":
# If false, spots that are flagged as QRT are not returned. # If false, spots that are flagged as QRT are not returned.
server/handlers/api/status.py
+6 -1
View File
@@ -14,7 +14,12 @@ from core.utils import safe_json_dumps
class APIStatusHandler(tornado.web.RequestHandler): class APIStatusHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/status""" """API request handler for /api/v2/status"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._status_data = None self._status_data = None
self._web_server_metrics = None self._web_server_metrics = None
super().__init__(application, request, **kwargs) super().__init__(application, request, **kwargs)
server/handlers/api/v1_addspot.py
+26 -9
View File
@@ -12,14 +12,19 @@ from core.config import ALLOW_SPOTTING
from core.constants import UNKNOWN_BAND from core.constants import UNKNOWN_BAND
from core.prometheus_metrics_handler import api_requests_counter from core.prometheus_metrics_handler import api_requests_counter
from core.sig_utils import get_ref_regex_for_sig from core.sig_utils import get_ref_regex_for_sig
from core.utils import safe_json_dumps, infer_band_from_freq from core.utils import infer_band_from_freq, safe_json_dumps
from data.spot import Spot from data.spot import Spot
class V1APISpotHandler(tornado.web.RequestHandler): class V1APISpotHandler(tornado.web.RequestHandler):
"""API request handler for /api/v1/spot (POST). Included in early Spothole v2 for backwards compatibility.""" """API request handler for /api/v1/spot (POST). Included in early Spothole v2 for backwards compatibility."""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any): def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._spots = None self._spots = None
self._web_server_metrics = None self._web_server_metrics = None
super().__init__(application, request, **kwargs) super().__init__(application, request, **kwargs)
@@ -45,7 +50,7 @@ class V1APISpotHandler(tornado.web.RequestHandler):
return return
# Reject if format not json # Reject if format not json
if not self.request.headers.get('Content-Type', '').startswith("application/json"): if not self.request.headers.get("Content-Type", "").startswith("application/json"):
self.set_status(415) self.set_status(415)
self.write(safe_json_dumps("Error - request Content-Type must be application/json")) self.write(safe_json_dumps("Error - request Content-Type must be application/json"))
self.set_header("Cache-Control", "no-store") self.set_header("Cache-Control", "no-store")
@@ -68,7 +73,9 @@ class V1APISpotHandler(tornado.web.RequestHandler):
# Reject if no timestamp, frequency, dx_call or de_call # Reject if no timestamp, frequency, dx_call or de_call
if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call: if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
self.set_status(422) self.set_status(422)
self.write(safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.")) self.write(
safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.")
)
self.set_header("Cache-Control", "no-store") self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json") self.set_header("Content-Type", "application/json")
return return
@@ -98,7 +105,8 @@ class V1APISpotHandler(tornado.web.RequestHandler):
# Reject if grid formatting incorrect # Reject if grid formatting incorrect
if spot.dx_grid and not re.match( if spot.dx_grid and not re.match(
r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$", r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
spot.dx_grid.upper()): spot.dx_grid.upper(),
):
self.set_status(422) self.set_status(422)
self.write(safe_json_dumps(f"Error - '{spot.dx_grid}' does not look like a valid Maidenhead grid.")) self.write(safe_json_dumps(f"Error - '{spot.dx_grid}' does not look like a valid Maidenhead grid."))
self.set_header("Cache-Control", "no-store") self.set_header("Cache-Control", "no-store")
@@ -106,11 +114,20 @@ class V1APISpotHandler(tornado.web.RequestHandler):
return return
# Reject if sig_ref format incorrect for sig # Reject if sig_ref format incorrect for sig
if spot.sig and spot.sig_refs and len(spot.sig_refs) > 0 and spot.sig_refs[0].id and get_ref_regex_for_sig( if (
spot.sig) and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id): spot.sig
and spot.sig_refs
and len(spot.sig_refs) > 0
and spot.sig_refs[0].id
and get_ref_regex_for_sig(spot.sig)
and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id)
):
self.set_status(422) self.set_status(422)
self.write(safe_json_dumps( self.write(
f"Error - '{spot.sig_refs[0].id}' does not look like a valid reference for {spot.sig}.")) safe_json_dumps(
f"Error - '{spot.sig_refs[0].id}' does not look like a valid reference for {spot.sig}."
)
)
self.set_header("Cache-Control", "no-store") self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json") self.set_header("Content-Type", "application/json")
return return

Some files were not shown because too many files have changed in this diff Show More