w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-09-20 14:27:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Use ruff linter to fix issues and provide consistent formatting

Browse Source
This commit is contained in:
Ian Renton
2026-08-15 08:25:54 +01:00
parent 7391c28cd0
commit af3f82c14d
121 changed files with 1989 additions and 996 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/spothole.iml → .idea/Spothole.iml
Generated
+2 -2
View File
@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<module external.system.id="pyproject.toml" type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$" />
<excludeFolder url="file://$MODULE_DIR$/.venv" />
<excludeFolder url="file://$MODULE_DIR$/static/vendor" />
</content>
.idea/codeStyles/codeStyleConfig.xml
Generated
+5
View File
@@ -0,0 +1,5 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
</state>
</component>
.idea/modules.xml
Generated
+1 -1
View File
@@ -2,7 +2,7 @@
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/spothole.iml" filepath="$PROJECT_DIR$/.idea/spothole.iml" />
<module fileurl="file://$PROJECT_DIR$/.idea/Spothole.iml" filepath="$PROJECT_DIR$/.idea/Spothole.iml" />
</modules>
</component>
</project>
.idea/runConfigurations/Run.xml
Generated
+3 -1
View File
@@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Run" type="PythonConfigurationType" factoryName="Python">
<module name="spothole" />
<module name="Spothole" />
<option name="ENV_FILES" value="" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
@@ -12,6 +12,8 @@
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<option name="DEBUG_JUST_MY_CODE" value="true" />
<option name="RUN_TOOL" value="" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/spothole.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
core/call_lookup_helper.py
+1 -1
View File
@@ -27,4 +27,4 @@ def get_call_info(callsign, lookup_credentials):
if callsign_data.fully_populated():
break
return callsign_data
return callsign_data
core/cleanup.py
+1 -1
View File
@@ -68,4 +68,4 @@ class CleanupTimer:
# Global object
CLEANUP_TIMER = CleanupTimer()
CLEANUP_TIMER = CleanupTimer()
core/config.py
+5 -2
View File
@@ -7,7 +7,8 @@ import yaml
# Check you have a config file
if not os.path.isfile("config.yml"):
logging.error(
"Your config file is missing. Ensure you have copied config-example.yml to config.yml and updated it according to your needs.")
"Your config file is missing. Ensure you have copied config-example.yml to config.yml and updated it according to your needs."
)
exit()
# Load config
@@ -30,7 +31,9 @@ LOG_LEVEL = config.get("log_level", "INFO")
LOG_WEB_REQUESTS = config.get("log_web_requests", False)
WEB_UI_OPTIONS["qrz_enabled"] = any(p["class"] == "QRZ" and p["enabled"] for p in config["callsign_data_providers"])
WEB_UI_OPTIONS["hamqth_enabled"] = any(p["class"] == "HamQTH" and p["enabled"] for p in config["callsign_data_providers"])
WEB_UI_OPTIONS["hamqth_enabled"] = any(
p["class"] == "HamQTH" and p["enabled"] for p in config["callsign_data_providers"]
)
WEB_UI_OPTIONS["recaptcha_site_key"] = RECAPTCHA_SITE_KEY
WEB_UI_OPTIONS["allow_upstream_spotting"] = ALLOW_SPOTTING and ALLOW_UPSTREAM_SPOTTING
core/constants.py
+169 -28
View File
@@ -11,36 +11,176 @@ HAMQTH_PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}"
# Special Interest Groups
SIGS = [
SIG(name="POTA", comment_names=["POTA"], description="Parks on the Air", ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST"),
SIG(name="SOTA", comment_names=["SOTA"], description="Summits on the Air", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}"),
SIG(name="WWFF", comment_names=["WWFF"], description="World Wide Flora & Fauna", ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}"),
SIG(name="GMA", comment_names=["GMA"], description="Global Mountain Activity", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}"),
SIG(name="WWBOTA", comment_names=["WWBOTA", "BOTA"], description="Worldwide Bunkers on the Air", ref_regex=r"B\/[A-Z0-9]{1,3}\-\d{3,4}"),
SIG(name="HEMA", comment_names=["HEMA"], description="HuMPs Excluding Marilyns Award", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}"),
SIG(name="IOTA", comment_names=["IOTA"], description="Islands on the Air", ref_regex=r"[A-Z]{2}\-\d{3}"),
SIG(name="MOTA", comment_names=["MOTA"], description="Mills on the Air", ref_regex=r"X\d{4,6}"),
SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}[\- ]\d{3,4}"),
SIG(name="ILLW", comment_names=["ILLW"], description="International Lighthouse & Lightship Weekend", ref_regex=r"[A-Z]{2}\d{4}"),
SIG(name="SIOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
SIG(name="WCA", comment_names=["WCA"], description="World Castles Award", ref_regex=r"[A-Z0-9]{1,3}\-\d{5}"),
SIG(name="ZLOTA", comment_names=["ZLOTA"], description="New Zealand on the Air", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}"),
SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"),
SIG(
name="POTA",
comment_names=["POTA"],
description="Parks on the Air",
ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST",
),
SIG(
name="SOTA",
comment_names=["SOTA"],
description="Summits on the Air",
ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
),
SIG(
name="WWFF",
comment_names=["WWFF"],
description="World Wide Flora & Fauna",
ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}",
),
SIG(
name="GMA",
comment_names=["GMA"],
description="Global Mountain Activity",
ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
),
SIG(
name="WWBOTA",
comment_names=["WWBOTA", "BOTA"],
description="Worldwide Bunkers on the Air",
ref_regex=r"B\/[A-Z0-9]{1,3}\-\d{3,4}",
),
SIG(
name="HEMA",
comment_names=["HEMA"],
description="HuMPs Excluding Marilyns Award",
ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}",
),
SIG(
name="IOTA",
comment_names=["IOTA"],
description="Islands on the Air",
ref_regex=r"[A-Z]{2}\-\d{3}",
),
SIG(
name="MOTA",
comment_names=["MOTA"],
description="Mills on the Air",
ref_regex=r"X\d{4,6}",
),
SIG(
name="ARLHS",
comment_names=["ARLHS"],
description="Amateur Radio Lighthouse Society",
ref_regex=r"[A-Z]{3}[\- ]\d{3,4}",
),
SIG(
name="ILLW",
comment_names=["ILLW"],
description="International Lighthouse & Lightship Weekend",
ref_regex=r"[A-Z]{2}\d{4}",
),
SIG(
name="SIOTA",
comment_names=["SIOTA"],
description="Silos on the Air",
ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d",
),
SIG(
name="WCA",
comment_names=["WCA"],
description="World Castles Award",
ref_regex=r"[A-Z0-9]{1,3}\-\d{5}",
),
SIG(
name="ZLOTA",
comment_names=["ZLOTA"],
description="New Zealand on the Air",
ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}",
),
SIG(
name="WOTA",
comment_names=["WOTA"],
description="Wainwrights on the Air",
ref_regex=r"[A-Z]{3}-[0-9]{2}",
),
SIG(name="BOTA", comment_names=[], description="Beaches on the Air"),
SIG(name="KRMNPA", comment_names=["KRMNPA"], description="Keith Roget Memorial National Parks Award", ref_regex=r"VKFF\-\d{4}"),
SIG(name="SANPCPA", comment_names=["SANPCPA"], description="South Australian National Parks and Conservation Parks Award", ref_regex=r"VKFF\-\d{4}"),
SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"LL[A-Z]{2}\-\d{4}"),
SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2,3}R\-\d{4}"),
SIG(name="Tiles", comment_names=[], description="Tiles on the Air", ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}"),
SIG(name="WAB", comment_names=["WAB"], description="Worked All Britain", ref_regex=r"[A-Z]{1,2}[0-9]{2}"),
SIG(name="WAI", comment_names=["WAI"], description="Worked All Ireland", ref_regex=r"[A-Z][0-9]{2}"),
SIG(name="DME", comment_names=["DME"], description="Diplomas de Municipios Españoles", ref_regex=r"\d{4,5}"),
SIG(name="Toilets", comment_names=[], description="Toilets on the Air", ref_regex=r"T\-[0-9]{2}")
SIG(
name="KRMNPA",
comment_names=["KRMNPA"],
description="Keith Roget Memorial National Parks Award",
ref_regex=r"VKFF\-\d{4}",
),
SIG(
name="SANPCPA",
comment_names=["SANPCPA"],
description="South Australian National Parks and Conservation Parks Award",
ref_regex=r"VKFF\-\d{4}",
),
SIG(
name="LLOTA",
comment_names=["LLOTA"],
description="Lagos y Lagunas on the Air",
ref_regex=r"LL[A-Z]{2}\-\d{4}",
),
SIG(
name="Towers",
comment_names=["TOTA"],
description="Towers on the Air",
ref_regex=r"[A-Z]{2,3}R\-\d{4}",
),
SIG(
name="Tiles",
comment_names=[],
description="Tiles on the Air",
ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}",
),
SIG(
name="WAB",
comment_names=["WAB"],
description="Worked All Britain",
ref_regex=r"[A-Z]{1,2}[0-9]{2}",
),
SIG(
name="WAI",
comment_names=["WAI"],
description="Worked All Ireland",
ref_regex=r"[A-Z][0-9]{2}",
),
SIG(
name="DME",
comment_names=["DME"],
description="Diplomas de Municipios Españoles",
ref_regex=r"\d{4,5}",
),
SIG(
name="Toilets",
comment_names=[],
description="Toilets on the Air",
ref_regex=r"T\-[0-9]{2}",
),
]
# Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
CW_MODES = ["CW"]
PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "FUSION", "M17"]
DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "PSK", "OLIVIA", "PKT", "MSK144"]
PHONE_MODES = [
"PHONE",
"SSB",
"USB",
"LSB",
"AM",
"FM",
"DV",
"DMR",
"DSTAR",
"C4FM",
"FUSION",
"M17",
]
DATA_MODES = [
"DATA",
"FT8",
"FT4",
"RTTY",
"SSTV",
"JS8",
"HELL",
"PSK",
"OLIVIA",
"PKT",
"MSK144",
]
ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
MODE_TYPES = ["CW", "PHONE", "DATA"]
SSB_SUB_MODES = ["USB", "LSB"]
@@ -58,7 +198,7 @@ MODE_ALIASES = {
"MFSK": "FSK",
"MFSK32": "FSK",
"DIGI": "DATA",
"DIGITAL": "DATA"
"DIGITAL": "DATA",
}
# Band definitions
@@ -88,7 +228,8 @@ BANDS = [
Band(name="10GHz", start_freq=10000000000, end_freq=10500000000),
Band(name="24GHz", start_freq=24000000000, end_freq=24050000000),
Band(name="47GHz", start_freq=47000000000, end_freq=47200000000),
Band(name="76GHz", start_freq=75500000000, end_freq=81500000000)]
Band(name="76GHz", start_freq=75500000000, end_freq=81500000000),
]
UNKNOWN_BAND = Band(name="Unknown", start_freq=0, end_freq=0)
# Continents
@@ -106,5 +247,5 @@ PROPAGATION_MODES = {
"MS": "Meteor scatter",
"RS": "Rain scatter",
"AS": "Aircraft scatter",
"SAT": "Satellite"
"SAT": "Satellite",
}
core/data_providers.py
+15 -6
View File
@@ -15,7 +15,6 @@ class DataProviders:
self.sig_ref_data_providers = []
self.callsign_data_providers = []
def setup(self):
for entry in config["spot_providers"]:
self.spot_providers.append(create_provider_from_config("providers.spot", entry))
@@ -34,7 +33,7 @@ class DataProviders:
def start_providers(providers, provider_type):
"""Helper method to activate enabled providers in the list."""
logging.info(f"Starting %s providers...", provider_type)
logging.info(f"Starting {provider_type} providers...")
for p in providers:
if p.enabled:
p.start()
@@ -43,11 +42,20 @@ class DataProviders:
# Start data providers before spot/alert providers so the lookup data is there already for incoming spots.
# Each category is fired off after a small delay to give the rest of Spothole chance to start up.
threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")).start()
threading.Timer(10.0, lambda: self.start_providers(self.callsign_data_providers, "callsign data")).start()
threading.Timer(
10.0,
lambda: self.start_providers(self.callsign_data_providers, "callsign data"),
).start()
threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")).start()
threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")).start()
threading.Timer(25.0, lambda: self.start_providers(self.solar_condition_providers, "solar condition")).start()
threading.Timer(30.0, lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data")).start()
threading.Timer(
25.0,
lambda: self.start_providers(self.solar_condition_providers, "solar condition"),
).start()
threading.Timer(
30.0,
lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data"),
).start()
def stop(self):
for sp in self.spot_providers:
@@ -69,5 +77,6 @@ class DataProviders:
if cdp.enabled:
cdp.stop()
# Global object
DATA_PROVIDERS = DataProviders()
DATA_PROVIDERS = DataProviders()
core/data_store.py
+26 -14
View File
@@ -4,7 +4,7 @@ from pathlib import Path
import diskcache
from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
from core.config import MAX_ALERT_AGE, MAX_SPOT_AGE
from core.live_data_cache import LiveDataCache
from data.solar_conditions import SolarConditions
@@ -64,7 +64,7 @@ class DataStore:
# of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
# syntax for keys to keep it a single level.
self.sigrefs = diskcache.Cache(f"{CACHE_DIR}sigrefs")
logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
logging.info(f"Loaded data for {len(self.sigrefs)} SIG references.")
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
@@ -75,23 +75,34 @@ class DataStore:
self.callsign_data_qrz = diskcache.Cache(f"{CACHE_DIR}callsign_data_qrz")
self.callsign_data_hamqth = diskcache.Cache(f"{CACHE_DIR}callsign_data_hamqth")
unique_keys = set()
for c in [self.callsign_data_countryfiles, self.callsign_data_clublogxml, self.callsign_data_clublogapi,
self.callsign_data_qrz, self.callsign_data_hamqth]:
for c in [
self.callsign_data_countryfiles,
self.callsign_data_clublogxml,
self.callsign_data_clublogapi,
self.callsign_data_qrz,
self.callsign_data_hamqth,
]:
unique_keys.update(c)
logging.info(f"Loaded data for %d callsigns.", len(unique_keys))
logging.info(f"Loaded data for {len(unique_keys)} callsigns.")
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
# specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
# before the spots and alerts are live in the system.
self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
snapshot_dir=f"{CACHE_DIR}spots",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys()))
self.spots = LiveDataCache(
maxsize=self._MAX_SPOT_COUNT,
ttl=MAX_SPOT_AGE,
snapshot_dir=f"{CACHE_DIR}spots",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC,
)
logging.info(f"Loaded {len(self.spots.keys())} spots from a previous run.")
self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
snapshot_dir=f"{CACHE_DIR}alerts",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
self.alerts = LiveDataCache(
maxsize=self._MAX_ALERT_COUNT,
ttl=MAX_ALERT_AGE,
snapshot_dir=f"{CACHE_DIR}alerts",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC,
)
logging.info(f"Loaded {len(self.alerts.keys())} alerts from a previous run.")
def regenerate_call_regex_to_dxcc_entity_map(self):
"""DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC
@@ -118,5 +129,6 @@ class DataStore:
self.callsign_data_qrz.close()
self.callsign_data_hamqth.close()
# Global object
DATA_STORE = DataStore()
DATA_STORE = DataStore()
core/geo_utils.py
+2 -2
View File
@@ -113,8 +113,8 @@ def lat_lon_for_grid_sw_corner_plus_size(grid):
while block * 2 < length:
if block % 2 == 0:
# Letters in this block
lon_cell_no = ord(grid[block * 2]) - ord('A')
lat_cell_no = ord(grid[block * 2 + 1]) - ord('A')
lon_cell_no = ord(grid[block * 2]) - ord("A")
lat_cell_no = ord(grid[block * 2 + 1]) - ord("A")
# Bail if the values aren't in range. Allowed values are A-R (0-17) for the first letter block, or
# A-X (0-23) thereafter.
max_cell_no = 17 if block == 0 else 23
core/live_data_cache.py
+3 -4
View File
@@ -34,8 +34,7 @@ class LiveDataCache:
try:
callback(value)
except Exception:
logging.exception("Listener raised an exception for key %s", key)
logging.exception(f"Listener raised an exception for key {key}")
def get(self, key, default=None):
with self._lock:
@@ -71,7 +70,7 @@ class LiveDataCache:
try:
self._disk_cache.set("snapshot", data)
except Exception as e:
logging.exception("Failed to write snapshot to %s", self._snapshot_dir, e)
logging.exception(f"Failed to write snapshot to {self._snapshot_dir}")
def _load_snapshot(self):
data = self._disk_cache.get("snapshot")
@@ -84,7 +83,7 @@ class LiveDataCache:
# Only restore entries that would still be within TTL
if now - saved_at < self._ttl:
self._cache[key] = value
logging.info("Loaded snapshot from %s", self._snapshot_dir)
logging.info(f"Loaded snapshot from {self._snapshot_dir}")
def _start_periodic_snapshot(self, interval):
def loop():
core/prometheus_metrics_handler.py
+11 -17
View File
@@ -1,4 +1,10 @@
from prometheus_client import CollectorRegistry, generate_latest, Counter, disable_created_metrics, Gauge
from prometheus_client import (
CollectorRegistry,
Counter,
Gauge,
disable_created_metrics,
generate_latest,
)
disable_created_metrics()
# Prometheus metrics registry
@@ -9,25 +15,13 @@ page_requests_counter = Counter(
"Total number of page requests received",
registry=registry,
)
api_requests_counter = Counter(
"spothole_api_requests",
"Total number of API requests received",
registry=registry
)
spots_gauge = Gauge(
"spothole_spots",
"Number of spots currently in the software",
registry=registry
)
alerts_gauge = Gauge(
"spothole_alerts",
"Number of alerts currently in the software",
registry=registry
)
api_requests_counter = Counter("spothole_api_requests", "Total number of API requests received", registry=registry)
spots_gauge = Gauge("spothole_spots", "Number of spots currently in the software", registry=registry)
alerts_gauge = Gauge("spothole_alerts", "Number of alerts currently in the software", registry=registry)
memory_use_gauge = Gauge(
"spothole_memory_usage_bytes",
"Current memory usage of the software in bytes",
registry=registry
registry=registry,
)
core/sig_lookup_helper.py
+2 -2
View File
@@ -1,6 +1,6 @@
import logging
from pyhamtools.locator import locator_to_latlong, latlong_to_locator
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from core.data_store import DATA_STORE
from core.geo_utils import wab_wai_square_to_lat_lon
@@ -83,7 +83,7 @@ def get_sig_ref_info(sig, ref_id):
else:
# Maybe a super new reference we don't know about yet, but more likely a typo or a test reference,
# just silently ignore it.
logging.debug("%s database did not contain data for ref %s", sig, ref_id)
logging.debug(f"{sig} database did not contain data for ref {ref_id}")
except Exception:
logging.exception(f"Exception when looking up sig_ref info for {sig} ref {ref_id}")
core/sig_utils.py
+1 -1
View File
@@ -21,4 +21,4 @@ def get_sig_name_from_comment_name(sig):
# Regex matching any SIG's "comment name", i.e. how it may be referred to in spot comments
ANY_SIG_REGEX = rf"({'|'.join((n for s in SIGS for n in s.comment_names))})"
ANY_SIG_REGEX = rf"({'|'.join(n for s in SIGS for n in s.comment_names)})"
core/status_reporter.py
+101 -46
View File
@@ -1,6 +1,6 @@
import os
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import psutil
import pytz
@@ -10,7 +10,7 @@ from core.config import SERVER_OWNER_CALLSIGN
from core.constants import SOFTWARE_VERSION
from core.data_providers import DATA_PROVIDERS
from core.data_store import DATA_STORE
from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge
from core.prometheus_metrics_handler import alerts_gauge, memory_use_gauge, spots_gauge
from server.webserver import WEB_SERVER
@@ -55,55 +55,110 @@ class StatusReporter:
DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
DATA_STORE.status_data["spot_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled,
"enabled_by_default_in_web_ui": p.enabled_by_default_in_web_ui, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
"last_spot": p.last_spot_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
DATA_PROVIDERS.spot_providers))
map(
lambda p: {
"name": p.name,
"enabled": p.enabled,
"enabled_by_default_in_web_ui": p.enabled_by_default_in_web_ui,
"status": p.status,
"last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
"last_spot": p.last_spot_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_spot_time.year > 2000
else 0,
},
DATA_PROVIDERS.spot_providers,
)
)
DATA_STORE.status_data["alert_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
DATA_PROVIDERS.alert_providers))
map(
lambda p: {
"name": p.name,
"enabled": p.enabled,
"status": p.status,
"last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
},
DATA_PROVIDERS.alert_providers,
)
)
DATA_STORE.status_data["solar_condition_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
DATA_PROVIDERS.solar_condition_providers))
map(
lambda p: {
"name": p.name,
"enabled": p.enabled,
"status": p.status,
"last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
},
DATA_PROVIDERS.solar_condition_providers,
)
)
DATA_STORE.status_data["static_data_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
DATA_PROVIDERS.static_data_providers))
map(
lambda p: {
"name": p.name,
"enabled": p.enabled,
"status": p.status,
"last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
},
DATA_PROVIDERS.static_data_providers,
)
)
DATA_STORE.status_data["sig_ref_data_providers"] = list(
map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
"reference_count": p.reference_count},
DATA_PROVIDERS.sig_ref_data_providers))
map(
lambda p: {
"sig_name": p.sig_name,
"enabled": p.enabled,
"status": p.status,
"last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
"reference_count": p.reference_count,
},
DATA_PROVIDERS.sig_ref_data_providers,
)
)
DATA_STORE.status_data["callsign_data_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
"lookup_count": p.lookup_count},
DATA_PROVIDERS.callsign_data_providers))
DATA_STORE.status_data["cleanup"] = {"status": CLEANUP_TIMER.status,
"last_ran": CLEANUP_TIMER.last_cleanup_time.replace(
tzinfo=pytz.UTC).timestamp() if CLEANUP_TIMER.last_cleanup_time else 0}
DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"],
"last_api_access": WEB_SERVER.web_server_metrics[
"last_api_access_time"].replace(
tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[
"last_api_access_time"] else 0,
"api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"],
"last_page_access": WEB_SERVER.web_server_metrics[
"last_page_access_time"].replace(
tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[
"last_page_access_time"] else 0,
"page_access_count": WEB_SERVER.web_server_metrics[
"page_access_counter"]}
map(
lambda p: {
"name": p.name,
"enabled": p.enabled,
"status": p.status,
"last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
if p.last_update_time.year > 2000
else 0,
"lookup_count": p.lookup_count,
},
DATA_PROVIDERS.callsign_data_providers,
)
)
DATA_STORE.status_data["cleanup"] = {
"status": CLEANUP_TIMER.status,
"last_ran": CLEANUP_TIMER.last_cleanup_time.replace(tzinfo=pytz.UTC).timestamp()
if CLEANUP_TIMER.last_cleanup_time
else 0,
}
DATA_STORE.status_data["webserver"] = {
"status": WEB_SERVER.web_server_metrics["status"],
"last_api_access": WEB_SERVER.web_server_metrics["last_api_access_time"]
.replace(tzinfo=pytz.UTC)
.timestamp()
if WEB_SERVER.web_server_metrics["last_api_access_time"]
else 0,
"api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"],
"last_page_access": WEB_SERVER.web_server_metrics["last_page_access_time"]
.replace(tzinfo=pytz.UTC)
.timestamp()
if WEB_SERVER.web_server_metrics["last_page_access_time"]
else 0,
"page_access_count": WEB_SERVER.web_server_metrics["page_access_counter"],
}
# Update Prometheus metrics
memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)
core/url_data_cache.py
+6 -3
View File
@@ -17,9 +17,12 @@ class URLDataCache(CachedSession):
_lock = threading.Lock()
def __init__(self, name):
super().__init__(f"{CACHE_DIR}urls/{name}", expire_after=timedelta(days=1),
allowable_codes=(200, 400, 401, 403, 404))
super().__init__(
f"{CACHE_DIR}urls/{name}",
expire_after=timedelta(days=1),
allowable_codes=(200, 400, 401, 403, 404),
)
def get(self, *args, **kwargs):
with self._lock:
return super().get(*args, **kwargs)
return super().get(*args, **kwargs)
core/utils.py
+40 -16
View File
@@ -4,7 +4,15 @@ import simplejson
from pyhamtools.frequency import freq_to_band
from pyhamtools.locator import latlong_to_locator
from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES
from core.constants import (
ALL_MODES,
BANDS,
CW_MODES,
DATA_MODES,
MODE_ALIASES,
PHONE_MODES,
UNKNOWN_BAND,
)
from core.data_store import DATA_STORE
from data.callsign import Callsign
@@ -63,11 +71,25 @@ def infer_mode_from_frequency(freq):
# a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
# might include other common digimode centres of activity here, but this achieves the main goal of keeping
# large numbers of clearly-FT* spots off the list of people filtering out digimodes.
if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or (
21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077):
if (
(7074 <= khz < 7077)
or (10136 <= khz < 10139)
or (14074 <= khz < 14077)
or (18100 <= khz < 18103)
or (21074 <= khz < 21077)
or (24915 <= khz < 24918)
or (28074 <= khz < 28077)
):
mode = "FT8"
if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or (
18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183):
if (
(7047.5 <= khz < 7050.5)
or (10140 <= khz < 10143)
or (14080 <= khz < 14083)
or (18104 <= khz < 18107)
or (21140 <= khz < 21143)
or (24919 <= khz < 24922)
or (28180 <= khz < 28183)
):
mode = "FT4"
return mode
except KeyError:
@@ -100,17 +122,19 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
if lat and lon:
grid = latlong_to_locator(lat, lon)
return Callsign(call=callsign,
home_call=home_call,
country=country,
dxcc_id=dxcc_id,
continent=continent,
cq_zone=cq_zone,
itu_zone=itu_zone,
latitude=lat,
longitude=lon,
grid=grid,
location_source="DXCC")
return Callsign(
call=callsign,
home_call=home_call,
country=country,
dxcc_id=dxcc_id,
continent=continent,
cq_zone=cq_zone,
itu_zone=itu_zone,
latitude=lat,
longitude=lon,
grid=grid,
location_source="DXCC",
)
except (KeyError, ValueError):
# Unknown callsign, can't look anything up, return a Callsign object with basic data so that gets cached
data/alert.py
+8 -5
View File
@@ -120,12 +120,12 @@ class Alert:
self.id = hashlib.sha256(str({"s": self.source, "sid": self.source_id}).encode("utf-8")).hexdigest()
else:
self.id = hashlib.sha256(
str({"s": self.source, "c": self.dx_calls, "t": self.start_time}).encode("utf-8")).hexdigest()
str({"s": self.source, "c": self.dx_calls, "t": self.start_time}).encode("utf-8")
).hexdigest()
# DX operator name lookup, using QRZ.com/HamQTH.
if self.dx_calls and not self.dx_names:
self.dx_names = list(
map(lambda c: get_call_info(c, credentials).name, self.dx_calls))
self.dx_names = list(map(lambda c: get_call_info(c, credentials).name, self.dx_calls))
except Exception:
logging.exception("Exception while inferring missing data from spot")
@@ -141,5 +141,8 @@ class Alert:
either having an end_time in the past, or if it only has a start_time, then that start time was more than 3 hours
ago. If it somehow doesn't have a start_time either, it is considered to be expired."""
return not self.start_time or (self.end_time and self.end_time < datetime.now(pytz.UTC).timestamp()) or (
not self.end_time and self.start_time < (datetime.now(pytz.UTC) - timedelta(hours=3)).timestamp())
return (
not self.start_time
or (self.end_time and self.end_time < datetime.now(pytz.UTC).timestamp())
or (not self.end_time and self.start_time < (datetime.now(pytz.UTC) - timedelta(hours=3)).timestamp())
)
data/callsign.py
+18 -8
View File
@@ -13,18 +13,18 @@ class Callsign:
# "Home" call, i.e. with any prefixes and suffixes stripped off
home_call: str | None = None
# Operator name
name : str | None = None
name: str | None = None
# QTH (location), free text
qth : str | None = None
qth: str | None = None
# Maidenhead grid locator. Depending on the source of lookup this could be a home location from QRZ/HamQTH or just
# the centre of the country they're operating in if no other data is available.
grid : str | None = None
grid: str | None = None
# Latitude. Depending on the source of lookup this could be a home location from QRZ/HamQTH or just
# the centre of the country they're operating in if no other data is available.
latitude : float | None = None
latitude: float | None = None
# Longitude. Depending on the source of lookup this could be a home location from QRZ/HamQTH or just
# the centre of the country they're operating in if no other data is available.
longitude : float | None = None
longitude: float | None = None
# Country in which the callsign indicates they are operating
country: str | None = None
# Continent in which the callsign indicates they are operating
@@ -42,6 +42,16 @@ class Callsign:
"""Utility method to indicate that the callsign data is fully populated. Multiple providers can return data for
a callsign, and we try them in sequence until we have all the data we can, in which case there's no point
querying any other providers."""
return self.home_call is not None and self.name is not None and self.qth is not None and self.grid is not None\
and self.latitude is not None and self.longitude is not None and self.country is not None and self.continent\
is not None and self.dxcc_id is not None and self.cq_zone is not None and self.itu_zone is not None
return (
self.home_call is not None
and self.name is not None
and self.qth is not None
and self.grid is not None
and self.latitude is not None
and self.longitude is not None
and self.country is not None
and self.continent is not None
and self.dxcc_id is not None
and self.cq_zone is not None
and self.itu_zone is not None
)
data/lookup_credentials.py
+1
View File
@@ -4,6 +4,7 @@ from dataclasses import dataclass
@dataclass
class LookupCredentials:
"""Per-request credentials for QRZ.com and HamQTH online callsign lookups."""
qrz_username: str = ""
qrz_password: str = ""
qrz_session_key: str = "" # alternative to username/password
data/sig.py
+6 -6
View File
@@ -4,12 +4,12 @@ from dataclasses import dataclass, field
@dataclass
class SIG:
"""Data class that defines a Special Interest Group. Each contains a name and a longer form description.
They also contain comment_names which attempts to separate out the way people might refer to it in
cluster comments from how it is referred to in the UI & API. (For example, "TOTA" in cluster spot comments
almost always means Towers on the Air, but no single programme is referred to in the UI as "TOTA" as
it's ambiguous between Towers, Toilets and Tiles. And while Beaches got the name "BOTA" first, "BOTA" spots
are much more likely to be bunkers.) Finally, there is a ref_regex which provides a regular expression to
match what references (such as parks and summits) look like for that programme."""
They also contain comment_names which attempts to separate out the way people might refer to it in
cluster comments from how it is referred to in the UI & API. (For example, "TOTA" in cluster spot comments
almost always means Towers on the Air, but no single programme is referred to in the UI as "TOTA" as
it's ambiguous between Towers, Toilets and Tiles. And while Beaches got the name "BOTA" first, "BOTA" spots
are much more likely to be bunkers.) Finally, there is a ref_regex which provides a regular expression to
match what references (such as parks and summits) look like for that programme."""
# SIG name as used in the UI and API, e.g. "Towers"
name: str
data/solar_conditions.py
+2 -2
View File
@@ -82,9 +82,9 @@ def _xray_blackout_scale(xray):
number = float(xray[1:])
except ValueError:
return 0
if letter == 'M':
if letter == "M":
return 1 if number < 5 else 2
if letter == 'X':
if letter == "X":
if number < 10:
return 3
if number < 20:
data/spot.py
+47 -25
View File
@@ -1,5 +1,4 @@
import hashlib
import hashlib
import json
import logging
import re
@@ -7,16 +6,25 @@ from dataclasses import dataclass
from datetime import datetime, timedelta
import pytz
from pyhamtools.locator import locator_to_latlong, latlong_to_locator
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from core.call_lookup_helper import get_call_info
from core.config import MAX_SPOT_AGE
from core.constants import MODE_ALIASES, PROPAGATION_MODES
from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
from core.sig_lookup_helper import populate_missing_sig_ref_info
from core.sig_utils import ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
from core.utils import infer_band_from_freq, infer_mode_from_comment, \
infer_mode_from_frequency, infer_mode_type_from_mode, get_flag_for_dxcc
from core.sig_utils import (
ANY_SIG_REGEX,
get_ref_regex_for_sig,
get_sig_name_from_comment_name,
)
from core.utils import (
get_flag_for_dxcc,
infer_band_from_freq,
infer_mode_from_comment,
infer_mode_from_frequency,
infer_mode_type_from_mode,
)
from data.sig_ref import SIGRef
@@ -141,10 +149,7 @@ class Spot:
objects such as the sig_refs list.."""
if self.sig_refs:
self.sig_refs = [
sig_ref if isinstance(sig_ref, SIGRef) else SIGRef(**sig_ref)
for sig_ref in self.sig_refs
]
self.sig_refs = [sig_ref if isinstance(sig_ref, SIGRef) else SIGRef(**sig_ref) for sig_ref in self.sig_refs]
def infer_missing(self, credentials=None):
"""Infer missing parameters where possible"""
@@ -207,8 +212,11 @@ class Spot:
# Spotter country, continent, zones etc. from callsign.
# DE call with no digits, or APRS servers starting "T2" are not things we can look up location for
de_call_info = get_call_info(self.de_call, credentials)
if self.de_call and any(char.isdigit() for char in self.de_call) and not (
self.de_call.startswith("T2") and self.source == "APRS-IS"):
if (
self.de_call
and any(char.isdigit() for char in self.de_call)
and not (self.de_call.startswith("T2") and self.source == "APRS-IS")
):
if not self.de_country:
self.de_country = de_call_info.country
if not self.de_continent:
@@ -279,9 +287,11 @@ class Spot:
# If so, add that to the sig_refs list for this spot.
ref_regex = get_ref_regex_for_sig(found_sig)
if ref_regex:
ref_matches = re.finditer(r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)",
self.comment,
re.IGNORECASE)
ref_matches = re.finditer(
r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)",
self.comment,
re.IGNORECASE,
)
for ref_match in ref_matches:
self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig))
@@ -312,8 +322,9 @@ class Spot:
# being the only source we have for propagation mode. Brace for nightmare regex from hell.
if self.comment:
grid_mode_grid_match = re.search(
r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
self.comment)
r"\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b",
self.comment,
)
if grid_mode_grid_match:
# regex matches, so extract grids:
if not self.de_grid:
@@ -329,13 +340,14 @@ class Spot:
self.propagation_mode = PROPAGATION_MODES[mode_tag]
else:
self.propagation_mode = mode_tag
logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag)
logging.info(f"Seen a new propagation mode tag not yet in the system: {mode_tag}")
# Parse "de_grid -> dx_grid" structures from the comment
if self.comment:
grid_mode_grid_match = re.search(
r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
self.comment)
r"\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b",
self.comment,
)
if grid_mode_grid_match:
# regex matches, so extract grids:
if not self.dx_grid:
@@ -370,7 +382,8 @@ class Spot:
self.id = hashlib.sha256(str({"s": self.source, "sid": self.source_id}).encode("utf-8")).hexdigest()
else:
self.id = hashlib.sha256(
str({"s": self.source, "c": self.dx_call, "t": self.time}).encode("utf-8")).hexdigest()
str({"s": self.source, "c": self.dx_call, "t": self.time}).encode("utf-8")
).hexdigest()
# DX operator details lookup. This should be the last resort compared to taking the data from the actual
# spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon or DXCC lat/lon
@@ -407,14 +420,23 @@ class Spot:
# DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
# is likely at home.
self.dx_location_good = bool(self.dx_latitude and self.dx_longitude and (
self.dx_location_source == "SPOT" or self.dx_location_source == "SIG REF LOOKUP"
self.dx_location_good = bool(
self.dx_latitude
and self.dx_longitude
and (
self.dx_location_source == "SPOT"
or self.dx_location_source == "SIG REF LOOKUP"
or self.dx_location_source == "GRID"
or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or ""))))
or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or ""))
)
)
# DE with no digits and APRS servers starting "T2" are not things we can look up location for
if self.de_call and any(char.isdigit() for char in str(self.de_call)) and not (
self.de_call.startswith("T2") and self.source == "APRS-IS"):
if (
self.de_call
and any(char.isdigit() for char in str(self.de_call))
and not (self.de_call.startswith("T2") and self.source == "APRS-IS")
):
# DE operator location lookup
if not self.de_latitude:
self.de_latitude = de_call_info.latitude
providers/alert/alert_provider.py
+1 -1
View File
@@ -29,7 +29,7 @@ class AlertProvider:
# Sort the batch so that earliest ones go in first. This helps keep the ordering correct when alerts are fired
# off to SSE listeners.
alerts = sorted(alerts, key=lambda a: (a.start_time if a and a.start_time else 0))
alerts = sorted(alerts, key=lambda a: a.start_time if a and a.start_time else 0)
for alert in alerts:
# Fill in any blanks and add to the list
alert.infer_missing()
providers/alert/bota.py
+16 -14
View File
@@ -3,9 +3,9 @@ from datetime import datetime, timedelta
import pytz
from bs4 import BeautifulSoup
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class BOTA(HTTPAlertProvider):
@@ -23,17 +23,17 @@ class BOTA(HTTPAlertProvider):
bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
if not bs.body:
return new_alerts
div = bs.body.find('div', attrs={'class': 'view-activations-public'})
div = bs.body.find("div", attrs={"class": "view-activations-public"})
if div:
table = div.find('table', attrs={'class': 'views-table'})
table = div.find("table", attrs={"class": "views-table"})
if table:
tbody = table.find('tbody')
tbody = table.find("tbody")
if not tbody:
return new_alerts
for row in tbody.find_all('tr'):
cells = row.find_all('td')
first_cell_anchor = cells[0].find('a') if len(cells) > 0 else None
second_cell_anchor = cells[1].find('a') if len(cells) > 1 else None
for row in tbody.find_all("tr"):
cells = row.find_all("td")
first_cell_anchor = cells[0].find("a") if len(cells) > 0 else None
second_cell_anchor = cells[1].find("a") if len(cells) > 1 else None
if not first_cell_anchor or not second_cell_anchor:
continue
first_cell_text = first_cell_anchor.get_text().strip()
@@ -41,7 +41,7 @@ class BOTA(HTTPAlertProvider):
dx_call = second_cell_anchor.get_text().strip().upper()
# Get the date, dealing with the fact we get no year so have to figure out if it's last year or next year
date_span = cells[2].find('span') if len(cells) > 2 else None
date_span = cells[2].find("span") if len(cells) > 2 else None
if not date_span:
continue
date_text = date_span.get_text().strip()
@@ -52,11 +52,13 @@ class BOTA(HTTPAlertProvider):
date_time = date_time.replace(year=datetime.now(pytz.UTC).year + 1)
# Convert to our alert format
alert = Alert(source=self.name,
dx_calls=[dx_call],
sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
start_time=date_time.timestamp(),
is_dxpedition=False)
alert = Alert(
source=self.name,
dx_calls=[dx_call],
sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
start_time=date_time.timestamp(),
is_dxpedition=False,
)
new_alerts.append(alert)
return new_alerts
providers/alert/http_alert_provider.py
+3 -3
View File
@@ -1,13 +1,13 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from providers.alert.alert_provider import AlertProvider
from core.constants import HTTP_HEADERS
from providers.alert.alert_provider import AlertProvider
class HTTPAlertProvider(AlertProvider):
providers/alert/ng3k.py
+21 -14
View File
@@ -6,8 +6,8 @@ import pytz
from rss_parser import Parser
from rss_parser.models.rss import RSS
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from providers.alert.http_alert_provider import HTTPAlertProvider
class NG3K(HTTPAlertProvider):
@@ -49,11 +49,16 @@ class NG3K(HTTPAlertProvider):
end_day = end_string.split(", ")[0].strip()
end_mon = start_mon
start_timestamp = datetime.strptime(f"{start_year} {start_mon} {start_day}", "%Y %b %d").replace(
tzinfo=pytz.UTC).timestamp()
end_timestamp = datetime.strptime(f"{end_year} {end_mon} {end_day} 23:59",
"%Y %b %d %H:%M").replace(
tzinfo=pytz.UTC).timestamp()
start_timestamp = (
datetime.strptime(f"{start_year} {start_mon} {start_day}", "%Y %b %d")
.replace(tzinfo=pytz.UTC)
.timestamp()
)
end_timestamp = (
datetime.strptime(f"{end_year} {end_mon} {end_day} 23:59", "%Y %b %d %H:%M")
.replace(tzinfo=pytz.UTC)
.timestamp()
)
# Sometimes the DX callsign is "real", sometimes you just get a prefix with the real working callsigns being
# provided in the "by" field. e.g. call="JW", by="By LA7XK as JW7XK, LA6VM as JW6VM, LA9DL as JW9DL". So
@@ -75,14 +80,16 @@ class NG3K(HTTPAlertProvider):
comment = extra_parts[3] if len(extra_parts) > 3 else ""
# Convert to our alert format
alert = Alert(source=self.name,
dx_calls=dx_calls,
dx_country=dx_country,
freqs_modes=bands + (f"; {modes}" if modes != "" else ""),
comment=f"{by}; {comment}; {qsl_info}",
start_time=start_timestamp,
end_time=end_timestamp,
is_dxpedition=True)
alert = Alert(
source=self.name,
dx_calls=dx_calls,
dx_country=dx_country,
freqs_modes=bands + (f"; {modes}" if modes != "" else ""),
comment=f"{by}; {comment}; {qsl_info}",
start_time=start_timestamp,
end_time=end_timestamp,
is_dxpedition=True,
)
# Add to our list.
new_alerts.append(alert)
providers/alert/parksnpeaks.py
+25 -12
View File
@@ -3,9 +3,9 @@ from datetime import datetime
import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class ParksNPeaks(HTTPAlertProvider):
@@ -30,8 +30,9 @@ class ParksNPeaks(HTTPAlertProvider):
else:
sig_ref = source_alert["WWFFID"]
sig_ref_name = source_alert["Location"]
start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp()
start_time = (
datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp()
)
sigrefs = []
# PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list,
@@ -40,17 +41,29 @@ class ParksNPeaks(HTTPAlertProvider):
sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)]
# Convert to our alert format
alert = Alert(source=self.name,
source_id=source_alert["alID"],
dx_calls=[source_alert["CallSign"].upper()],
freqs_modes=f"{source_alert['Freq']} {source_alert['MODE']}",
comment=source_alert["Comments"],
sig_refs=sigrefs,
start_time=start_time,
is_dxpedition=False)
alert = Alert(
source=self.name,
source_id=source_alert["alID"],
dx_calls=[source_alert["CallSign"].upper()],
freqs_modes=f"{source_alert['Freq']} {source_alert['MODE']}",
comment=source_alert["Comments"],
sig_refs=sigrefs,
start_time=start_time,
is_dxpedition=False,
)
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig and sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "SANPCPA", "LLOTA", "QRP"]:
if sig and sig not in [
"POTA",
"SOTA",
"WWFF",
"SIOTA",
"ZLOTA",
"KRMNPA",
"SANPCPA",
"LLOTA",
"QRP",
]:
logging.warning(f"PNP alert found with sig {sig}, developer needs to add support for this!")
# If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
providers/alert/pota.py
+26 -13
View File
@@ -2,9 +2,9 @@ from datetime import datetime
import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class POTA(HTTPAlertProvider):
@@ -21,18 +21,31 @@ class POTA(HTTPAlertProvider):
# Iterate through source data
for source_alert in http_response.json():
# Convert to our alert format
alert = Alert(source=self.name,
source_id=source_alert["scheduledActivitiesId"],
dx_calls=[source_alert["activator"].upper()],
freqs_modes=source_alert["frequencies"],
comment=source_alert["comments"],
sig_refs=[SIGRef(id=source_alert["reference"], sig="POTA", name=source_alert["name"],
url=f"https://pota.app/#/park/{source_alert['reference']}")],
start_time=datetime.strptime(source_alert["startDate"] + source_alert["startTime"],
"%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"],
"%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
is_dxpedition=False)
alert = Alert(
source=self.name,
source_id=source_alert["scheduledActivitiesId"],
dx_calls=[source_alert["activator"].upper()],
freqs_modes=source_alert["frequencies"],
comment=source_alert["comments"],
sig_refs=[
SIGRef(
id=source_alert["reference"],
sig="POTA",
name=source_alert["name"],
url=f"https://pota.app/#/park/{source_alert['reference']}",
)
],
start_time=datetime.strptime(
source_alert["startDate"] + source_alert["startTime"],
"%Y-%m-%d%H:%M",
)
.replace(tzinfo=pytz.UTC)
.timestamp(),
end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"], "%Y-%m-%d%H:%M")
.replace(tzinfo=pytz.UTC)
.timestamp(),
is_dxpedition=False,
)
# Add to our list, but exclude any old spots that POTA can sometimes give us where even the end time is
# in the past. Don't worry about de-duping, removing old alerts etc. at this point; other code will do
providers/alert/sota.py
+21 -13
View File
@@ -2,9 +2,9 @@ from datetime import datetime
import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class SOTA(HTTPAlertProvider):
@@ -26,18 +26,26 @@ class SOTA(HTTPAlertProvider):
summit_points = None
if len(details) > 2:
summit_points = int(details[-1].split(" ")[0])
alert = Alert(source=self.name,
source_id=source_alert["id"],
dx_calls=[source_alert["activatingCallsign"].upper()],
dx_names=[source_alert["activatorName"].upper()],
freqs_modes=source_alert["frequency"],
comment=source_alert["comments"],
sig_refs=[
SIGRef(id=f"{source_alert['associationCode']}/{source_alert['summitCode']}", sig="SOTA",
name=summit_name, activation_score=summit_points)],
start_time=datetime.strptime(source_alert["dateActivated"],
"%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=pytz.UTC).timestamp(),
is_dxpedition=False)
alert = Alert(
source=self.name,
source_id=source_alert["id"],
dx_calls=[source_alert["activatingCallsign"].upper()],
dx_names=[source_alert["activatorName"].upper()],
freqs_modes=source_alert["frequency"],
comment=source_alert["comments"],
sig_refs=[
SIGRef(
id=f"{source_alert['associationCode']}/{source_alert['summitCode']}",
sig="SOTA",
name=summit_name,
activation_score=summit_points,
)
],
start_time=datetime.strptime(source_alert["dateActivated"], "%Y-%m-%dT%H:%M:%SZ")
.replace(tzinfo=pytz.UTC)
.timestamp(),
is_dxpedition=False,
)
# Add to our list
new_alerts.append(alert)
providers/alert/wota.py
+15 -10
View File
@@ -5,9 +5,9 @@ import pytz
from rss_parser import Parser as RSSParser
from rss_parser.models.rss import RSS
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class WOTA(HTTPAlertProvider):
@@ -25,9 +25,12 @@ class WOTA(HTTPAlertProvider):
rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig")))
# Iterate through source data
for source_alert in rss.channel.items:
# Reject GUID missing or zero
if not source_alert.guid or not source_alert.guid.content or source_alert.guid.content == "http://www.wota.org.uk/alerts/0":
if (
not source_alert.guid
or not source_alert.guid.content
or source_alert.guid.content == "http://www.wota.org.uk/alerts/0"
):
continue
# Pick apart the title
@@ -51,13 +54,15 @@ class WOTA(HTTPAlertProvider):
time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
# Convert to our alert format
alert = Alert(source=self.name,
source_id=source_alert.guid.content,
dx_calls=[dx_call],
freqs_modes=freqs_modes,
comment=comment,
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
start_time=time.timestamp())
alert = Alert(
source=self.name,
source_id=source_alert.guid.content,
dx_calls=[dx_call],
freqs_modes=freqs_modes,
comment=comment,
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
start_time=time.timestamp(),
)
# Add to our list.
new_alerts.append(alert)
providers/alert/wwff.py
+16 -12
View File
@@ -2,9 +2,9 @@ from datetime import datetime
import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class WWFF(HTTPAlertProvider):
@@ -21,17 +21,21 @@ class WWFF(HTTPAlertProvider):
# Iterate through source data
for source_alert in http_response.json():
# Convert to our alert format
alert = Alert(source=self.name,
source_id=source_alert["id"],
dx_calls=[source_alert["activator_call"].upper()],
freqs_modes=f"{source_alert['band']} {source_alert['mode']}",
comment=source_alert["remarks"],
sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")],
start_time=datetime.strptime(source_alert["utc_start"],
"%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
end_time=datetime.strptime(source_alert["utc_end"],
"%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
is_dxpedition=False)
alert = Alert(
source=self.name,
source_id=source_alert["id"],
dx_calls=[source_alert["activator_call"].upper()],
freqs_modes=f"{source_alert['band']} {source_alert['mode']}",
comment=source_alert["remarks"],
sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")],
start_time=datetime.strptime(source_alert["utc_start"], "%Y-%m-%d %H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
end_time=datetime.strptime(source_alert["utc_end"], "%Y-%m-%d %H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
is_dxpedition=False,
)
# Add to our list
new_alerts.append(alert)
providers/callsigndata/api_query_callsign_data_provider.py
+1 -1
View File
@@ -5,7 +5,7 @@ class APIQueryCallsignDataProvider(CallsignDataProvider):
"""Generic callsign data provider class for providers that fetch their data from the web on-demand using an API."""
def __init__(self, name, provider_config, storage):
""" Set up the provider."""
"""Set up the provider."""
super().__init__(name, provider_config, storage)
if self.enabled:
providers/callsigndata/callsign_data_provider.py
-1
View File
@@ -51,7 +51,6 @@ class CallsignDataProvider:
else:
return None
def _perform_new_lookup(self, callsign, lookup_credentials):
"""Makes a new request to the data source for callsign data."""
providers/callsigndata/clublogapi.py
+6 -4
View File
@@ -2,12 +2,14 @@ import logging
from datetime import datetime
import pytz
from pyhamtools import LookupLib, Callinfo
from pyhamtools import Callinfo, LookupLib
from core.data_store import DATA_STORE
from core.utils import get_callsign_object_from_pyhamtools_callinfo
from data.callsign import Callsign
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
from providers.callsigndata.api_query_callsign_data_provider import (
APIQueryCallsignDataProvider,
)
class ClublogAPI(APIQueryCallsignDataProvider):
@@ -24,11 +26,11 @@ class ClublogAPI(APIQueryCallsignDataProvider):
else:
provider_config["enabled"] = False
logging.warning(
"Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
"Clublog XML callsign data provider configured but no api key was provided, this has been disabled."
)
super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
def _perform_new_lookup(self, callsign, lookup_credentials):
callsign_data = Callsign(call=callsign)
providers/callsigndata/clublogxml.py
+14 -5
View File
@@ -1,12 +1,14 @@
import gzip
import logging
from pyhamtools import LookupLib, Callinfo
from pyhamtools import Callinfo, LookupLib
from core.data_store import DATA_STORE
from core.utils import get_callsign_object_from_pyhamtools_callinfo
from data.callsign import Callsign
from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
from providers.callsigndata.file_download_callsign_data_provider import (
FileDownloadCallsignDataProvider,
)
class ClublogXML(FileDownloadCallsignDataProvider):
@@ -24,10 +26,17 @@ class ClublogXML(FileDownloadCallsignDataProvider):
if self._api_key == "":
provider_config["enabled"] = False
logging.warning(
"Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
"Clublog XML callsign data provider configured but no api key was provided, this has been disabled."
)
super().__init__("Clublog XML", provider_config, f"{self.DATA_URL}?api={self._api_key}",
self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS, DATA_STORE.callsign_data_clublogxml)
super().__init__(
"Clublog XML",
provider_config,
f"{self.DATA_URL}?api={self._api_key}",
self.CACHE_PATH_ZIPPED,
self.POLL_INTERVAL_DAYS,
DATA_STORE.callsign_data_clublogxml,
)
def _handle_file(self, path):
try:
providers/callsigndata/countryfiles.py
+12 -4
View File
@@ -1,11 +1,13 @@
import logging
from pyhamtools import LookupLib, Callinfo
from pyhamtools import Callinfo, LookupLib
from core.data_store import DATA_STORE
from core.utils import get_callsign_object_from_pyhamtools_callinfo
from data.callsign import Callsign
from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
from providers.callsigndata.file_download_callsign_data_provider import (
FileDownloadCallsignDataProvider,
)
class CountryFiles(FileDownloadCallsignDataProvider):
@@ -17,8 +19,14 @@ class CountryFiles(FileDownloadCallsignDataProvider):
_callinfo = None
def __init__(self, provider_config):
super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS,
DATA_STORE.callsign_data_countryfiles)
super().__init__(
"CountryFiles.com",
provider_config,
self.DATA_URL,
self.CACHE_PATH,
self.POLL_INTERVAL_DAYS,
DATA_STORE.callsign_data_countryfiles,
)
def _handle_file(self, path):
try:
providers/callsigndata/file_download_callsign_data_provider.py
+6 -5
View File
@@ -1,6 +1,6 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
from requests import ReadTimeout
@@ -15,7 +15,7 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
"""Generic callsign data provider class for providers that fetch their data from the web by downloading a file."""
def __init__(self, name, provider_config, url, cache_file_path, poll_interval, storage):
""" Set up the provider, note poll_interval is in *days*."""
"""Set up the provider, note poll_interval is in *days*."""
super().__init__(name, provider_config, storage)
self._url = url
self._cache_file_path = cache_file_path
@@ -30,8 +30,7 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting.
logging.info(
f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
logging.info(f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}")
self._thread.start()
@@ -68,7 +67,9 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
else:
self.status = "Error"
logging.warning(f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}.")
logging.warning(
f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}."
)
except ConnectionError:
self.status = "Error"
providers/callsigndata/hamqth.py
+42 -31
View File
@@ -1,6 +1,6 @@
import logging
import urllib.parse
from datetime import timedelta, datetime
from datetime import datetime, timedelta
import pytz
import xmltodict
@@ -10,10 +10,12 @@ from requests_cache import CachedSession
from core.config import SERVER_OWNER_CALLSIGN
from core.constants import HTTP_HEADERS, SOFTWARE_VERSION
from core.data_store import DATA_STORE, CACHE_DIR
from core.data_store import CACHE_DIR, DATA_STORE
from core.url_data_cache import URLDataCache
from data.callsign import Callsign
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
from providers.callsigndata.api_query_callsign_data_provider import (
APIQueryCallsignDataProvider,
)
class HamQTH(APIQueryCallsignDataProvider):
@@ -26,14 +28,15 @@ class HamQTH(APIQueryCallsignDataProvider):
self._URL_DATA_CACHE = URLDataCache("hamqth")
# Separate URL cache for session key lookups. Once a session key is returned from logging in with a username
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/hamqth-creds",
expire_after=timedelta(minutes=55))
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/hamqth-creds", expire_after=timedelta(minutes=55))
def _perform_new_lookup(self, callsign, lookup_credentials):
# If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because
# # someone might provide credentials next time around.
if not lookup_credentials or not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
or lookup_credentials.hamqth_session_id):
if not lookup_credentials or not (
(lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
or lookup_credentials.hamqth_session_id
):
return None
try:
@@ -45,7 +48,8 @@ class HamQTH(APIQueryCallsignDataProvider):
try:
session_data = self._CREDENTIALS_CACHE.get(
f"{self._HAMQTH_BASE_URL}?u={urllib.parse.quote_plus(lookup_credentials.hamqth_username)}&p={urllib.parse.quote_plus(lookup_credentials.hamqth_password)}",
headers=HTTP_HEADERS).content
headers=HTTP_HEADERS,
).content
dict_data = xmltodict.parse(session_data)
if "session_id" in dict_data["HamQTH"]["session"]:
session_id = str(dict_data["HamQTH"]["session"]["session_id"])
@@ -67,13 +71,16 @@ class HamQTH(APIQueryCallsignDataProvider):
if home_call != callsign:
calls_to_try.append(home_call)
except ValueError:
logging.debug("Could not look up home call for callsign %s", callsign)
logging.debug(f"Could not look up home call for callsign {callsign}")
# Try looking up each call using the API
for lookup_call in calls_to_try:
try:
response = self._URL_DATA_CACHE.get(
f"{self._HAMQTH_BASE_URL}?id={session_id}&callsign={urllib.parse.quote_plus(lookup_call)}&prg={self._PRG}", headers=HTTP_HEADERS, timeout=10)
f"{self._HAMQTH_BASE_URL}?id={session_id}&callsign={urllib.parse.quote_plus(lookup_call)}&prg={self._PRG}",
headers=HTTP_HEADERS,
timeout=10,
)
if response.ok:
# Found data, convert it to our object and return it
data = xmltodict.parse(response.content)["HamQTH"]["search"]
@@ -83,19 +90,18 @@ class HamQTH(APIQueryCallsignDataProvider):
return self.hamqth_response_to_callsign(callsign, data)
elif not response.from_cache:
logging.warning("HTTP %d looking up callsign %s using HamQTH", response.status_code,
lookup_call)
logging.warning(f"HTTP {response.status_code} looking up callsign {lookup_call} using HamQTH")
except (KeyError, ValueError):
continue
except ConnectionError:
logging.warning(f"Connection error when looking up callsign %s using HamQTH", lookup_call)
logging.warning(f"Connection error when looking up callsign {lookup_call} using HamQTH")
continue
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when looking up callsign %s using HamQTH", lookup_call)
logging.warning(f"Timeout when looking up callsign {lookup_call} using HamQTH")
continue
except Exception:
logging.exception("Exception when looking up callsign %s using HamQTH", lookup_call)
logging.exception(f"Exception when looking up callsign {lookup_call} using HamQTH")
continue
# Not found in HamQTH; return a Callsign object with no data so we cache that and don't keep retrying
@@ -114,9 +120,12 @@ class HamQTH(APIQueryCallsignDataProvider):
# Check for sensible latitudes
lat = None
lon = None
if "latitude" in data and "longitude" in data and (
float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float(
data["latitude"]) < 89.9:
if (
"latitude" in data
and "longitude" in data
and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
and -89.9 < float(data["latitude"]) < 89.9
):
lat = float(data["latitude"])
lon = float(data["longitude"])
@@ -125,16 +134,18 @@ class HamQTH(APIQueryCallsignDataProvider):
if "grid" in data and not data["grid"].startswith("AA00"):
grid = data["grid"]
return Callsign(call=callsign,
home_call=callinfo.Callinfo.get_homecall(callsign),
name=data["nick"] if "nick" in data else None,
qth=data["qth"] if "qth" in data else None,
country=data["country"] if "country" in data else None,
continent=data["continent"] if "continent" in data else None,
latitude=lat,
longitude=lon,
grid=grid,
dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cq"]) if "cq" in data else None,
itu_zone=int(data["itu"]) if "itu" in data else None,
location_source="HOME QTH")
return Callsign(
call=callsign,
home_call=callinfo.Callinfo.get_homecall(callsign),
name=data["nick"] if "nick" in data else None,
qth=data["qth"] if "qth" in data else None,
country=data["country"] if "country" in data else None,
continent=data["continent"] if "continent" in data else None,
latitude=lat,
longitude=lon,
grid=grid,
dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cq"]) if "cq" in data else None,
itu_zone=int(data["itu"]) if "itu" in data else None,
location_source="HOME QTH",
)
providers/callsigndata/qrz.py
+45 -34
View File
@@ -1,6 +1,6 @@
import logging
import urllib.parse
from datetime import timedelta, datetime
from datetime import datetime, timedelta
import pytz
import xmltodict
@@ -9,10 +9,12 @@ from requests import ConnectTimeout, ReadTimeout
from requests_cache import CachedSession
from core.constants import HTTP_HEADERS
from core.data_store import DATA_STORE, CACHE_DIR
from core.data_store import CACHE_DIR, DATA_STORE
from core.url_data_cache import URLDataCache
from data.callsign import Callsign
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
from providers.callsigndata.api_query_callsign_data_provider import (
APIQueryCallsignDataProvider,
)
class QRZ(APIQueryCallsignDataProvider):
@@ -24,14 +26,14 @@ class QRZ(APIQueryCallsignDataProvider):
self._URL_DATA_CACHE = URLDataCache("qrz")
# Separate URL cache for session key lookups. Once a session key is returned from logging in with a username
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/qrz-creds",
expire_after=timedelta(minutes=55))
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/qrz-creds", expire_after=timedelta(minutes=55))
def _perform_new_lookup(self, callsign, lookup_credentials):
# If we don't have QRZ credentials, skip this lookup. Return None so we don't *cache* the lack of data, because
# someone might provide credentials next time around.
if not lookup_credentials or not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
or lookup_credentials.qrz_session_key):
if not lookup_credentials or not (
(lookup_credentials.qrz_username and lookup_credentials.qrz_password) or lookup_credentials.qrz_session_key
):
return None
try:
@@ -43,7 +45,8 @@ class QRZ(APIQueryCallsignDataProvider):
try:
login_response = self._CREDENTIALS_CACHE.get(
f"{self._QRZ_BASE_URL}?username={urllib.parse.quote_plus(lookup_credentials.qrz_username)}&password={urllib.parse.quote_plus(lookup_credentials.qrz_password)}&agent=spothole",
headers=HTTP_HEADERS).content
headers=HTTP_HEADERS,
).content
login_data = xmltodict.parse(login_response)
session = login_data.get("QRZDatabase", {}).get("Session", {})
if "Key" in session:
@@ -66,14 +69,16 @@ class QRZ(APIQueryCallsignDataProvider):
if home_call != callsign:
calls_to_try.append(home_call)
except ValueError:
logging.debug("Could not look up home call for callsign %s", callsign)
logging.debug(f"Could not look up home call for callsign {callsign}")
# Try looking up each call using the API
for lookup_call in calls_to_try:
try:
response = self._URL_DATA_CACHE.get(
f"{self._QRZ_BASE_URL}?s={session_key}&callsign={urllib.parse.quote_plus(lookup_call)}",
headers=HTTP_HEADERS, timeout=10)
headers=HTTP_HEADERS,
timeout=10,
)
if response.ok:
qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
if qrz_response:
@@ -88,24 +93,25 @@ class QRZ(APIQueryCallsignDataProvider):
elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
# Errors here are normally just "callsign not in database", no need to log that ourselves
# above debug level.
logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call,
qrz_response.get("Session").get("Error"))
logging.debug(
f"QRZ returned an error looking up callsign {lookup_call}: {qrz_response.get('Session').get('Error')}"
)
elif not response.from_cache:
logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call)
logging.warning(f"QRZ returned a malformed response looking up callsign {lookup_call}")
elif not response.from_cache:
logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call)
logging.warning(f"HTTP {response.status_code} looking up callsign {lookup_call} using QRZ")
except (KeyError, ValueError):
continue
except ConnectionError:
logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call)
logging.warning(f"Connection error when looking up callsign {lookup_call} using QRZ")
continue
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call)
logging.warning(f"Timeout when looking up callsign {lookup_call} using QRZ.")
continue
except Exception:
logging.exception("Exception when looking up callsign %s using QRZ", lookup_call)
logging.exception(f"Exception when looking up callsign {lookup_call} using QRZ")
continue
# Not found in QRZ; return a Callsign object with no data so we cache that and don't keep retrying
@@ -128,16 +134,19 @@ class QRZ(APIQueryCallsignDataProvider):
if "fname" in data:
name = data["fname"]
if "nick" in data:
name = f"{name} \"{data['nick']}\""
name = f'{name} "{data["nick"]}"'
if "name" in data:
name = f"{name} {data['name']}"
# Check for sensible latitudes
lat = None
lon = None
if "latitude" in data and "longitude" in data and (
float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float(
data["latitude"]) < 89.9:
if (
"latitude" in data
and "longitude" in data
and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
and -89.9 < float(data["latitude"]) < 89.9
):
lat = float(data["latitude"])
lon = float(data["longitude"])
@@ -146,16 +155,18 @@ class QRZ(APIQueryCallsignDataProvider):
if "grid" in data and not data["grid"].startswith("AA00"):
grid = data["grid"]
return Callsign(call=callsign,
home_call=callinfo.Callinfo.get_homecall(callsign),
name=name,
qth=data["addr2"] if "addr2" in data else None,
country=data["country"] if "country" in data else None,
continent=data["continent"] if "continent" in data else None,
latitude=lat,
longitude=lon,
grid=grid,
dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
location_source="HOME QTH")
return Callsign(
call=callsign,
home_call=callinfo.Callinfo.get_homecall(callsign),
name=name,
qth=data["addr2"] if "addr2" in data else None,
country=data["country"] if "country" in data else None,
continent=data["continent"] if "continent" in data else None,
latitude=lat,
longitude=lon,
grid=grid,
dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
location_source="HOME QTH",
)
providers/sigrefdata/arlhs.py
+15 -9
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ARLHS(FileDownloadSIGRefDataProvider):
@@ -20,14 +22,18 @@ class ARLHS(FileDownloadSIGRefDataProvider):
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
if "ARLHS" in row and row["ARLHS"] != "":
ref_id = row["ARLHS"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
ref_type="Lighthouse",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Lighthouse",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
grid=row["Maidenhead Locator"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/dme.py
+21 -10
View File
@@ -4,7 +4,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
from providers.sigrefdata.local_file_sig_ref_data_provider import (
LocalFileSIGRefDataProvider,
)
class DME(LocalFileSIGRefDataProvider):
@@ -21,16 +23,25 @@ class DME(LocalFileSIGRefDataProvider):
with open(path, encoding="latin-1") as _f:
for row in csv.DictReader(_f, delimiter=";"):
ref_id = row["COD_INE"][:5]
latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
"LATITUD_ETRS89_REGCAN95") else None
longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
"LONGITUD_ETRS89_REGCAN95") else None
latitude = (
float(row["LATITUD_ETRS89_REGCAN95"].replace(",", "."))
if row.get("LATITUD_ETRS89_REGCAN95")
else None
)
longitude = (
float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", "."))
if row.get("LONGITUD_ETRS89_REGCAN95")
else None
)
ref = SIGRef(sig=self.SIG, id=ref_id,
ref_type="Town",
name=f"{row['NOMBRE_ACTUAL']}, {row['PROVINCIA']}",
latitude=latitude,
longitude=longitude)
ref = SIGRef(
sig=self.SIG,
id=ref_id,
ref_type="Town",
name=f"{row['NOMBRE_ACTUAL']}, {row['PROVINCIA']}",
latitude=latitude,
longitude=longitude,
)
if latitude and longitude:
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
providers/sigrefdata/file_download_sig_ref_data_provider.py
+3 -4
View File
@@ -1,6 +1,6 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
from requests import ReadTimeout
@@ -15,7 +15,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
"""Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
def __init__(self, sig_name, provider_config, url, poll_interval):
""" Set up the provider, note poll_interval is in *days*."""
"""Set up the provider, note poll_interval is in *days*."""
super().__init__(sig_name, provider_config)
self._url = url
self._poll_interval = poll_interval
@@ -26,8 +26,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting.
logging.info(
f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
logging.info(f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}")
self._thread.start()
providers/sigrefdata/gma.py
+18 -11
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class GMA(FileDownloadSIGRefDataProvider):
@@ -19,16 +21,21 @@ class GMA(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
ref_type="Summit",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
altitude=float(row["Height (m)"].replace("m", "")) if "Height (m)" in row and row[
"Height (m)"] != "" else None,
grid=row["Maidenhead Locator"]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Summit",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
altitude=float(row["Height (m)"].replace("m", ""))
if "Height (m)" in row and row["Height (m)"] != ""
else None,
grid=row["Maidenhead Locator"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/illw.py
+15 -9
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ILLW(FileDownloadSIGRefDataProvider):
@@ -20,14 +22,18 @@ class ILLW(FileDownloadSIGRefDataProvider):
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
if "ILLW" in row and row["ILLW"] != "":
ref_id = row["ILLW"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
ref_type="Lighthouse",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Lighthouse",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
grid=row["Maidenhead Locator"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/iota.py
+19 -4
View File
@@ -4,7 +4,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class IOTA(FileDownloadSIGRefDataProvider):
@@ -29,10 +31,23 @@ class IOTA(FileDownloadSIGRefDataProvider):
try:
grid = latlong_to_locator(latitude, longitude, 6)
except ValueError:
logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
logging.debug(
"Error converting lat/lon to locator for an IOTA reference %f %f",
latitude,
longitude,
)
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
ref_type="Island", grid=grid, latitude=latitude, longitude=longitude))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=ref["name"],
ref_type="Island",
grid=grid,
latitude=latitude,
longitude=longitude,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
# of the data in this case
providers/sigrefdata/krmnpa.py
+4 -2
View File
@@ -1,4 +1,6 @@
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import ParksNPeaksKMLSIGRefDataProvider
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import (
ParksNPeaksKMLSIGRefDataProvider,
)
class KRMNPA(ParksNPeaksKMLSIGRefDataProvider):
@@ -9,4 +11,4 @@ class KRMNPA(ParksNPeaksKMLSIGRefDataProvider):
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=KRMNPA&poiFormat=4"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
providers/sigrefdata/llota.py
+15 -7
View File
@@ -3,7 +3,9 @@ from time import sleep
from pyhamtools.locator import locator_to_latlong
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class LLOTA(FileDownloadSIGRefDataProvider):
@@ -25,12 +27,18 @@ class LLOTA(FileDownloadSIGRefDataProvider):
grid = str(ref["grid_locator"])
ll = locator_to_latlong(grid)
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
ref_type="Lake",
url=f"https://llota.app/list/ref/{ref_id}",
grid=grid,
latitude=ll[0],
longitude=ll[1]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=str(ref["name"]),
ref_type="Lake",
url=f"https://llota.app/list/ref/{ref_id}",
grid=grid,
latitude=ll[0],
longitude=ll[1],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
# of the data in this case
providers/sigrefdata/mota.py
+15 -9
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class MOTA(FileDownloadSIGRefDataProvider):
@@ -19,14 +21,18 @@ class MOTA(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
ref_type="Mill",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Mill",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
grid=row["Maidenhead Locator"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/pnp_kml_sig_ref_data_provider.py
+13 -7
View File
@@ -5,7 +5,9 @@ from fastkml import kml
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
@@ -15,7 +17,7 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
REF_PATTERN = re.compile(r"VKFF-\d+")
def __init__(self, sig_name, provider_config, url, poll_interval):
""" Set up the provider, note poll_interval is in *days*."""
"""Set up the provider, note poll_interval is in *days*."""
super().__init__(sig_name, provider_config, url, poll_interval)
def _http_response_to_data(self, http_response):
@@ -37,11 +39,15 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
longitude, latitude = placemark.geometry.x, placemark.geometry.y
ref = SIGRef(sig=self.sig_name, id=ref_id, name=placemark.name,
ref_type="Park",
url=f"https://parksnpeaks.org/getPark.php?actPark={ref_id}",
latitude=latitude,
longitude=longitude)
ref = SIGRef(
sig=self.sig_name,
id=ref_id,
name=placemark.name,
ref_type="Park",
url=f"https://parksnpeaks.org/getPark.php?actPark={ref_id}",
latitude=latitude,
longitude=longitude,
)
if latitude and longitude:
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
providers/sigrefdata/pota.py
+15 -9
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class POTA(FileDownloadSIGRefDataProvider):
@@ -19,14 +21,18 @@ class POTA(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
ref_type="Park",
url=f"https://pota.app/#/park/{ref_id}",
grid=row["grid"] if "grid" in row else None,
latitude=float(row["latitude"]) if "latitude" in row and row[
"latitude"] != "" else None,
longitude=float(row["longitude"]) if "longitude" in row and row[
"longitude"] != "" else None))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["name"] if "name" in row else None,
ref_type="Park",
url=f"https://pota.app/#/park/{ref_id}",
grid=row["grid"] if "grid" in row else None,
latitude=float(row["latitude"]) if "latitude" in row and row["latitude"] != "" else None,
longitude=float(row["longitude"]) if "longitude" in row and row["longitude"] != "" else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/sanpcpa.py
+3 -1
View File
@@ -1,4 +1,6 @@
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import ParksNPeaksKMLSIGRefDataProvider
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import (
ParksNPeaksKMLSIGRefDataProvider,
)
class SANPCPA(ParksNPeaksKMLSIGRefDataProvider):
providers/sigrefdata/sig_ref_data_provider.py
+1 -3
View File
@@ -19,13 +19,11 @@ class SIGRefDataProvider:
self.reference_count = 0
self._stop = False
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
raise NotImplementedError("Subclasses must implement this method")
def stop(self):
"""Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
super()."""
@@ -47,4 +45,4 @@ class SIGRefDataProvider:
break
self.reference_count = len(new_data)
logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)
logging.info(f"Loaded {self.reference_count} references for {self.sig_name} into the data store.")
providers/sigrefdata/siota.py
+14 -6
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class SIOTA(FileDownloadSIGRefDataProvider):
@@ -19,11 +21,17 @@ class SIOTA(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["SILO_CODE"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
ref_type="Silo",
grid=row["LOCATOR"] if "LOCATOR" in row else None,
latitude=float(row["LAT"]) if "LAT" in row else None,
longitude=float(row["LNG"]) if "LNG" in row else None))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["NAME"] if "NAME" in row else None,
ref_type="Silo",
grid=row["LOCATOR"] if "LOCATOR" in row else None,
latitude=float(row["LAT"]) if "LAT" in row else None,
longitude=float(row["LNG"]) if "LNG" in row else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/sota.py
+14 -8
View File
@@ -4,7 +4,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class SOTA(FileDownloadSIGRefDataProvider):
@@ -24,13 +26,17 @@ class SOTA(FileDownloadSIGRefDataProvider):
latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
altitude = float(row["AltM"]) if "AltM" in row and row["AltM"] != "" else None
ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
ref_type="Summit",
url=f"https://www.sotadata.org.uk/en/summit/{ref_id}",
latitude=latitude,
longitude=longitude,
altitude=altitude,
activation_score=int(row["Points"]) if "Points" in row else None)
ref = SIGRef(
sig=self.SIG,
id=ref_id,
name=row["SummitName"] if "SummitName" in row else None,
ref_type="Summit",
url=f"https://www.sotadata.org.uk/en/summit/{ref_id}",
latitude=latitude,
longitude=longitude,
altitude=altitude,
activation_score=int(row["Points"]) if "Points" in row else None,
)
if latitude and longitude:
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
providers/sigrefdata/toilets.py
+13 -3
View File
@@ -1,7 +1,9 @@
import csv
from data.sig_ref import SIGRef
from providers.sigrefdata.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
from providers.sigrefdata.local_file_sig_ref_data_provider import (
LocalFileSIGRefDataProvider,
)
class Toilets(LocalFileSIGRefDataProvider):
@@ -19,8 +21,16 @@ class Toilets(LocalFileSIGRefDataProvider):
csv_data = _f.read()
dr = csv.DictReader(csv_data.splitlines())
for row in dr:
new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], ref_type="Toilet",
latitude=float(row["lat"]), longitude=float(row["lon"])))
new_data.append(
SIGRef(
sig=self.SIG,
id=row["ref"],
name=row["ref"],
ref_type="Toilet",
latitude=float(row["lat"]),
longitude=float(row["lon"]),
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
# of the data in this case
providers/sigrefdata/towers.py
+15 -7
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class Towers(FileDownloadSIGRefDataProvider):
@@ -19,12 +21,18 @@ class Towers(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
ref_id = row["Ref"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
ref_type="Tower",
url=f"https://wwtota.com/seznam/karta_rozhledny.php?ref={ref_id}",
grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Nazev"] if "Nazev" in row else None,
ref_type="Tower",
url=f"https://wwtota.com/seznam/karta_rozhledny.php?ref={ref_id}",
grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/wca.py
+16 -8
View File
@@ -5,7 +5,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WCA(FileDownloadSIGRefDataProvider):
@@ -34,14 +36,20 @@ class WCA(FileDownloadSIGRefDataProvider):
longitude = float(split[1])
grid = latlong_to_locator(latitude, longitude)
except ValueError:
logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for {ref_id}")
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
ref_type="Castle",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=latitude,
longitude=longitude,
grid=grid))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
ref_type="Castle",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=latitude,
longitude=longitude,
grid=grid,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/wota.py
+16 -7
View File
@@ -1,7 +1,9 @@
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WOTA(FileDownloadSIGRefDataProvider):
@@ -25,12 +27,19 @@ class WOTA(FileDownloadSIGRefDataProvider):
number = int(ref_id.upper().replace("LDO-", ""))
url = f"https://www.wota.org.uk/MM_LDO-{number + 214!s}"
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
ref_type="Summit",
grid=feature["properties"]["qthLocator"],
latitude=feature["geometry"]["coordinates"][1],
longitude=feature["geometry"]["coordinates"][0],
altitude=feature["properties"]["height"]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=feature["properties"]["title"],
url=url,
ref_type="Summit",
grid=feature["properties"]["qthLocator"],
latitude=feature["geometry"]["coordinates"][1],
longitude=feature["geometry"]["coordinates"][0],
altitude=feature["properties"]["height"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/wwbota.py
+15 -7
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WWBOTA(FileDownloadSIGRefDataProvider):
@@ -19,12 +21,18 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["Reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
ref_type="Bunker",
url=f"https://bunkerwiki.org/?s={ref_id}" if ref_id.startswith("B/G") else None,
grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Bunker",
url=f"https://bunkerwiki.org/?s={ref_id}" if ref_id.startswith("B/G") else None,
grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/wwff.py
+19 -10
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WWFF(FileDownloadSIGRefDataProvider):
@@ -19,15 +21,22 @@ class WWFF(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
ref_type="Park",
url=f"https://wwff.co/directory/?showRef={ref_id}",
grid=row["iaruLocator"] if "iaruLocator" in row and row[
"iaruLocator"] != "-" else None,
latitude=float(row["latitude"]) if "latitude" in row and row[
"latitude"] != "" and row["latitude"] != "-" else None,
longitude=float(row["longitude"]) if "longitude" in row and row[
"longitude"] != "" and row["longitude"] != "-" else None))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["name"] if "name" in row else None,
ref_type="Park",
url=f"https://wwff.co/directory/?showRef={ref_id}",
grid=row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None,
latitude=float(row["latitude"])
if "latitude" in row and row["latitude"] != "" and row["latitude"] != "-"
else None,
longitude=float(row["longitude"])
if "longitude" in row and row["longitude"] != "" and row["longitude"] != "-"
else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/zlota.py
+12 -6
View File
@@ -3,7 +3,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ZLOTA(FileDownloadSIGRefDataProvider):
@@ -25,11 +27,15 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
latitude = ref["latitude"]
longitude = ref["longitude"]
new_ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
ref_type=ref["asset_type"].title(),
url=f"https://ontheair.nz/assets/{ref_id.replace('/', '_')}",
latitude=latitude,
longitude=longitude)
new_ref = SIGRef(
sig=self.SIG,
id=ref_id,
name=ref["name"],
ref_type=ref["asset_type"].title(),
url=f"https://ontheair.nz/assets/{ref_id.replace('/', '_')}",
latitude=latitude,
longitude=longitude,
)
# Check lat/lon validity and update grid accordingly
if latitude and longitude:
providers/solarconditions/giroionosonde.py
+27 -13
View File
@@ -1,11 +1,11 @@
import csv
import logging
from datetime import datetime, timezone, timedelta
from threading import Thread, Event
from datetime import datetime, timedelta, timezone
from threading import Event, Thread
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from providers.solarconditions.ionosonde_utils import compute_band_states
@@ -40,9 +40,16 @@ class GIROIonosonde(SolarConditionsProvider):
# entries so KC2G cache data is preserved.
existing = self._solar_conditions.ionosonde_data or {}
new_entries = {
s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
"luf": None, "band_states": None}
for s in self._stations if s["ursi"] not in existing
s["ursi"]: {
"ursi": s["ursi"],
"name": s["name"],
"fof2": None,
"muf": None,
"luf": None,
"band_states": None,
}
for s in self._stations
if s["ursi"] not in existing
}
if new_entries:
self.update_data({"ionosonde_data": {**existing, **new_entries}})
@@ -50,7 +57,7 @@ class GIROIonosonde(SolarConditionsProvider):
@staticmethod
def _load_stations():
stations = []
with open(STATIONS_INDEX, newline='') as f:
with open(STATIONS_INDEX, newline="") as f:
for row in csv.reader(f):
if len(row) >= 2:
stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
@@ -94,8 +101,14 @@ class GIROIonosonde(SolarConditionsProvider):
# Merge GIRO's readings into any existing data for this station.
existing = ionosonde_data.get(ursi, {})
merged_fof2 = {**{float(t): v for t, v in (existing.get("fof2") or {}).items()}, **fof2}
merged_muf = {**{float(t): v for t, v in (existing.get("muf") or {}).items()}, **muf}
merged_fof2 = {
**{float(t): v for t, v in (existing.get("fof2") or {}).items()},
**fof2,
}
merged_muf = {
**{float(t): v for t, v in (existing.get("muf") or {}).items()},
**muf,
}
merged_luf = dict(luf) if luf else {}
merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts}
@@ -104,7 +117,8 @@ class GIROIonosonde(SolarConditionsProvider):
band_states = compute_band_states(merged_fof2, merged_muf, merged_luf)
ionosonde_data[ursi] = {
"ursi": ursi, "name": name,
"ursi": ursi,
"name": name,
"fof2": merged_fof2 or None,
"muf": merged_muf or None,
"luf": merged_luf or None,
@@ -132,7 +146,7 @@ class GIROIonosonde(SolarConditionsProvider):
return None, None, None
return self._parse_all(http_response.text)
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing Giro ionosonde API.")
logging.warning("Timeout when accessing Giro ionosonde API.")
return None, None, None
except ConnectionError:
logging.warning("Connection error when accessing Giro ionosonde API.")
@@ -147,14 +161,14 @@ class GIROIonosonde(SolarConditionsProvider):
luf_data = {}
for line in text.splitlines():
line = line.strip()
if not line or line.startswith('#'):
if not line or line.startswith("#"):
continue
# Data rows have the following format: timestamp CS foF2 QD MUFD QD fmin QD
parts = line.split()
if len(parts) >= 5:
try:
# Python 3.8 TZ parsing fudge
ts = datetime.fromisoformat(parts[0].replace('Z', '+00:00')).timestamp()
ts = datetime.fromisoformat(parts[0].replace("Z", "+00:00")).timestamp()
except ValueError:
continue
try:
providers/solarconditions/hamqsl.py
+11 -5
View File
@@ -2,9 +2,12 @@ import logging
from xml.etree import ElementTree
import pytz
from dateutil import parser as dateutil_parser, tz as dateutil_tz
from dateutil import parser as dateutil_parser
from dateutil import tz as dateutil_tz
from providers.solarconditions.http_solar_conditions_provider import HTTPSolarConditionsProvider
from providers.solarconditions.http_solar_conditions_provider import (
HTTPSolarConditionsProvider,
)
POLL_INTERVAL = 3600 # 1 hour
URL = "https://www.hamqsl.com/solarxml.php"
@@ -92,7 +95,8 @@ class HamQSL(HTTPSolarConditionsProvider):
"aurora_latitude": float_val("latdegree"),
"solar_wind": float_val("solarwind"),
"magnetic_field": float_val("magneticfield"),
"geomag_field": text("geomagfield").title()
"geomag_field": text("geomagfield")
.title()
.replace("Vr Quiet", "Very Quiet")
.replace("Unsettld", "Unsettled")
.replace("Min Strm", "Minor Storm")
@@ -102,8 +106,10 @@ class HamQSL(HTTPSolarConditionsProvider):
"geomag_noise": text("signalnoise"),
"hf_conditions": hf_conditions,
"vhf_conditions": {
"vhf_aurora_northern_hemi": (vhf_map.get(("vhf-aurora", "northern_hemi")) or "").title().replace(
"Lat Aur", "Latitude") or None,
"vhf_aurora_northern_hemi": (vhf_map.get(("vhf-aurora", "northern_hemi")) or "")
.title()
.replace("Lat Aur", "Latitude")
or None,
"es_2m_europe": vhf_map.get(("E-Skip", "europe")),
"es_4m_europe": vhf_map.get(("E-Skip", "europe_4m")),
"es_6m_europe": vhf_map.get(("E-Skip", "europe_6m")),
providers/solarconditions/http_solar_conditions_provider.py
+3 -4
View File
@@ -1,10 +1,10 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider
@@ -22,8 +22,7 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
self._stop_event = Event()
def start(self):
logging.info(
f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
logging.info(f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}")
self._thread.start()
providers/solarconditions/kc2gprop.py
+4 -4
View File
@@ -1,10 +1,10 @@
import logging
from datetime import datetime, timezone, timedelta
from threading import Thread, Event
from datetime import datetime, timedelta, timezone
from threading import Event, Thread
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from providers.solarconditions.ionosonde_utils import compute_band_states
@@ -119,7 +119,7 @@ class KC2GProp(SolarConditionsProvider):
except ConnectionError:
logging.warning("Connection error when accessing KC2G ionosonde API.")
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing KC2G ionosonde API.")
logging.warning("Timeout when accessing KC2G ionosonde API.")
except Exception:
self.status = "Error"
logging.exception("Exception in KC2G ionosonde data provider")
providers/solarconditions/noaa3dayforecast.py
+22 -12
View File
@@ -2,7 +2,9 @@ import logging
import re
from datetime import datetime, timezone
from providers.solarconditions.http_solar_conditions_provider import HTTPSolarConditionsProvider
from providers.solarconditions.http_solar_conditions_provider import (
HTTPSolarConditionsProvider,
)
POLL_INTERVAL = 10800 # Every 3 hours
URL = "https://services.swpc.noaa.gov/text/3-day-forecast.txt"
@@ -32,13 +34,13 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Find the date header line by scanning the next few lines for month & day patterns
date_header_idx = None
for j in range(start_idx + 1, min(start_idx + 6, len(lines))):
if re.search(r'[A-Za-z]{3}\s+\d{2}', lines[j]):
if re.search(r"[A-Za-z]{3}\s+\d{2}", lines[j]):
date_header_idx = j
break
if date_header_idx is None:
logging.warning(f"NOAA 3-day forecast: could not find date header after '{section_header}'")
return None
date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', lines[date_header_idx])
date_matches = re.findall(r"([A-Za-z]{3})\s+(\d{2})", lines[date_header_idx])
if not date_matches:
logging.warning(f"NOAA 3-day forecast: no dates in header: {lines[date_header_idx]}")
return None
@@ -55,20 +57,20 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Parse data rows. Each non-empty line should have a text label followed by percentage values
result = {}
for line in lines[date_header_idx + 1:]:
for line in lines[date_header_idx + 1 :]:
line_stripped = line.strip()
if not line_stripped:
if result:
break
continue
pct_matches = list(re.finditer(r'\b(\d+)%', line_stripped))
pct_matches = list(re.finditer(r"\b(\d+)%", line_stripped))
if not pct_matches:
if result:
break
continue
# Row label is everything before the first percentage value
row_label = line_stripped[:line_stripped.index(pct_matches[0].group())].strip()
row_label = line_stripped[: line_stripped.index(pct_matches[0].group())].strip()
row_data = {}
for j, match in enumerate(pct_matches):
if j >= len(column_timestamps):
@@ -94,7 +96,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Extract the year from the header line, e.g. "NOAA Kp index breakdown Apr 2-Apr 4, 2026"
header_line = lines[start_idx]
year_match = re.search(r'\b(\d{4})\b', header_line)
year_match = re.search(r"\b(\d{4})\b", header_line)
if not year_match:
logging.warning(f"NOAA K-index forecast: could not extract year from: {header_line}")
return None
@@ -106,7 +108,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
return None
date_header_line = lines[start_idx + 2]
date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', date_header_line)
date_matches = re.findall(r"([A-Za-z]{3})\s+(\d{2})", date_header_line)
if not date_matches:
logging.warning(f"NOAA K-index forecast: could not parse date headers from: {date_header_line}")
return None
@@ -121,8 +123,8 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Parse each data row, e.g. "00-03UT 2.00 3.00 2.00"
k_index_forecast = {}
for line in lines[start_idx + 3:]:
time_match = re.match(r'^(\d{2})-(\d{2})UT\s+(.*)', line.strip())
for line in lines[start_idx + 3 :]:
time_match = re.match(r"^(\d{2})-(\d{2})UT\s+(.*)", line.strip())
if not time_match:
if k_index_forecast:
break
@@ -130,7 +132,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
start_hour = int(time_match.group(1))
# Split on 2 or more spaces so that e.g. "5.67 (G2)" stays as one token per column
raw_values = re.split(r' {2,}', time_match.group(3).strip())
raw_values = re.split(r" {2,}", time_match.group(3).strip())
for i, val in enumerate(raw_values):
if i >= len(column_dates):
@@ -142,7 +144,15 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
continue
date = column_dates[i]
start_dt = datetime(date.year, date.month, date.day, start_hour, 0, 0, tzinfo=timezone.utc)
start_dt = datetime(
date.year,
date.month,
date.day,
start_hour,
0,
0,
tzinfo=timezone.utc,
)
# Key the data dict by start time
key = start_dt.timestamp()
providers/solarconditions/solar_conditions_provider.py
+1 -1
View File
@@ -7,7 +7,7 @@ from core.data_store import DATA_STORE
class SolarConditionsProvider:
"""Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
propagation data."""
propagation data."""
def __init__(self, name, provider_config):
"""Constructor"""
providers/spot/aprsis.py
+11 -10
View File
@@ -43,16 +43,17 @@ class APRSIS(SpotProvider):
via_parts = str(data["via"]).split("-")
de_call = via_parts[0].upper()
de_ssid = via_parts[1].upper() if len(via_parts) > 1 else None
spot = Spot(source="APRS-IS",
dx_call=dx_call,
dx_ssid=dx_ssid,
de_call=de_call,
de_ssid=de_ssid,
comment=str(data["comment"]) if "comment" in data else None,
dx_latitude=float(data["latitude"]) if "latitude" in data else None,
dx_longitude=float(data["longitude"]) if "longitude" in data else None,
time=datetime.now(
pytz.UTC).timestamp()) # APRS-IS spots are live so we can assume spot time is "now"
spot = Spot(
source="APRS-IS",
dx_call=dx_call,
dx_ssid=dx_ssid,
de_call=de_call,
de_ssid=de_ssid,
comment=str(data["comment"]) if "comment" in data else None,
dx_latitude=float(data["latitude"]) if "latitude" in data else None,
dx_longitude=float(data["longitude"]) if "longitude" in data else None,
time=datetime.now(pytz.UTC).timestamp(),
) # APRS-IS spots are live so we can assume spot time is "now"
# Add to our list
self._submit(spot)
providers/spot/dxcluster.py
+23 -13
View File
@@ -18,10 +18,12 @@ class DXCluster(SpotProvider):
_LINE_PATTERN_EXCLUDE_RBN = re.compile(
r"^DX de ([a-z0-9/]+):\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE)
re.IGNORECASE,
)
_LINE_PATTERN_ALLOW_RBN = re.compile(
r"^DX de ([a-z0-9/]+)-?#?:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE)
re.IGNORECASE,
)
def __init__(self, provider_config):
"""Constructor requires hostname and port"""
@@ -31,10 +33,13 @@ class DXCluster(SpotProvider):
self._hostname = provider_config["host"]
self._port = provider_config["port"]
self._login_prompt = provider_config["login_prompt"] if "login_prompt" in provider_config else "login:"
self._login_callsign = provider_config[
"login_callsign"] if "login_callsign" in provider_config else SERVER_OWNER_CALLSIGN
self._login_callsign = (
provider_config["login_callsign"] if "login_callsign" in provider_config else SERVER_OWNER_CALLSIGN
)
self._allow_rbn_spots = provider_config["allow_rbn_spots"] if "allow_rbn_spots" in provider_config else False
self._spot_line_pattern = self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
self._spot_line_pattern = (
self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
)
self._telnet = None
self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}")
self._thread.daemon = True
@@ -77,14 +82,19 @@ class DXCluster(SpotProvider):
match = self._spot_line_pattern.match(telnet_output.decode("latin-1"))
if match:
spot_time = datetime.strptime(match.group(5), "%H%MZ")
spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(),
tzinfo=pytz.UTC)
spot = Spot(source=self.name,
dx_call=match.group(3),
de_call=match.group(1),
freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(),
time=spot_datetime.timestamp())
spot_datetime = datetime.combine(
datetime.now(pytz.UTC).date(),
spot_time.time(),
tzinfo=pytz.UTC,
)
spot = Spot(
source=self.name,
dx_call=match.group(3),
de_call=match.group(1),
freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(),
time=spot_datetime.timestamp(),
)
# Add to our list
self._submit(spot)
providers/spot/gma.py
+64 -29
View File
@@ -27,7 +27,12 @@ class GMA(HTTPSpotProvider):
logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
self._url_data_cache = URLDataCache("GMA")
super().__init__("GMA", provider_config, f"{self.SPOTS_URL}?key={self._api_key}", self.POLL_INTERVAL_SEC)
super().__init__(
"GMA",
provider_config,
f"{self.SPOTS_URL}?key={self._api_key}",
self.POLL_INTERVAL_SEC,
)
def _http_response_to_spots(self, http_response):
new_spots = []
@@ -41,43 +46,67 @@ class GMA(HTTPSpotProvider):
# Seen some real janky times from GMA, if we don't understand it just ignore this spot
try:
time = datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
tzinfo=pytz.UTC).timestamp()
time = (
datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M")
.replace(tzinfo=pytz.UTC)
.timestamp()
)
except ValueError:
continue
spot = Spot(source=self.name,
dx_call=source_spot["ACTIVATOR"].upper(),
de_call=source_spot["SPOTTER"].upper(),
# Seen GMA spots with no frequency or with "QRT" in this field
freq=float(source_spot["QRG"]) * 1000 if (
source_spot["QRG"] != "" and source_spot["QRG"] != "QRT") else None,
# Filter out some weird mode strings
mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
comment=source_spot["TEXT"],
sig_refs=[SIGRef(id=source_spot["REF"], sig="", name=source_spot["NAME"], latitude=lat,
longitude=lon)],
time=time,
dx_latitude=lat,
dx_longitude=lon,
qrt=source_spot["QRG"] == "QRT")
spot = Spot(
source=self.name,
dx_call=source_spot["ACTIVATOR"].upper(),
de_call=source_spot["SPOTTER"].upper(),
# Seen GMA spots with no frequency or with "QRT" in this field
freq=float(source_spot["QRG"]) * 1000
if (source_spot["QRG"] != "" and source_spot["QRG"] != "QRT")
else None,
# Filter out some weird mode strings
mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
comment=source_spot["TEXT"],
sig_refs=[
SIGRef(
id=source_spot["REF"],
sig="",
name=source_spot["NAME"],
latitude=lat,
longitude=lon,
)
],
time=time,
dx_latitude=lat,
dx_longitude=lon,
qrt=source_spot["QRG"] == "QRT",
)
# GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
if "REF" in source_spot:
try:
ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
headers=HTTP_HEADERS)
ref_response = self._url_data_cache.get(
self.REF_INFO_URL_ROOT + source_spot["REF"],
headers=HTTP_HEADERS,
)
# Sometimes this is blank even if it's a 200 response, so handle that
if ref_response.ok and ref_response.text is not None and ref_response.text != "" and ref_response.text != "\n":
if (
ref_response.ok
and ref_response.text is not None
and ref_response.text != ""
and ref_response.text != "\n"
):
ref_info = ref_response.json()
# If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
# spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
# and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry
# to determine if it's a SOTA summit.
if spot.sig_refs and "reftype" in ref_info and ref_info["reftype"] not in ["POTA",
"WWFF"] and (
ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info[
"sota"] == ""):
if (
spot.sig_refs
and "reftype" in ref_info
and ref_info["reftype"] not in ["POTA", "WWFF"]
and (
ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info["sota"] == ""
)
):
match ref_info["reftype"]:
case "Summit":
spot.sig_refs[0].sig = "GMA"
@@ -98,7 +127,9 @@ class GMA(HTTPSpotProvider):
spot.sig_refs[0].sig = "MOTA"
spot.sig = "MOTA"
case _:
logging.warning(f"GMA spot found with ref type {ref_info['reftype']}, developer needs to add support for this!")
logging.warning(
f"GMA spot found with ref type {ref_info['reftype']}, developer needs to add support for this!"
)
spot.sig_refs[0].sig = ref_info["reftype"]
spot.sig = ref_info["reftype"]
@@ -109,12 +140,16 @@ class GMA(HTTPSpotProvider):
elif not ref_response.from_cache:
if not ref_response.ok:
logging.warning(
f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot['REF']}")
f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot['REF']}"
)
else:
logging.warning(
f"GMA API returned a malformed response when looking up ref {source_spot['REF']}")
f"GMA API returned a malformed response when looking up ref {source_spot['REF']}"
)
except:
logging.exception(f"Exception when looking up {self.REF_INFO_URL_ROOT}{source_spot['REF']}, ignoring this spot for now")
logging.exception(
f"Exception when looking up {self.REF_INFO_URL_ROOT}{source_spot['REF']}, ignoring this spot for now"
)
else:
logging.warning(f"The GMA API returned an unexpected response (HTTP {http_response.status_code}).")
providers/spot/hema.py
+24 -14
View File
@@ -4,7 +4,7 @@ from datetime import datetime
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from data.sig_ref import SIGRef
@@ -52,25 +52,35 @@ class HEMA(HTTPSpotProvider):
continue
# Convert to our spot format
spot = Spot(source=self.name,
dx_call=spot_items[2].upper(),
de_call=spotter_comment_match.group(1).upper(),
freq=float(freq_mode_match.group(1)) * 1000000,
mode=freq_mode_match.group(2).upper(),
comment=spotter_comment_match.group(2),
spot = Spot(
source=self.name,
dx_call=spot_items[2].upper(),
de_call=spotter_comment_match.group(1).upper(),
freq=float(freq_mode_match.group(1)) * 1000000,
mode=freq_mode_match.group(2).upper(),
comment=spotter_comment_match.group(2),
sig="HEMA",
sig_refs=[
SIGRef(
id=spot_items[3].upper(),
sig="HEMA",
sig_refs=[SIGRef(id=spot_items[3].upper(), sig="HEMA", name=spot_items[4],
latitude=float(spot_items[7]), longitude=float(spot_items[8]))],
time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(
tzinfo=pytz.UTC).timestamp(),
dx_latitude=float(spot_items[7]),
dx_longitude=float(spot_items[8]))
name=spot_items[4],
latitude=float(spot_items[7]),
longitude=float(spot_items[8]),
)
],
time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M")
.replace(tzinfo=pytz.UTC)
.timestamp(),
dx_latitude=float(spot_items[7]),
dx_longitude=float(spot_items[8]),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other
# code will do that for us.
new_spots.append(spot)
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing HEMA spots API.")
logging.warning("Timeout when accessing HEMA spots API.")
except ConnectionError:
logging.warning("Connection error when accessing HEMA spots API.")
return new_spots
providers/spot/http_spot_provider.py
+1 -1
View File
@@ -1,6 +1,6 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
import requests
providers/spot/llota.py
+17 -9
View File
@@ -25,16 +25,24 @@ class LLOTA(HTTPSpotProvider):
comment = str(source_spot["history"][-1]["comment"])
spotter = str(source_spot["history"][-1]["spotter_callsign"])
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["callsign"].upper(),
de_call=spotter.upper() if spotter else None,
freq=float(source_spot["frequency"]) * 1000000,
mode=source_spot["mode"].upper(),
comment=comment,
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["callsign"].upper(),
de_call=spotter.upper() if spotter else None,
freq=float(source_spot["frequency"]) * 1000000,
mode=source_spot["mode"].upper(),
comment=comment,
sig="LLOTA",
sig_refs=[
SIGRef(
id=source_spot["reference"],
sig="LLOTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="LLOTA", name=source_spot["reference_name"])],
time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp())
name=source_spot["reference_name"],
)
],
time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp(),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
providers/spot/parksnpeaks.py
+46 -16
View File
@@ -18,7 +18,17 @@ class ParksNPeaks(HTTPSpotProvider):
SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
SUBMIT_URL = "https://www.parksnpeaks.org/api/SPOT/"
SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
SUBMITTABLE_SIGS = ["POTA", "SOTA", "WWFF", "HEMA", "WOTA", "ZLOTA", "SIOTA", "KRMNPA", "SANPCPA"]
SUBMITTABLE_SIGS = [
"POTA",
"SOTA",
"WWFF",
"HEMA",
"WOTA",
"ZLOTA",
"SIOTA",
"KRMNPA",
"SANPCPA",
]
def __init__(self, provider_config):
super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -29,18 +39,23 @@ class ParksNPeaks(HTTPSpotProvider):
if http_response and http_response != "":
for source_spot in http_response.json():
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["actID"],
dx_call=source_spot["actCallsign"].upper(),
de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
# typo exists in API
freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "")
.replace("+/-", "").strip()) * 1000000 if (source_spot["actFreq"] != "") else None,
# Seen PNP spots with empty frequency, and with comma-separated thousands digits
mode=source_spot["actMode"].upper(),
comment=source_spot["actComments"],
time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp())
spot = Spot(
source=self.name,
source_id=source_spot["actID"],
dx_call=source_spot["actCallsign"].upper(),
de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
# typo exists in API
freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "").replace("+/-", "").strip())
* 1000000
if (source_spot["actFreq"] != "")
else None,
# Seen PNP spots with empty frequency, and with comma-separated thousands digits
mode=source_spot["actMode"].upper(),
comment=source_spot["actComments"],
time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
)
# Extract a de_call if it's in the comment but not in the "actSpoter" field
m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment or "")
@@ -53,7 +68,12 @@ class ParksNPeaks(HTTPSpotProvider):
sig_ref = source_spot["actSiteID"]
if sig and sig != "" and sig != "QRP" and sig_ref and sig_ref != "":
spot.sig = sig
sig_refs = [SIGRef(id=source_spot["actSiteID"], sig=source_spot["actClass"].upper())]
sig_refs = [
SIGRef(
id=source_spot["actSiteID"],
sig=source_spot["actClass"].upper(),
)
]
spot.sig_refs = sig_refs
# Free text location is not present in all spots, so only add it if it's set
@@ -61,7 +81,16 @@ class ParksNPeaks(HTTPSpotProvider):
sig_refs[0].name = source_spot["actLocation"]
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "SANPCPA", "LLOTA"]:
if sig not in [
"POTA",
"SOTA",
"WWFF",
"SIOTA",
"ZLOTA",
"KRMNPA",
"SANPCPA",
"LLOTA",
]:
logging.warning(f"PNP spot found with sig {sig}, developer needs to add support for this!")
# Add new spot to the list
@@ -77,7 +106,8 @@ class ParksNPeaks(HTTPSpotProvider):
api_key = credentials.get("api_key", "")
if not user_id or not api_key:
raise ValueError(
"Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account.")
"Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account."
)
sig_ref = spot.sig_refs[0].id if spot.sig_refs else ""
body = {
"actClass": spot.sig or "",
providers/spot/pota.py
+24 -14
View File
@@ -24,21 +24,31 @@ class POTA(HTTPSpotProvider):
# Iterate through source data
for source_spot in http_response.json():
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["spotId"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["frequency"]) * 1000,
mode=source_spot["mode"].upper(),
comment=source_spot["comments"],
spot = Spot(
source=self.name,
source_id=source_spot["spotId"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["frequency"]) * 1000,
mode=source_spot["mode"].upper(),
comment=source_spot["comments"],
sig="POTA",
sig_refs=[
SIGRef(
id=source_spot["reference"],
sig="POTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="POTA", name=source_spot["name"],
latitude=source_spot["latitude"], longitude=source_spot["longitude"])],
time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp(),
dx_grid=source_spot["grid6"],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"])
name=source_spot["name"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
)
],
time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
dx_grid=source_spot["grid6"],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
providers/spot/rbn.py
+15 -9
View File
@@ -18,7 +18,8 @@ class RBN(SpotProvider):
_LINE_PATTERN = re.compile(
r"^DX de ([a-z0-9/]+)-.*:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE)
re.IGNORECASE,
)
def __init__(self, provider_config):
"""Constructor requires port number."""
@@ -64,14 +65,19 @@ class RBN(SpotProvider):
match = self._LINE_PATTERN.match(telnet_output.decode("latin-1"))
if match:
spot_time = datetime.strptime(match.group(5), "%H%MZ")
spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(),
tzinfo=pytz.UTC)
spot = Spot(source=self.name,
dx_call=match.group(3),
de_call=match.group(1),
freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(),
time=spot_datetime.timestamp())
spot_datetime = datetime.combine(
datetime.now(pytz.UTC).date(),
spot_time.time(),
tzinfo=pytz.UTC,
)
spot = Spot(
source=self.name,
dx_call=match.group(3),
de_call=match.group(1),
freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(),
time=spot_datetime.timestamp(),
)
# Add to our list
self._submit(spot)
providers/spot/sota.py
+36 -23
View File
@@ -2,9 +2,9 @@ import logging
from datetime import datetime
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS, SSB_SUB_MODES, DV_SUB_MODES
from core.constants import DV_SUB_MODES, HTTP_HEADERS, SSB_SUB_MODES
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -41,24 +41,33 @@ class SOTA(HTTPSpotProvider):
# Iterate through source data
for source_spot in source_data:
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activatorCallsign"].upper(),
dx_name=source_spot["activatorName"],
de_call=source_spot["callsign"].upper(),
freq=(float(source_spot["frequency"]) * 1000000) if (
source_spot["frequency"] is not None) else None,
# Seen SOTA spots with no frequency!
mode=source_spot["mode"].upper(),
comment=source_spot["comments"],
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activatorCallsign"].upper(),
dx_name=source_spot["activatorName"],
de_call=source_spot["callsign"].upper(),
freq=(float(source_spot["frequency"]) * 1000000)
if (source_spot["frequency"] is not None)
else None,
# Seen SOTA spots with no frequency!
mode=source_spot["mode"].upper(),
comment=source_spot["comments"],
sig="SOTA",
sig_refs=[
SIGRef(
id=source_spot["summitCode"],
sig="SOTA",
sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA",
name=source_spot["summitName"], latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
activation_score=source_spot["points"])],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp())
name=source_spot["summitName"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
activation_score=source_spot["points"],
)
],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp(),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
@@ -66,7 +75,7 @@ class SOTA(HTTPSpotProvider):
except ConnectionError:
logging.warning("Connection error when accessing SOTA spots API")
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing SOTA spots API.")
logging.warning("Timeout when accessing SOTA spots API.")
return new_spots
def can_submit_spot(self, sig):
@@ -102,10 +111,14 @@ class SOTA(HTTPSpotProvider):
"mode": mode or "",
"callsign": spot.de_call,
"comments": spot.comment or "",
"type": "TEST" # todo replatce with NORMAL/QRT once testing complete
"type": "TEST", # todo replatce with NORMAL/QRT once testing complete
}
headers = {
**HTTP_HEADERS,
"Authorization": f"bearer {access_token}",
"id_token": id_token,
"Content-Type": "application/json",
}
headers = {**HTTP_HEADERS, "Authorization": f"bearer {access_token}", "id_token": id_token,
"Content-Type": "application/json"}
response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
if not response.ok:
raise RuntimeError(f"SOTA API returned {response.status_code!s}: {response.text}")
providers/spot/spot_provider.py
+1 -1
View File
@@ -32,7 +32,7 @@ class SpotProvider:
# Sort the batch so that earliest ones go in first. This helps keep the ordering correct when spots are fired
# off to SSE listeners.
spots = sorted(spots, key=lambda s: (s.time if s and s.time else 0))
spots = sorted(spots, key=lambda s: s.time if s and s.time else 0)
for spot in spots:
if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time:
# Fill in any blanks and add to the list
providers/spot/sse_spot_provider.py
+13 -7
View File
@@ -37,8 +37,7 @@ class SSESpotProvider(SpotProvider):
try:
event_source.close()
except Exception:
logging.exception(
f"Exception closing SSE connection for {self.name} during stop()")
logging.exception(f"Exception closing SSE connection for {self.name} during stop()")
if self._thread:
self._thread.join(timeout=15)
@@ -60,14 +59,20 @@ class SSESpotProvider(SpotProvider):
try:
logging.debug(f"Connecting to {self.name} spot API...")
self.status = "Connecting"
with EventSource(self._url, headers=HTTP_HEADERS, latest_event_id=self._last_event_id, timeout=10,
on_open=self._on_open, on_error=self._on_error) as event_source:
with EventSource(
self._url,
headers=HTTP_HEADERS,
latest_event_id=self._last_event_id,
timeout=10,
on_open=self._on_open,
on_error=self._on_error,
) as event_source:
self._set_event_source(event_source)
try:
for event in event_source:
if self._stop_event.is_set():
break
if event.type == 'message':
if event.type == "message":
try:
self._last_event_id = event.last_event_id
new_spot = self._sse_message_to_spot(event.data)
@@ -80,7 +85,8 @@ class SSESpotProvider(SpotProvider):
except Exception:
logging.exception(
f"Exception processing message from SSE Spot Provider ({self.name})")
f"Exception processing message from SSE Spot Provider ({self.name})"
)
finally:
self._set_event_source(None)
@@ -89,7 +95,7 @@ class SSESpotProvider(SpotProvider):
logging.exception(f"Exception in SSE Spot Provider ({self.name})")
else:
self.status = "Disconnected"
self._stop_event.wait(timeout=5) # Wait before trying to reconnect
self._stop_event.wait(timeout=5) # Wait before trying to reconnect
def _sse_message_to_spot(self, message_data):
"""Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass
providers/spot/tiles.py
+47 -26
View File
@@ -14,8 +14,22 @@ class Tiles(HTTPSpotProvider):
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/spots?active_hours=24"
SUBMIT_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/self-spot"
VALID_MODES = ["SSB", "CW", "FT8", "FT4", "FM", "DMR", "D-STAR", "M17", "AX.25", "JS8Call", "PSK31", "Olivia",
"VarAC", "Other"]
VALID_MODES = [
"SSB",
"CW",
"FT8",
"FT4",
"FM",
"DMR",
"D-STAR",
"M17",
"AX.25",
"JS8Call",
"PSK31",
"Olivia",
"VarAC",
"Other",
]
def __init__(self, provider_config):
super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -25,25 +39,33 @@ class Tiles(HTTPSpotProvider):
# Iterate through source data
for source_spot in http_response.json()["spots"]:
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["call_sign"].upper(),
# No separate spotter callsign, assume all spots are self-spots
de_call=source_spot["call_sign"].upper(),
freq=float(strip_extra_decimal_points(source_spot["frequency"])) * 1000000,
mode=source_spot["mode"].upper(),
comment=source_spot["notes"],
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["call_sign"].upper(),
# No separate spotter callsign, assume all spots are self-spots
de_call=source_spot["call_sign"].upper(),
freq=float(strip_extra_decimal_points(source_spot["frequency"])) * 1000000,
mode=source_spot["mode"].upper(),
comment=source_spot["notes"],
sig="Tiles",
# Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
# Just take the grid reference itself as the single Tiles SIG reference.
sig_refs=[
SIGRef(
id=source_spot["maidenhead_grid"],
sig="Tiles",
# Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
# Just take the grid reference itself as the single Tiles SIG reference.
sig_refs=[SIGRef(id=source_spot["maidenhead_grid"], sig="Tiles",
name=source_spot["maidenhead_grid"], latitude=source_spot["latitude"],
longitude=source_spot["longitude"])],
time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
dx_grid=source_spot["maidenhead_grid"],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
dx_location_source="GRID")
name=source_spot["maidenhead_grid"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
)
],
time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
dx_grid=source_spot["maidenhead_grid"],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
dx_location_source="GRID",
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
@@ -56,7 +78,6 @@ class Tiles(HTTPSpotProvider):
def submit_spot(self, spot, credentials):
# Tiles on the air currently only supports *self* spots
if spot.dx_call == spot.de_call:
# Figure out a valid mode. Borrowed this from PoLo :)
# https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js
if spot.mode:
@@ -80,24 +101,24 @@ class Tiles(HTTPSpotProvider):
"lat": spot.dx_latitude or None,
"lon": spot.dx_longitude or None,
"qrt": spot.qrt or False,
"pin": credentials.get("offline_spot_gateway_pin", "")
"pin": credentials.get("offline_spot_gateway_pin", ""),
}
headers = {**HTTP_HEADERS, "Content-Type": "application/json"}
response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
if not response.ok:
raise RuntimeError(
f"Tiles on the Air API returned {response.status_code!s}: {response.text}")
raise RuntimeError(f"Tiles on the Air API returned {response.status_code!s}: {response.text}")
else:
raise RuntimeError("The Tiles on the Air API requires a mode to be set.")
else:
raise RuntimeError(
"The Tiles on the Air API only supports self-spots, the DX call and spotter call must match.")
"The Tiles on the Air API only supports self-spots, the DX call and spotter call must match."
)
# Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles'
# strange frequency format where we can sometimes have e.g. "14.123.5".
def strip_extra_decimal_points(s):
parts = s.split('.', 1)
parts = s.split(".", 1)
if len(parts) == 1:
return s
return f"{parts[0]}.{parts[1].replace('.', '')}"
providers/spot/towers.py
+11 -8
View File
@@ -26,14 +26,17 @@ class Towers(HTTPSpotProvider):
likely_freq = float(source_spot["freq"]) * 1000
if likely_freq < 1000000:
likely_freq = likely_freq * 1000
spot = Spot(source=self.name,
dx_call=source_spot["call"].upper(),
freq=likely_freq,
comment=source_spot["comment"],
sig="Towers",
sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
time=datetime.strptime(response_json["updated"][:10] + source_spot["time"],
"%Y-%m-%d%H:%M").timestamp())
spot = Spot(
source=self.name,
dx_call=source_spot["call"].upper(),
freq=likely_freq,
comment=source_spot["comment"],
sig="Towers",
sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
time=datetime.strptime(
response_json["updated"][:10] + source_spot["time"], "%Y-%m-%d%H:%M"
).timestamp(),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
providers/spot/ukpacketnet.py
+38 -23
View File
@@ -36,42 +36,56 @@ class UKPacketNet(HTTPSpotProvider):
# First build a "full" comment combining some of the extra info
comment = listed_port["comment"] if "comment" in listed_port else ""
comment = f"{comment} {listed_port['mode']}" if "mode" in listed_port else comment
comment = f"{comment} {listed_port['modulation']}" if "modulation" in listed_port else comment
comment = f"{comment} {listed_port['baud']!s} baud" if "baud" in listed_port and listed_port[
"baud"] > 0 else comment
comment = (
f"{comment} {listed_port['modulation']}" if "modulation" in listed_port else comment
)
comment = (
f"{comment} {listed_port['baud']!s} baud"
if "baud" in listed_port and listed_port["baud"] > 0
else comment
)
# Get frequency from the comment if it's not set properly in the data structure. This is
# very hacky but a lot of node comments contain their frequency as the first or second
# word of their comment, but not in the proper data structure field.
freq = listed_port["freq"] if "freq" in listed_port and listed_port[
"freq"] > 0 else None
freq = (
listed_port["freq"] if "freq" in listed_port and listed_port["freq"] > 0 else None
)
if not freq and comment:
possible_freq = comment.split(" ")[0].upper().replace("MHZ", "")
if re.match(r"^[0-9.]+$",
possible_freq) and possible_freq != "1200" and possible_freq != "9600":
if (
re.match(r"^[0-9.]+$", possible_freq)
and possible_freq != "1200"
and possible_freq != "9600"
):
freq = float(possible_freq) * 1000000
if not freq and len(comment.split(" ")) > 1:
possible_freq = comment.split(" ")[1].upper().replace("MHZ", "")
if re.match(r"^[0-9.]+$",
possible_freq) and possible_freq != "1200" and possible_freq != "9600":
if (
re.match(r"^[0-9.]+$", possible_freq)
and possible_freq != "1200"
and possible_freq != "9600"
):
freq = float(possible_freq) * 1000000
# Check for a found frequency likely having been in kHz, sorry to all GHz packet folks
if freq and freq > 1000000000:
freq = freq / 1000
# Now build the spot object
spot = Spot(source=self.name,
dx_call=heard["callsign"].upper(),
de_call=node["callsign"].upper(),
freq=freq,
mode="PKT",
comment=comment,
time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp(),
de_grid=node["location"]["locator"] if "locator" in node[
"location"] else None,
de_latitude=node["location"]["coords"]["lat"],
de_longitude=node["location"]["coords"]["lon"])
spot = Spot(
source=self.name,
dx_call=heard["callsign"].upper(),
de_call=node["callsign"].upper(),
freq=freq,
mode="PKT",
comment=comment,
time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
de_grid=node["location"]["locator"] if "locator" in node["location"] else None,
de_latitude=node["location"]["coords"]["lat"],
de_longitude=node["location"]["coords"]["lon"],
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
@@ -84,8 +98,9 @@ class UKPacketNet(HTTPSpotProvider):
# data, and we can use that to look these up.
for spot in new_spots:
if spot.dx_call in nodes:
spot.dx_grid = nodes[spot.dx_call]["location"]["locator"] if "locator" in nodes[spot.dx_call][
"location"] else None
spot.dx_grid = (
nodes[spot.dx_call]["location"]["locator"] if "locator" in nodes[spot.dx_call]["location"] else None
)
spot.dx_latitude = nodes[spot.dx_call]["location"]["coords"]["lat"]
spot.dx_longitude = nodes[spot.dx_call]["location"]["coords"]["lon"]
providers/spot/websocket_spot_provider.py
+1 -2
View File
@@ -60,8 +60,7 @@ class WebsocketSpotProvider(SpotProvider):
logging.debug(f"Received data from {self.name} spot API.")
except Exception:
logging.exception(
f"Exception processing message from Websocket Spot Provider ({self.name})")
logging.exception(f"Exception processing message from Websocket Spot Provider ({self.name})")
except Exception as e:
self.status = "Error"
providers/spot/wota.py
+18 -13
View File
@@ -28,10 +28,13 @@ class WOTA(HTTPSpotProvider):
rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
# Iterate through source data
for source_spot in rss.channel.items:
try:
# Reject GUID missing or zero
if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0":
if (
not source_spot.guid
or not source_spot.guid.content
or source_spot.guid.content == "http://www.wota.org.uk/spots/0"
):
continue
# Pick apart the title
@@ -48,7 +51,7 @@ class WOTA(HTTPSpotProvider):
# Pick apart the description
desc_split = source_spot.description.split(". ")
freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
freq_mode_split = re.split(r'[\-\s]+', freq_mode)
freq_mode_split = re.split(r"[\-\s]+", freq_mode)
freq_hz = float(freq_mode_split[0].replace("'", ".")) * 1000000
mode = None
if len(freq_mode_split) > 1:
@@ -64,16 +67,18 @@ class WOTA(HTTPSpotProvider):
time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot.guid.content,
dx_call=dx_call,
de_call=spotter,
freq=freq_hz,
mode=mode,
comment=comment,
sig="WOTA",
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
time=time.timestamp())
spot = Spot(
source=self.name,
source_id=source_spot.guid.content,
dx_call=dx_call,
de_call=spotter,
freq=freq_hz,
mode=mode,
comment=comment,
sig="WOTA",
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
time=time.timestamp(),
)
new_spots.append(spot)
except Exception as e:
providers/spot/wwbota.py
+24 -17
View File
@@ -20,25 +20,32 @@ class WWBOTA(SSESpotProvider):
# n-fer activations.
refs = []
for ref in source_spot["references"]:
sigref = SIGRef(id=ref["reference"], sig="WWBOTA", name=ref["name"], latitude=ref["lat"],
longitude=ref["long"])
sigref = SIGRef(
id=ref["reference"],
sig="WWBOTA",
name=ref["name"],
latitude=ref["lat"],
longitude=ref["long"],
)
refs.append(sigref)
spot = Spot(source=self.name,
dx_call=source_spot["call"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["freq"]) * 1000000,
mode=source_spot["mode"].upper(),
comment=source_spot["comment"],
sig="WWBOTA",
sig_refs=refs,
time=datetime.fromisoformat(source_spot["time"].replace("Z", "+00:00")).timestamp(),
# WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
# now, we will just pick the first one to use as our grid, latitude and longitude.
dx_grid=source_spot["references"][0]["locator"],
dx_latitude=source_spot["references"][0]["lat"],
dx_longitude=source_spot["references"][0]["long"],
qrt=source_spot["type"] == "QRT")
spot = Spot(
source=self.name,
dx_call=source_spot["call"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["freq"]) * 1000000,
mode=source_spot["mode"].upper(),
comment=source_spot["comment"],
sig="WWBOTA",
sig_refs=refs,
time=datetime.fromisoformat(source_spot["time"].replace("Z", "+00:00")).timestamp(),
# WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
# now, we will just pick the first one to use as our grid, latitude and longitude.
dx_grid=source_spot["references"][0]["locator"],
dx_latitude=source_spot["references"][0]["lat"],
dx_longitude=source_spot["references"][0]["long"],
qrt=source_spot["type"] == "QRT",
)
# WWBOTA does support a special "Test" spot type, we need to avoid adding that.
return spot if source_spot["type"] != "Test" else None
providers/spot/wwff.py
+21 -12
View File
@@ -21,19 +21,28 @@ class WWFF(HTTPSpotProvider):
# Iterate through source data
for source_spot in http_response.json():
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["frequency_khz"]) * 1000,
mode=source_spot["mode"].upper(),
comment=source_spot["remarks"],
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["frequency_khz"]) * 1000,
mode=source_spot["mode"].upper(),
comment=source_spot["remarks"],
sig="WWFF",
sig_refs=[
SIGRef(
id=source_spot["reference"],
sig="WWFF",
sig_refs=[SIGRef(id=source_spot["reference"], sig="WWFF", name=source_spot["reference_name"],
latitude=source_spot["latitude"], longitude=source_spot["longitude"])],
time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"])
name=source_spot["reference_name"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
)
],
time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
providers/spot/xota.py
+17 -10
View File
@@ -29,14 +29,21 @@ class XOTA(WebsocketSpotProvider):
string = b.decode("utf-8")
source_spot = json.loads(string)
ref_id = f"{self._sig_ref_prefix} {source_spot['reference']['title']}"
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["stationCallSign"].upper(),
freq=float(source_spot["freq"]) * 1000,
mode=source_spot["mode"].upper(),
sig=self.SIG,
sig_refs=[
SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"])],
time=datetime.now(pytz.UTC).timestamp(),
qrt=source_spot["state"] != "active")
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["stationCallSign"].upper(),
freq=float(source_spot["freq"]) * 1000,
mode=source_spot["mode"].upper(),
sig=self.SIG,
sig_refs=[
SIGRef(
id=ref_id,
sig=self.SIG or "",
url=source_spot["reference"]["website"],
)
],
time=datetime.now(pytz.UTC).timestamp(),
qrt=source_spot["state"] != "active",
)
return spot
providers/spot/zlota.py
+19 -10
View File
@@ -26,17 +26,26 @@ class ZLOTA(HTTPSpotProvider):
freq_hz = freq_hz * 1000
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=freq_hz,
mode=source_spot["mode"].upper().strip(),
comment=source_spot["comments"],
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=freq_hz,
mode=source_spot["mode"].upper().strip(),
comment=source_spot["comments"],
sig="ZLOTA",
sig_refs=[
SIGRef(
id=source_spot["reference"],
sig="ZLOTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="ZLOTA", name=source_spot["name"])],
time=datetime.fromisoformat(source_spot["referenced_time"].replace("Z", "+00:00")).astimezone(
pytz.UTC).timestamp())
name=source_spot["name"],
)
],
time=datetime.fromisoformat(source_spot["referenced_time"].replace("Z", "+00:00"))
.astimezone(pytz.UTC)
.timestamp(),
)
new_spots.append(spot)
return new_spots
providers/staticdata/cqzonedata.py
+4 -2
View File
@@ -5,7 +5,9 @@ import geopandas
from shapely import prepare
from core.data_store import DATA_STORE
from providers.staticdata.local_file_static_data_provider import LocalFileStaticDataProvider
from providers.staticdata.local_file_static_data_provider import (
LocalFileStaticDataProvider,
)
class CQZoneData(LocalFileStaticDataProvider):
@@ -21,7 +23,7 @@ class CQZoneData(LocalFileStaticDataProvider):
with open(path) as f:
cq_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
for idx in cq_zone_data.index:
prepare(cq_zone_data.at[idx, 'geometry'])
prepare(cq_zone_data.at[idx, "geometry"])
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to prepare the rest
# of the data in this case
providers/staticdata/file_download_static_data_provider.py
+8 -7
View File
@@ -1,6 +1,6 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
from requests import ReadTimeout
@@ -13,10 +13,10 @@ from providers.staticdata.static_data_provider import StaticDataProvider
class FileDownloadStaticDataProvider(StaticDataProvider):
"""Generic static reference data provider class for providers that fetch their data from the web by downloading a
file."""
file."""
def __init__(self, name, provider_config, url, poll_interval):
""" Set up the provider, note poll_interval is in *days*."""
"""Set up the provider, note poll_interval is in *days*."""
super().__init__(name, provider_config)
self._url = url
self._poll_interval = poll_interval
@@ -27,8 +27,7 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting.
logging.info(
f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.")
logging.info(f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}")
self._thread.start()
@@ -57,7 +56,9 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
logging.info(f"Updated static reference data for {self.name}")
else:
self.status = "Error"
logging.warning(f"HTTP {http_response.status_code} when downloading static reference data for {self.name}.")
logging.warning(
f"HTTP {http_response.status_code} when downloading static reference data for {self.name}."
)
except ConnectionError:
self.status = "Error"
@@ -72,6 +73,6 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
def _handle_http_response(self, http_response):
"""Handle an HTTP response returned by the server and load the data from it. Return true if successful,
false otherwise."""
false otherwise."""
raise NotImplementedError("Subclasses must implement this method")
providers/staticdata/ituzonedata.py
+4 -2
View File
@@ -5,7 +5,9 @@ import geopandas
from shapely import prepare
from core.data_store import DATA_STORE
from providers.staticdata.local_file_static_data_provider import LocalFileStaticDataProvider
from providers.staticdata.local_file_static_data_provider import (
LocalFileStaticDataProvider,
)
class ITUZoneData(LocalFileStaticDataProvider):
@@ -21,7 +23,7 @@ class ITUZoneData(LocalFileStaticDataProvider):
with open(path) as f:
itu_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
for idx in itu_zone_data.index:
prepare(itu_zone_data.at[idx, 'geometry'])
prepare(itu_zone_data.at[idx, "geometry"])
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to prepare the rest
# of the data in this case
providers/staticdata/k0swe.py
+3 -3
View File
@@ -1,7 +1,9 @@
import logging
from core.data_store import DATA_STORE
from providers.staticdata.file_download_static_data_provider import FileDownloadStaticDataProvider
from providers.staticdata.file_download_static_data_provider import (
FileDownloadStaticDataProvider,
)
class K0SWE(FileDownloadStaticDataProvider):
@@ -44,5 +46,3 @@ class K0SWE(FileDownloadStaticDataProvider):
except Exception:
logging.exception("Exception when loading K0SWE dxcc.json.")
return False
providers/staticdata/static_data_provider.py
-2
View File
@@ -15,13 +15,11 @@ class StaticDataProvider:
self.status = "Not Started" if self.enabled else "Disabled"
self.reference_count = 0
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
raise NotImplementedError("Subclasses must implement this method")
def stop(self):
"""Stop any threads and prepare for application shutdown"""
requirements.txt
+2 -1
View File
@@ -19,4 +19,5 @@ tornado_eventsource~=3.0.0
geopandas~=0.13.2
simplejson~=4.1.1
cachetools~=7.1.6
fastkml~=1.4.0
fastkml~=1.4.0
ruff~=0.16.3
server/handlers/api/addspot.py
+39 -24
View File
@@ -14,8 +14,7 @@ from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRE
from core.constants import UNKNOWN_BAND
from core.prometheus_metrics_handler import api_requests_counter
from core.sig_utils import get_ref_regex_for_sig
from core.utils import infer_band_from_freq
from core.utils import safe_json_dumps
from core.utils import infer_band_from_freq, safe_json_dumps
from data.spot import Spot
from providers.spot.spot_provider import SpotProvider
@@ -25,7 +24,12 @@ RECAPTCHA_VERIFY_URL = "https://www.google.com/recaptcha/api/siteverify"
class APISpotHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/spot (POST)"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._spots = None
self._web_server_metrics = None
self._spot_providers = None
@@ -53,7 +57,7 @@ class APISpotHandler(tornado.web.RequestHandler):
return
# Reject if format not json
if not self.request.headers.get('Content-Type', '').startswith("application/json"):
if not self.request.headers.get("Content-Type", "").startswith("application/json"):
self.set_status(415)
self.write(safe_json_dumps("Error - request Content-Type must be application/json"))
self.set_header("Cache-Control", "no-store")
@@ -82,13 +86,10 @@ class APISpotHandler(tornado.web.RequestHandler):
upstream_credentials = handling.get("upstream_credentials", {})
captcha_token = handling.get("captcha_token", None)
# Spothole v2.0 release only: deny upstream spotting. Spothole API breaking changes were in v2.0 but
# functionality is not ready yet. TODO
submit_upstream = False
# Verify CAPTCHA if required
if RECAPTCHA_SECRET_KEY:
if not captcha_token:
@@ -111,7 +112,8 @@ class APISpotHandler(tornado.web.RequestHandler):
if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
self.set_status(422)
self.write(
safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum."))
safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.")
)
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -133,29 +135,37 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject if frequency not in a known band
if infer_band_from_freq(spot.freq) == UNKNOWN_BAND:
self.set_status(422)
self.write(
safe_json_dumps(f"Error - Frequency of {spot.freq / 1000.0!s}kHz is not in a known band."))
self.write(safe_json_dumps(f"Error - Frequency of {spot.freq / 1000.0!s}kHz is not in a known band."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
# Reject if grid formatting incorrect
if spot.dx_grid and not re.match(
r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
spot.dx_grid.upper()):
r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
spot.dx_grid.upper(),
):
self.set_status(422)
self.write(
safe_json_dumps(f"Error - '{spot.dx_grid}' does not look like a valid Maidenhead grid."))
self.write(safe_json_dumps(f"Error - '{spot.dx_grid}' does not look like a valid Maidenhead grid."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
# Reject if sig_ref format incorrect for sig
if spot.sig and spot.sig_refs and len(spot.sig_refs) > 0 and spot.sig_refs[0].id and get_ref_regex_for_sig(
spot.sig) and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id):
if (
spot.sig
and spot.sig_refs
and len(spot.sig_refs) > 0
and spot.sig_refs[0].id
and get_ref_regex_for_sig(spot.sig)
and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id)
):
self.set_status(422)
self.write(safe_json_dumps(
f"Error - '{spot.sig_refs[0].id}' does not look like a valid reference for {spot.sig}."))
self.write(
safe_json_dumps(
f"Error - '{spot.sig_refs[0].id}' does not look like a valid reference for {spot.sig}."
)
)
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -185,7 +195,8 @@ class APISpotHandler(tornado.web.RequestHandler):
if not spot.dx_grid and upstream_provider_name == "Tiles":
self.set_status(422)
self.write(
safe_json_dumps("Error - a grid reference is required to submit upstream to Tiles on the Air."))
safe_json_dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.")
)
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -210,7 +221,9 @@ class APISpotHandler(tornado.web.RequestHandler):
upstream_warning = str(e)
except Exception:
logging.exception(f"Failed to submit spot upstream to {upstream_provider_name}")
upstream_warning = f"Spot was saved locally but upstream submission to {upstream_provider_name} failed."
upstream_warning = (
f"Spot was saved locally but upstream submission to {upstream_provider_name} failed."
)
else:
upstream_warning = f"No enabled provider named '{upstream_provider_name}' supports upstream submission for {spot.sig if spot.sig else ''} spots."
@@ -250,10 +263,12 @@ class APISpotHandler(tornado.web.RequestHandler):
"""Verify a Google reCAPTCHA v2 token. Returns True if valid."""
try:
response = requests.post(RECAPTCHA_VERIFY_URL,
data={"secret": RECAPTCHA_SECRET_KEY, "response": token},
timeout=(5, 10))
response = requests.post(
RECAPTCHA_VERIFY_URL,
data={"secret": RECAPTCHA_SECRET_KEY, "response": token},
timeout=(5, 10),
)
return response.ok and response.json().get("success", False)
except Exception:
logging.exception(f"reCAPTCHA verification request failed")
logging.exception("reCAPTCHA verification request failed")
return False
server/handlers/api/alerts.py
+19 -10
View File
@@ -17,7 +17,12 @@ from data.lookup_credentials import extract_credentials
class APIAlertsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/alerts"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._alerts = None
self._web_server_metrics = None
super().__init__(application, request, **kwargs)
@@ -82,8 +87,7 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
def custom_headers(self):
"""Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data"""
return {"Cache-Control": "no-store",
"X-Accel-Buffering": "no"}
return {"Cache-Control": "no-store", "X-Accel-Buffering": "no"}
def open(self):
try:
@@ -142,10 +146,10 @@ def get_alert_list_with_filters(all_alerts, query):
a = all_alerts.get(k)
if a is not None:
alerts.append(a)
alerts = sorted(alerts, key=lambda alert: (alert.start_time if alert and alert.start_time else 0))
alerts = sorted(alerts, key=lambda alert: alert.start_time if alert and alert.start_time else 0)
alerts = list(filter(lambda alert: alert_allowed_by_query(alert, query), alerts))
if "limit" in query.keys():
alerts = alerts[:int(query.get("limit"))]
alerts = alerts[: int(query.get("limit"))]
return alerts
@@ -164,8 +168,11 @@ def alert_allowed_by_query(alert, query):
# Check the duration if end_time is provided. If end_time is not provided, assume the activation is
# "short", i.e. it always passes this check. If dxpeditions_skip_max_duration_check is true and
# the alert is a dxpedition, it also always passes the check.
if alert.is_dxpedition and (query.get(
"dxpeditions_skip_max_duration_check").upper() == "TRUE" if "dxpeditions_skip_max_duration_check" in query.keys() else False):
if alert.is_dxpedition and (
query.get("dxpeditions_skip_max_duration_check").upper() == "TRUE"
if "dxpeditions_skip_max_duration_check" in query.keys()
else False
):
continue
if alert.end_time and alert.start_time and alert.end_time - alert.start_time > max_duration:
return False
@@ -192,8 +199,10 @@ def alert_allowed_by_query(alert, query):
return False
case "text_includes":
text_includes = query.get(k).strip()
if (not alert.dx_call or text_includes.upper() not in alert.dx_call.upper()) \
and (not alert.comment or text_includes.upper() not in alert.comment.upper()) \
and (not alert.freqs_modes or text_includes.upper() not in alert.freqs_modes.upper()):
if (
(not alert.dx_call or text_includes.upper() not in alert.dx_call.upper())
and (not alert.comment or text_includes.upper() not in alert.comment.upper())
and (not alert.freqs_modes or text_includes.upper() not in alert.freqs_modes.upper())
):
return False
return True
server/handlers/api/dxstats.py
+12 -4
View File
@@ -21,7 +21,12 @@ BANDS_SET = frozenset(BANDS)
class APIDxStatsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/dxstats"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._spots = None
self._web_server_metrics = None
super().__init__(application, request, **kwargs)
@@ -46,12 +51,15 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
continue
if not spot.time or spot.time < one_hour_ago:
continue
if spot.de_continent in CONTINENTS_SET and spot.dx_continent in CONTINENTS_SET and spot.band in BANDS_SET:
if (
spot.de_continent in CONTINENTS_SET
and spot.dx_continent in CONTINENTS_SET
and spot.band in BANDS_SET
):
counts[spot.de_continent, spot.dx_continent, spot.band] += 1
result = {
de: {dx: {band: counts[de, dx, band] for band in BANDS} for dx in CONTINENTS}
for de in CONTINENTS
de: {dx: {band: counts[de, dx, band] for band in BANDS} for dx in CONTINENTS} for de in CONTINENTS
}
self.write(json.dumps(result))
server/handlers/api/lookups.py
+32 -11
View File
@@ -10,7 +10,11 @@ from tornado.web import Application
from core.call_lookup_helper import get_call_info
from core.constants import SIGS
from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone
from core.geo_utils import (
lat_lon_for_grid_sw_corner_plus_size,
lat_lon_to_cq_zone,
lat_lon_to_itu_zone,
)
from core.prometheus_metrics_handler import api_requests_counter
from core.sig_lookup_helper import populate_missing_sig_ref_info
from core.sig_utils import get_ref_regex_for_sig
@@ -22,7 +26,12 @@ from data.sig_ref import SIGRef
class APILookupCallHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/lookup/call"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._web_server_metrics = None
super().__init__(application, request, **kwargs)
@@ -42,7 +51,7 @@ class APILookupCallHandler(tornado.web.RequestHandler):
query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
# The "call" query param must exist and look like a callsign
if "call" in query_params.keys():
if "call" in query_params:
call = str(query_params.get("call")).upper()
if re.match(r"^[A-Z0-9/\-]*$", call):
credentials = extract_credentials(self.request.headers)
@@ -68,7 +77,12 @@ class APILookupCallHandler(tornado.web.RequestHandler):
class APILookupSIGRefHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/lookup/sigref"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._web_server_metrics = None
super().__init__(application, request, **kwargs)
@@ -89,7 +103,7 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
# "sig" and "id" query params must exist, SIG must be known, and if we have a reference regex for that SIG,
# the provided id must match it.
if "sig" in query_params.keys() and "id" in query_params.keys():
if "sig" in query_params and "id" in query_params:
sig = str(query_params.get("sig")).upper()
ref_id = str(query_params.get("id")).upper()
if sig in list(map(lambda p: p.name.upper(), SIGS)):
@@ -98,8 +112,9 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
self.write(safe_json_dumps(data))
else:
self.write(safe_json_dumps(
f"Error - '{ref_id}' does not look like a valid reference ID for {sig}."))
self.write(
safe_json_dumps(f"Error - '{ref_id}' does not look like a valid reference ID for {sig}.")
)
self.set_status(422)
else:
self.write(safe_json_dumps(f"Error - sig '{sig}' is not known."))
@@ -120,7 +135,12 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
class APILookupGridHandler(tornado.web.RequestHandler):
"""API request handler for /api/v2/lookup/grid"""
def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
def __init__(
self,
application: "Application",
request: httputil.HTTPServerRequest,
**kwargs: Any,
):
self._web_server_metrics = None
super().__init__(application, request, **kwargs)
@@ -140,7 +160,7 @@ class APILookupGridHandler(tornado.web.RequestHandler):
query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
# "grid" query param must exist.
if "grid" in query_params.keys():
if "grid" in query_params:
grid = str(query_params.get("grid")).upper()
lat, lon, lat_cell_size, lon_cell_size = lat_lon_for_grid_sw_corner_plus_size(grid)
if lat is not None and lon is not None and lat_cell_size is not None and lon_cell_size is not None:
@@ -154,7 +174,7 @@ class APILookupGridHandler(tornado.web.RequestHandler):
"latitude": center_lat,
"longitude": center_lon,
"cq_zone": center_cq_zone,
"itu_zone": center_itu_zone
"itu_zone": center_itu_zone,
},
"southwest": {
"latitude": lat,
@@ -163,7 +183,8 @@ class APILookupGridHandler(tornado.web.RequestHandler):
"northeast": {
"latitude": lat + lat_cell_size,
"longitude": lon + lon_cell_size,
}}
},
}
self.write(safe_json_dumps(response))
else:

Some files were not shown because too many files have changed in this diff Show More