mirror of
https://git.ianrenton.com/ian/spothole.git
synced 2026-09-20 22:37:44 +00:00
Autogenerated type safety parameterisation of all methods
This commit is contained in:
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
@@ -12,7 +14,7 @@ from data.activity_ref import ActivityRef
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_activity_ref_info(activity_name, ref_id):
|
||||
def get_activity_ref_info(activity_name: str, ref_id: str) -> ActivityRef | None:
|
||||
"""Look up details of an activity reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in an
|
||||
activity name and a reference ID (both strings) and returns an ActivityRef object populated with as much data
|
||||
as we can find. This makes use of activity ref data in the data store, live lookups from the web, or just
|
||||
@@ -139,7 +141,7 @@ def get_activity_ref_info(activity_name, ref_id):
|
||||
return activity_ref
|
||||
|
||||
|
||||
def populate_missing_activity_ref_info(activity_ref):
|
||||
def populate_missing_activity_ref_info(activity_ref: ActivityRef) -> ActivityRef:
|
||||
"""Look up details of an activity reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in an
|
||||
activity_ref object which must at minimum have a "sig" and an "id". The rest of the object will be populated
|
||||
and returned. Any data currently in the object will be kept, only missing data in the object will be populated
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activities import ACTIVITIES
|
||||
from data.activity import Activity
|
||||
|
||||
|
||||
def get_activity_by_name(name):
|
||||
def get_activity_by_name(name: str) -> Activity | None:
|
||||
"""Utility function to resolve an arbitrary, case-insensitive activity name string (e.g. from a spot comment, a
|
||||
provider, or an API request) to the matching known Activity. Returns None if no match is found."""
|
||||
|
||||
@@ -13,7 +17,7 @@ def get_activity_by_name(name):
|
||||
return None
|
||||
|
||||
|
||||
def get_ref_regex_for_activity(activity):
|
||||
def get_ref_regex_for_activity(activity: str) -> str | None:
|
||||
"""Utility function to get the regex string for an activity reference for a named activity. If no match is
|
||||
found, None will be returned."""
|
||||
|
||||
@@ -21,14 +25,14 @@ def get_ref_regex_for_activity(activity):
|
||||
return found.ref_regex if found else None
|
||||
|
||||
|
||||
def get_icon_for_activity(activity):
|
||||
def get_icon_for_activity(activity: str) -> str | None:
|
||||
"""Utility function to get the icon for a named activity. If no match is found, None will be returned."""
|
||||
|
||||
found = get_activity_by_name(activity)
|
||||
return found.icon if found else None
|
||||
|
||||
|
||||
def get_activity_name_from_comment_name(activity):
|
||||
def get_activity_name_from_comment_name(activity: str) -> ActivityName | None:
|
||||
"""Utility function to get the name of an activity from its "comment name". Generally these will be the same
|
||||
but there are some cases (e.g. is "TOTA" Towers, Tiles or Toilets?) where we need to transform one to the
|
||||
other."""
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from core.data_providers import DATA_PROVIDERS
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
|
||||
|
||||
def get_call_info(callsign, lookup_credentials):
|
||||
def get_call_info(callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign:
|
||||
"""Utility method to get the best set of data for a callsign as we can, using all enabled providers.
|
||||
lookup_credentials is an optional object that carries the user's QRZ.com/HamQTH credentials, if they provided them,
|
||||
to enable lookup using those providers."""
|
||||
|
||||
+10
-8
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
@@ -12,25 +14,25 @@ logger = logging.getLogger(__name__)
|
||||
class CleanupTimer:
|
||||
"""Provides a timed cleanup of the spot list."""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""Constructor"""
|
||||
|
||||
self._cleanup_interval = None
|
||||
self._cleanup_interval: float | None = None
|
||||
self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Starting"
|
||||
self._thread = None
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event = Event()
|
||||
|
||||
def setup(self, cleanup_interval):
|
||||
def setup(self, cleanup_interval: float) -> None:
|
||||
self._cleanup_interval = cleanup_interval
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the cleanup timer"""
|
||||
|
||||
self._thread = Thread(target=self._run, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
self._stop_event.set()
|
||||
@@ -39,11 +41,11 @@ class CleanupTimer:
|
||||
if self._thread.is_alive():
|
||||
logger.warning("Cleanup worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while not self._stop_event.wait(timeout=self._cleanup_interval):
|
||||
self._cleanup()
|
||||
|
||||
def _cleanup(self):
|
||||
def _cleanup(self) -> None:
|
||||
"""Perform cleanup and reschedule next timer"""
|
||||
|
||||
try:
|
||||
|
||||
+21
-18
@@ -1,7 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
@@ -16,25 +19,25 @@ if not os.path.isfile("config.yml"):
|
||||
|
||||
# Load config
|
||||
with open("config.yml") as f:
|
||||
config = yaml.safe_load(f)
|
||||
config: dict[str, Any] = yaml.safe_load(f)
|
||||
logger.info("Loaded config.")
|
||||
|
||||
BASE_URL = config.get("base_url", "http://localhost:8080")
|
||||
MAX_SPOT_AGE = config.get("max_spot_age_sec", 3600)
|
||||
MAX_ALERT_AGE = config.get("max_alert_age_sec", 604800)
|
||||
SERVER_OWNER_CALLSIGN = config.get("server_owner_callsign", "N0CALL")
|
||||
WEB_SERVER_PORT = config.get("web_server_port", 8080)
|
||||
TELNET_SERVER_ENABLED = config.get("telnet_server_enabled", False)
|
||||
TELNET_SERVER_ADDRESS = config.get("telnet_server_address", "localhost")
|
||||
TELNET_SERVER_PORT = config.get("telnet_server_port", 7373)
|
||||
ALLOW_SPOTTING = config.get("allow_spotting", True)
|
||||
ALLOW_UPSTREAM_SPOTTING = config.get("allow_upstream_spotting", True)
|
||||
WEB_UI_OPTIONS = config.get("web_ui_options", {})
|
||||
API_ONLY_MODE = config.get("api_only_mode", False)
|
||||
RECAPTCHA_SECRET_KEY = config.get("recaptcha_secret_key", "")
|
||||
RECAPTCHA_SITE_KEY = config.get("recaptcha_site_key", "")
|
||||
LOG_LEVEL = config.get("log_level", "INFO")
|
||||
LOG_WEB_REQUESTS = config.get("log_web_requests", False)
|
||||
BASE_URL: str = config.get("base_url", "http://localhost:8080")
|
||||
MAX_SPOT_AGE: int = config.get("max_spot_age_sec", 3600)
|
||||
MAX_ALERT_AGE: int = config.get("max_alert_age_sec", 604800)
|
||||
SERVER_OWNER_CALLSIGN: str = config.get("server_owner_callsign", "N0CALL")
|
||||
WEB_SERVER_PORT: int = config.get("web_server_port", 8080)
|
||||
TELNET_SERVER_ENABLED: bool = config.get("telnet_server_enabled", False)
|
||||
TELNET_SERVER_ADDRESS: str = config.get("telnet_server_address", "localhost")
|
||||
TELNET_SERVER_PORT: int = config.get("telnet_server_port", 7373)
|
||||
ALLOW_SPOTTING: bool = config.get("allow_spotting", True)
|
||||
ALLOW_UPSTREAM_SPOTTING: bool = config.get("allow_upstream_spotting", True)
|
||||
WEB_UI_OPTIONS: dict[str, Any] = config.get("web_ui_options", {})
|
||||
API_ONLY_MODE: bool = config.get("api_only_mode", False)
|
||||
RECAPTCHA_SECRET_KEY: str = config.get("recaptcha_secret_key", "")
|
||||
RECAPTCHA_SITE_KEY: str = config.get("recaptcha_site_key", "")
|
||||
LOG_LEVEL: str = config.get("log_level", "INFO")
|
||||
LOG_WEB_REQUESTS: bool = config.get("log_web_requests", False)
|
||||
|
||||
WEB_UI_OPTIONS["qrz_enabled"] = any(p["class"] == "QRZ" and p["enabled"] for p in config["callsign_data_providers"])
|
||||
WEB_UI_OPTIONS["hamqth_enabled"] = any(
|
||||
@@ -44,7 +47,7 @@ WEB_UI_OPTIONS["recaptcha_site_key"] = RECAPTCHA_SITE_KEY
|
||||
WEB_UI_OPTIONS["allow_upstream_spotting"] = ALLOW_SPOTTING and ALLOW_UPSTREAM_SPOTTING
|
||||
|
||||
|
||||
def create_provider_from_config(package, config_providers_entry):
|
||||
def create_provider_from_config(package: str, config_providers_entry: dict[str, Any]) -> Any:
|
||||
"""Utility method to get a provider based on the class specified in its config entry. You must also provide the
|
||||
package to look for it in, as there are several types of provider. e.g. package "providers.spot", where the config
|
||||
entry is for a POTA spot provider."""
|
||||
|
||||
+8
-6
@@ -1,15 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from core.config import SERVER_OWNER_CALLSIGN
|
||||
from data.band import Band
|
||||
|
||||
# General software
|
||||
SOFTWARE_VERSION = "2.2-pre"
|
||||
SOFTWARE_VERSION: str = "2.2-pre"
|
||||
|
||||
# HTTP headers used for spot providers that use HTTP
|
||||
HTTP_HEADERS = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"}
|
||||
HAMQTH_PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}".replace(" ", "_")
|
||||
HTTP_HEADERS: dict[str, str] = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"}
|
||||
HAMQTH_PRG: str = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}".replace(" ", "_")
|
||||
|
||||
# Band definitions
|
||||
BANDS = [
|
||||
BANDS: list[Band] = [
|
||||
Band(name="2200m", start_freq=135700, end_freq=137800),
|
||||
Band(name="600m", start_freq=472000, end_freq=479000),
|
||||
Band(name="160m", start_freq=1800000, end_freq=2000000, is_ham_hf=True),
|
||||
@@ -37,11 +39,11 @@ BANDS = [
|
||||
Band(name="47GHz", start_freq=47000000000, end_freq=47200000000),
|
||||
Band(name="76GHz", start_freq=75500000000, end_freq=81500000000),
|
||||
]
|
||||
UNKNOWN_BAND = Band(name="Unknown", start_freq=0, end_freq=0)
|
||||
UNKNOWN_BAND: Band = Band(name="Unknown", start_freq=0, end_freq=0)
|
||||
|
||||
# Propagation modes used in VHF/UHF DX cluster comments, e.g. "JN61ES<ES>JM56XT". I don't think there's an official list
|
||||
# of these anywhere, but here are some I've seen or seen reference to
|
||||
PROPAGATION_MODES = {
|
||||
PROPAGATION_MODES: dict[str, str] = {
|
||||
"F2": "F2 layer ionospheric",
|
||||
"ES": "Sporadic-E",
|
||||
"TR": "Tropospheric ducting",
|
||||
|
||||
+28
-13
@@ -1,8 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
from core.config import config, create_provider_from_config
|
||||
from providers.activityrefdata.activity_ref_data_provider import ActivityRefDataProvider
|
||||
from providers.alert.alert_provider import AlertProvider
|
||||
from providers.callsigndata.callsign_data_provider import CallsignDataProvider
|
||||
from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider
|
||||
from providers.spot.spot_provider import SpotProvider
|
||||
from providers.staticdata.static_data_provider import StaticDataProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -10,16 +18,16 @@ logger = logging.getLogger(__name__)
|
||||
class DataProviders:
|
||||
"""Global object for storing data providers."""
|
||||
|
||||
def __init__(self):
|
||||
self.spot_providers = []
|
||||
self.alert_providers = []
|
||||
self.solar_condition_providers = []
|
||||
self.static_data_providers = []
|
||||
self.sig_ref_data_providers = []
|
||||
self.callsign_data_providers = []
|
||||
self._startup_timers = []
|
||||
def __init__(self) -> None:
|
||||
self.spot_providers: list[SpotProvider] = []
|
||||
self.alert_providers: list[AlertProvider] = []
|
||||
self.solar_condition_providers: list[SolarConditionsProvider] = []
|
||||
self.static_data_providers: list[StaticDataProvider] = []
|
||||
self.sig_ref_data_providers: list[ActivityRefDataProvider] = []
|
||||
self.callsign_data_providers: list[CallsignDataProvider] = []
|
||||
self._startup_timers: list[threading.Timer] = []
|
||||
|
||||
def setup(self):
|
||||
def setup(self) -> None:
|
||||
for entry in config["spot_providers"]:
|
||||
self.spot_providers.append(create_provider_from_config("providers.spot", entry))
|
||||
for entry in config["alert_providers"]:
|
||||
@@ -34,7 +42,12 @@ class DataProviders:
|
||||
self.callsign_data_providers.append(create_provider_from_config("providers.callsigndata", entry))
|
||||
|
||||
@staticmethod
|
||||
def start_providers(providers, provider_type):
|
||||
def start_providers(
|
||||
providers: list[
|
||||
SpotProvider | AlertProvider | SolarConditionsProvider | StaticDataProvider | ActivityRefDataProvider | CallsignDataProvider
|
||||
],
|
||||
provider_type: str,
|
||||
) -> None:
|
||||
"""Helper method to activate enabled providers in the list."""
|
||||
|
||||
logger.info(f"Starting {provider_type} providers...")
|
||||
@@ -42,7 +55,7 @@ class DataProviders:
|
||||
if p.enabled:
|
||||
p.start()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Start data providers before spot/alert providers so the lookup data is there already for incoming spots.
|
||||
# Each category is fired off after a small delay to give the rest of Spothole chance to start up.
|
||||
self._startup_timers = [
|
||||
@@ -60,7 +73,7 @@ class DataProviders:
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
# Cancel any startup timers that haven't fired yet
|
||||
for t in self._startup_timers:
|
||||
t.cancel()
|
||||
@@ -81,7 +94,9 @@ class DataProviders:
|
||||
if not all_providers:
|
||||
return
|
||||
|
||||
def stop_provider(p):
|
||||
def stop_provider(
|
||||
p: SpotProvider | AlertProvider | SolarConditionsProvider | StaticDataProvider | ActivityRefDataProvider | CallsignDataProvider,
|
||||
) -> None:
|
||||
try:
|
||||
p.stop()
|
||||
except Exception:
|
||||
|
||||
+27
-18
@@ -1,14 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import diskcache
|
||||
import geopandas
|
||||
|
||||
from core.config import MAX_ALERT_AGE, MAX_SPOT_AGE
|
||||
from core.live_data_cache import LiveDataCache
|
||||
from core.single_object_data_cache import SingleObjectDataCache
|
||||
from data.solar_conditions import SolarConditions
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Deferred to avoid a circular import: data.alert and data.spot both import core.data_store at module level.
|
||||
from data.alert import Alert
|
||||
from data.spot import Spot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CACHE_DIR = "./cache/"
|
||||
@@ -18,31 +27,31 @@ class DataStore:
|
||||
"""Data caching/storage object. Handles storage of spots, alerts, solar conditions, activity reference data, and
|
||||
callsign lookup data using different caching strategies for each."""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
# Constants
|
||||
self._MAX_SPOT_COUNT = 100000
|
||||
self._MAX_ALERT_COUNT = 100000
|
||||
self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
|
||||
self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
|
||||
# Caches
|
||||
self.alerts = None
|
||||
self.spots = None
|
||||
self.callsign_data_countryfiles = None
|
||||
self.callsign_data_clublogxml = None
|
||||
self.callsign_data_clublogapi = None
|
||||
self.callsign_data_qrz = None
|
||||
self.callsign_data_hamqth = None
|
||||
self.dxcc_data = None
|
||||
self.dxcc_lookup_by_call_regex = []
|
||||
self.activity_refs = None
|
||||
self.status = None
|
||||
self.solar_conditions = None
|
||||
self.alerts: LiveDataCache[Alert] | None = None
|
||||
self.spots: LiveDataCache[Spot] | None = None
|
||||
self.callsign_data_countryfiles: diskcache.Cache | None = None
|
||||
self.callsign_data_clublogxml: diskcache.Cache | None = None
|
||||
self.callsign_data_clublogapi: diskcache.Cache | None = None
|
||||
self.callsign_data_qrz: diskcache.Cache | None = None
|
||||
self.callsign_data_hamqth: diskcache.Cache | None = None
|
||||
self.dxcc_data: diskcache.Cache | None = None
|
||||
self.dxcc_lookup_by_call_regex: list[tuple[re.Pattern[str], Any]] = []
|
||||
self.activity_refs: diskcache.Cache | None = None
|
||||
self.status: SingleObjectDataCache[dict[str, Any]] | None = None
|
||||
self.solar_conditions: SingleObjectDataCache[SolarConditions] | None = None
|
||||
# ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
|
||||
# caches, they can just be straight objects
|
||||
self.cq_zone_data = None
|
||||
self.itu_zone_data = None
|
||||
self.cq_zone_data: geopandas.GeoDataFrame | None = None
|
||||
self.itu_zone_data: geopandas.GeoDataFrame | None = None
|
||||
|
||||
def setup(self):
|
||||
def setup(self) -> None:
|
||||
Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# For solar data and status data, we use a wrapper around disk cache where each cache contains only a single
|
||||
@@ -100,7 +109,7 @@ class DataStore:
|
||||
)
|
||||
logger.info(f"Loaded {len(self.alerts.keys())} alerts from a previous run.")
|
||||
|
||||
def regenerate_call_regex_to_dxcc_entity_map(self):
|
||||
def regenerate_call_regex_to_dxcc_entity_map(self) -> None:
|
||||
"""DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC
|
||||
entity it belongs to. But getting every DXCC entity data object out of DiskCache, iterating, compiling its regex
|
||||
and testing the callsign every time is expensive. So instead we build a separate in-memory lookup of compiled
|
||||
@@ -110,7 +119,7 @@ class DataStore:
|
||||
for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]:
|
||||
self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
|
||||
|
||||
def close(self):
|
||||
def close(self) -> None:
|
||||
self.spots.close()
|
||||
self.alerts.close()
|
||||
self.solar_conditions.close()
|
||||
|
||||
+5
-3
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
|
||||
@@ -44,7 +46,7 @@ class Mode(str, Enum):
|
||||
return not (self.is_cw or self.is_phone)
|
||||
|
||||
@staticmethod
|
||||
def from_name(name):
|
||||
def from_name(name: str) -> Mode | None:
|
||||
"""Convert a string to an enum mode using the alias table."""
|
||||
|
||||
if not name:
|
||||
@@ -140,7 +142,7 @@ class ActivityName(str, Enum):
|
||||
PGA = "PGA"
|
||||
TOILETS = "Toilets"
|
||||
|
||||
def __str__(self):
|
||||
def __str__(self) -> str:
|
||||
return str(self.value)
|
||||
|
||||
|
||||
@@ -179,7 +181,7 @@ class ActivityType(str, Enum):
|
||||
# we already know, or we want to normalise things for consistency. The lookup table for this is here. Incoming spots
|
||||
# that match a key in this table will be converted to the corresponding value, so only the modes above will actually be
|
||||
# present in the spots.
|
||||
MODE_ALIASES = {
|
||||
MODE_ALIASES: dict[str, str] = {
|
||||
"USB": "SSB",
|
||||
"LSB": "SSB",
|
||||
"DIGITALVOICE": "DV",
|
||||
|
||||
+14
-10
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from math import floor, isnan
|
||||
@@ -14,7 +16,7 @@ TRANSFORMER_IRISH_GRID_TO_WGS84 = Transformer.from_crs("EPSG:29903", "EPSG:4326"
|
||||
TRANSFORMER_CI_UTM_GRID_TO_WGS84 = Transformer.from_crs("+proj=utm +zone=30 +ellps=WGS84", "EPSG:4326")
|
||||
|
||||
|
||||
def lat_lon_to_cq_zone(lat, lon):
|
||||
def lat_lon_to_cq_zone(lat: float, lon: float) -> int | None:
|
||||
"""Finds out which CQ zone a lat/lon point is in."""
|
||||
|
||||
if DATA_STORE.cq_zone_data is not None:
|
||||
@@ -36,7 +38,7 @@ def lat_lon_to_cq_zone(lat, lon):
|
||||
return None
|
||||
|
||||
|
||||
def lat_lon_to_itu_zone(lat, lon):
|
||||
def lat_lon_to_itu_zone(lat: float, lon: float) -> int | None:
|
||||
"""Finds out which ITU zone a lat/lon point is in."""
|
||||
|
||||
if DATA_STORE.itu_zone_data is not None:
|
||||
@@ -58,7 +60,7 @@ def lat_lon_to_itu_zone(lat, lon):
|
||||
return None
|
||||
|
||||
|
||||
def lat_lon_for_grid_centre(grid):
|
||||
def lat_lon_for_grid_centre(grid: str) -> list[float] | None:
|
||||
"""Convert a Maidenhead grid reference of arbitrary precision to the lat/long of the centre point of the square.
|
||||
Returns None if the grid format is invalid."""
|
||||
|
||||
@@ -69,7 +71,7 @@ def lat_lon_for_grid_centre(grid):
|
||||
return None
|
||||
|
||||
|
||||
def lat_lon_for_grid_sw_corner(grid):
|
||||
def lat_lon_for_grid_sw_corner(grid: str) -> list[float] | None:
|
||||
"""Convert a Maidenhead grid reference of arbitrary precision to the lat/long of the southwest corner of the square.
|
||||
Returns None if the grid format is invalid."""
|
||||
|
||||
@@ -80,7 +82,7 @@ def lat_lon_for_grid_sw_corner(grid):
|
||||
return None
|
||||
|
||||
|
||||
def lat_lon_for_grid_ne_corner(grid):
|
||||
def lat_lon_for_grid_ne_corner(grid: str) -> list[float] | None:
|
||||
"""Convert a Maidenhead grid reference of arbitrary precision to the lat/long of the northeast corner of the square.
|
||||
Returns None if the grid format is invalid."""
|
||||
|
||||
@@ -91,7 +93,9 @@ def lat_lon_for_grid_ne_corner(grid):
|
||||
return None
|
||||
|
||||
|
||||
def lat_lon_for_grid_sw_corner_plus_size(grid):
|
||||
def lat_lon_for_grid_sw_corner_plus_size(
|
||||
grid: str,
|
||||
) -> tuple[float, float, float, float] | tuple[None, None, None, None]:
|
||||
"""Convert a Maidenhead grid reference of arbitrary precision to lat/long, including in the result the size of the
|
||||
lowest grid square. This is a utility method used by the main methods that return the centre, southwest, and
|
||||
northeast coordinates of a grid square.
|
||||
@@ -162,7 +166,7 @@ def lat_lon_for_grid_sw_corner_plus_size(grid):
|
||||
return lat, lon, lat_cell_size, lon_cell_size
|
||||
|
||||
|
||||
def wab_wai_square_to_lat_lon(ref):
|
||||
def wab_wai_square_to_lat_lon(ref: str) -> tuple[float, float] | None:
|
||||
"""Convert a Worked All Britain or Worked All Ireland reference to a lat/lon point."""
|
||||
|
||||
# First check we have a valid grid square, and based on what it looks like, use either the Ordnance Survey, Irish,
|
||||
@@ -178,7 +182,7 @@ def wab_wai_square_to_lat_lon(ref):
|
||||
return None
|
||||
|
||||
|
||||
def os_grid_square_to_lat_lon(ref):
|
||||
def os_grid_square_to_lat_lon(ref: str) -> tuple[float, float]:
|
||||
"""Get a lat/lon point for the centre of an Ordnance Survey grid square"""
|
||||
|
||||
# Convert the letters into multipliers for the 500km squares and 100km squares
|
||||
@@ -209,7 +213,7 @@ def os_grid_square_to_lat_lon(ref):
|
||||
return lat, lon
|
||||
|
||||
|
||||
def irish_grid_square_to_lat_lon(ref):
|
||||
def irish_grid_square_to_lat_lon(ref: str) -> tuple[float, float]:
|
||||
"""Get a lat/lon point for the centre of an Irish Grid square."""
|
||||
|
||||
# Convert the letters into multipliers for the 100km squares
|
||||
@@ -237,7 +241,7 @@ def irish_grid_square_to_lat_lon(ref):
|
||||
return lat, lon
|
||||
|
||||
|
||||
def utm_grid_square_to_lat_lon(ref):
|
||||
def utm_grid_square_to_lat_lon(ref: str) -> tuple[float, float]:
|
||||
"""Get a lat/lon point for the centre of a UTM grid square (supports only squares WA & WV for the Channel Islands, nothing else implemented)"""
|
||||
|
||||
# Take the numeric parts of the grid square and multiply by 10000 to get metres from the corner of the letter-based grid square
|
||||
|
||||
+24
-18
@@ -1,33 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Generic, TypeVar
|
||||
|
||||
import diskcache
|
||||
from cachetools import TTLCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VT = TypeVar("VT")
|
||||
|
||||
class LiveDataCache:
|
||||
|
||||
class LiveDataCache(Generic[VT]):
|
||||
"""Cache for spots and alerts. Uses the fast in-memory TTLCache for normal data I/O, including the TTL to enforce
|
||||
maximum lifetime, and adds a separate diskcache to which we can save and load the TTLCache to provide persistence.
|
||||
Also adds thread safety so spots and alerts can come from any thread, and a listener mechanism so the web server
|
||||
can get a callback when new spots/alerts are added, and send them to any SSE clients."""
|
||||
|
||||
def __init__(self, maxsize, ttl, snapshot_dir, snapshot_interval_sec):
|
||||
self._cache = TTLCache(maxsize=maxsize, ttl=ttl)
|
||||
def __init__(self, maxsize: int, ttl: int, snapshot_dir: str, snapshot_interval_sec: int) -> None:
|
||||
self._cache: TTLCache = TTLCache(maxsize=maxsize, ttl=ttl)
|
||||
self._lock = threading.Lock()
|
||||
self._ttl = ttl
|
||||
self._listeners = []
|
||||
self._listeners: list[Callable[[VT], None]] = []
|
||||
self._listeners_lock = threading.Lock()
|
||||
self._snapshot_dir = snapshot_dir
|
||||
self._disk_cache = diskcache.Cache(str(snapshot_dir))
|
||||
self._stop_event = threading.Event()
|
||||
self._snapshot_thread = None
|
||||
self._snapshot_thread: threading.Thread | None = None
|
||||
self._load_snapshot()
|
||||
self._start_periodic_snapshot(snapshot_interval_sec)
|
||||
|
||||
def set(self, key, value):
|
||||
def set(self, key: str, value: VT) -> None:
|
||||
with self._lock:
|
||||
self._cache[key] = value
|
||||
|
||||
@@ -40,34 +46,34 @@ class LiveDataCache:
|
||||
except Exception:
|
||||
logger.exception(f"Listener raised an exception for key {key}")
|
||||
|
||||
def get(self, key, default=None):
|
||||
def get(self, key: str, default: VT | None = None) -> VT | None:
|
||||
with self._lock:
|
||||
return self._cache.get(key, default)
|
||||
|
||||
def delete(self, key):
|
||||
def delete(self, key: str) -> None:
|
||||
with self._lock:
|
||||
self._cache.pop(key, None)
|
||||
|
||||
def keys(self):
|
||||
def keys(self) -> list[str]:
|
||||
with self._lock:
|
||||
return list(self._cache.keys())
|
||||
|
||||
def values(self):
|
||||
def values(self) -> list[VT]:
|
||||
with self._lock:
|
||||
return list(self._cache.values())
|
||||
|
||||
def add_listener(self, callback):
|
||||
def add_listener(self, callback: Callable[[VT], None]) -> None:
|
||||
"""Register callback(value) which will be called whenever a new spot/alert item is added via set(). Used by the
|
||||
web server (via SSEBroadcaster) to send SSE clients an update on every new spot."""
|
||||
|
||||
with self._listeners_lock:
|
||||
self._listeners.append(callback)
|
||||
|
||||
def remove_listener(self, callback):
|
||||
def remove_listener(self, callback: Callable[[VT], None]) -> None:
|
||||
with self._listeners_lock:
|
||||
self._listeners.remove(callback)
|
||||
|
||||
def save_snapshot(self):
|
||||
def save_snapshot(self) -> None:
|
||||
with self._lock:
|
||||
# Store the time with the data so we can avoid loading anything nxt time that's older than TTL
|
||||
data = [(k, v, time.time()) for k, v in self._cache.items()]
|
||||
@@ -76,9 +82,9 @@ class LiveDataCache:
|
||||
except Exception:
|
||||
logger.exception(f"Failed to write snapshot to {self._snapshot_dir}")
|
||||
|
||||
def _load_snapshot(self):
|
||||
def _load_snapshot(self) -> None:
|
||||
try:
|
||||
data = self._disk_cache.get("snapshot")
|
||||
data: list[tuple[str, VT, float]] | None = self._disk_cache.get("snapshot")
|
||||
except Exception: # noqa: BLE001 (If any exceptions we treat the data as junk and start from scratch, it's probably due to a version upgrade having incompatible data structures, fine to not log the exception in this case)
|
||||
logger.warning(f"Failed to load snapshot from {self._snapshot_dir}, clearing it.")
|
||||
self._disk_cache.clear()
|
||||
@@ -94,8 +100,8 @@ class LiveDataCache:
|
||||
self._cache[key] = value
|
||||
logger.info(f"Loaded snapshot from {self._snapshot_dir}")
|
||||
|
||||
def _start_periodic_snapshot(self, interval):
|
||||
def loop():
|
||||
def _start_periodic_snapshot(self, interval: int) -> None:
|
||||
def loop() -> None:
|
||||
while not self._stop_event.wait(timeout=interval):
|
||||
self.save_snapshot()
|
||||
|
||||
@@ -104,7 +110,7 @@ class LiveDataCache:
|
||||
)
|
||||
self._snapshot_thread.start()
|
||||
|
||||
def close(self):
|
||||
def close(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._snapshot_thread:
|
||||
self._snapshot_thread.join(timeout=15)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from prometheus_client import (
|
||||
CollectorRegistry,
|
||||
Counter,
|
||||
@@ -25,7 +27,7 @@ memory_use_gauge = Gauge(
|
||||
)
|
||||
|
||||
|
||||
def get_metrics():
|
||||
def get_metrics() -> bytes:
|
||||
"""Get a Prometheus metrics response for the web server"""
|
||||
|
||||
return generate_latest(registry)
|
||||
|
||||
@@ -1,18 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import Generic, TypeVar
|
||||
|
||||
import diskcache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
class SingleObjectDataCache:
|
||||
|
||||
class SingleObjectDataCache(Generic[T]):
|
||||
"""Cache for status and solar conditions. This uses DiskCache, but unlike the standard DiskCache users like
|
||||
activity ref and callsign lookup handlers, status and solar conditions are persisted as a single object. If we
|
||||
just load the object from DiskCache and modify it, DiskCache doesn't know that it's been updated and needs
|
||||
re-caching, so we provide a store() method that any functions updating the object can call afterwards."""
|
||||
|
||||
def __init__(self, cache_dir, object_if_empty):
|
||||
def __init__(self, cache_dir: str, object_if_empty: T) -> None:
|
||||
"""Initialize a SingleObjectDataCache. Provide the directory to load the cache from and save it to. If the cache
|
||||
is empty, the provided object_if_empty parameter will be used to initialise it."""
|
||||
|
||||
@@ -22,24 +27,24 @@ class SingleObjectDataCache:
|
||||
if "object" not in self._cache:
|
||||
self._cache.add("object", object_if_empty)
|
||||
try:
|
||||
self._obj = self._cache.get("object")
|
||||
self._obj: T = self._cache.get("object")
|
||||
except Exception: # noqa: BLE001 (If any exceptions we treat the data as junk and start from scratch, it's probably due to a version upgrade having incompatible data structures, fine to not log the exception in this case)
|
||||
logger.warning(f"Failed to load cache from {cache_dir}, clearing it.")
|
||||
self._cache.clear()
|
||||
self._cache.add("object", object_if_empty)
|
||||
self._obj = object_if_empty
|
||||
|
||||
def get(self):
|
||||
def get(self) -> T:
|
||||
"""Get the data object. This can then be manipulated as necessary across multiple threads. Any function
|
||||
modifying the object must remember to call store() afterwards."""
|
||||
return self._obj
|
||||
|
||||
def store(self):
|
||||
def store(self) -> None:
|
||||
"""Store the updated object in the cache. Any function modifying the object must remember to call this
|
||||
afterwards."""
|
||||
with self._lock:
|
||||
self._cache.set("object", self._obj)
|
||||
|
||||
def close(self):
|
||||
def close(self) -> None:
|
||||
self.store()
|
||||
self._cache.close()
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
@@ -21,11 +23,11 @@ logger = logging.getLogger(__name__)
|
||||
class StatusReporter:
|
||||
"""Provides a timed update of the application's status data."""
|
||||
|
||||
def __init__(self, run_interval):
|
||||
def __init__(self, run_interval: float) -> None:
|
||||
"""Constructor"""
|
||||
|
||||
self._run_interval = run_interval
|
||||
self._thread = None
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event = Event()
|
||||
self._startup_time = datetime.now(pytz.UTC)
|
||||
|
||||
@@ -33,13 +35,13 @@ class StatusReporter:
|
||||
DATA_STORE.status.get()["server_owner_callsign"] = SERVER_OWNER_CALLSIGN
|
||||
DATA_STORE.status.store()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the reporter thread"""
|
||||
|
||||
self._thread = Thread(target=self._run, name="StatusReporter", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
self._stop_event.set()
|
||||
@@ -48,7 +50,7 @@ class StatusReporter:
|
||||
if self._thread.is_alive():
|
||||
logger.warning("Status reporter worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
"""Thread entry point: report immediately on startup, then on each interval until stopped"""
|
||||
|
||||
while True:
|
||||
@@ -56,7 +58,7 @@ class StatusReporter:
|
||||
if self._stop_event.wait(timeout=self._run_interval):
|
||||
break
|
||||
|
||||
def _report(self):
|
||||
def _report(self) -> None:
|
||||
"""Write status information"""
|
||||
|
||||
DATA_STORE.status.get()["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from requests import Response
|
||||
from requests_cache import CachedSession
|
||||
|
||||
from core.data_store import CACHE_DIR
|
||||
@@ -14,7 +18,7 @@ class URLDataCache(CachedSession):
|
||||
used across multiple threads, though note that URL lookups will block each other this way, so it is still better to
|
||||
create one of these objects per thread if possible."""
|
||||
|
||||
def __init__(self, name):
|
||||
def __init__(self, name: str) -> None:
|
||||
super().__init__(
|
||||
f"{CACHE_DIR}urls/{name}",
|
||||
expire_after=timedelta(days=1),
|
||||
@@ -22,6 +26,6 @@ class URLDataCache(CachedSession):
|
||||
)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def get(self, *args, **kwargs):
|
||||
def get(self, *args: Any, **kwargs: Any) -> Response:
|
||||
with self._lock:
|
||||
return super().get(*args, **kwargs)
|
||||
|
||||
+10
-5
@@ -1,19 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import simplejson
|
||||
from pyhamtools import Callinfo
|
||||
from pyhamtools.frequency import freq_to_band
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.constants import BANDS, UNKNOWN_BAND
|
||||
from core.data_store import DATA_STORE
|
||||
from core.enums import MODE_ALIASES, Continent, Mode, ModeType
|
||||
from data.band import Band
|
||||
from data.callsign import Callsign, LocationSourceForCallsign
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def safe_json_dumps(obj):
|
||||
def safe_json_dumps(obj: Any) -> str:
|
||||
"""Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
|
||||
which are invalid in JSON."""
|
||||
|
||||
@@ -59,7 +64,7 @@ def infer_mode_type_from_mode(mode: str) -> ModeType | None:
|
||||
return None
|
||||
|
||||
|
||||
def infer_band_from_freq(freq):
|
||||
def infer_band_from_freq(freq: float) -> Band:
|
||||
"""Infer a band from a frequency in Hz"""
|
||||
|
||||
for b in BANDS:
|
||||
@@ -68,7 +73,7 @@ def infer_band_from_freq(freq):
|
||||
return UNKNOWN_BAND
|
||||
|
||||
|
||||
def infer_mode_from_frequency(freq):
|
||||
def infer_mode_from_frequency(freq: float) -> Mode | None:
|
||||
"""Infer a mode from the frequency (in Hz) according to the band plan. Just a guess really."""
|
||||
|
||||
try:
|
||||
@@ -113,14 +118,14 @@ def infer_mode_from_frequency(freq):
|
||||
return None
|
||||
|
||||
|
||||
def get_flag_for_dxcc(dxcc):
|
||||
def get_flag_for_dxcc(dxcc: int) -> str | None:
|
||||
"""Get an emoji flag for a given DXCC entity ID"""
|
||||
|
||||
dxcc_data = DATA_STORE.dxcc_data.get(dxcc, None)
|
||||
return dxcc_data["flag"] if dxcc_data else None
|
||||
|
||||
|
||||
def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
|
||||
def get_callsign_object_from_pyhamtools_callinfo(callsign: str, callinfo: Callinfo) -> Callsign:
|
||||
"""Utility function to take the data provided by a PyHamTools CallInfo object and populate our own Callsign data
|
||||
object from it"""
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType, ActivityType
|
||||
from data.activity import Activity
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType, ActivityType
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from core.enums import ActivityRefType
|
||||
|
||||
+10
-6
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
@@ -11,6 +13,8 @@ from core.activity_utils import get_icon_for_activity
|
||||
from core.call_lookup_helper import get_call_info
|
||||
from core.enums import Continent
|
||||
from core.utils import get_flag_for_dxcc
|
||||
from data.activity_ref import ActivityRef
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -25,9 +29,9 @@ class Alert:
|
||||
# DX (alerting) operator info
|
||||
|
||||
# Callsigns of the operators that has been alerted
|
||||
dx_calls: list | None = None
|
||||
dx_calls: list[str] | None = None
|
||||
# Names of the operators that has been alerted
|
||||
dx_names: list | None = None
|
||||
dx_names: list[str | None] | None = None
|
||||
# Country of the DX operator
|
||||
dx_country: str | None = None
|
||||
# Country flag of the DX operator
|
||||
@@ -64,7 +68,7 @@ class Alert:
|
||||
sig: str | None = None
|
||||
# Activity references. We allow multiple here for e.g. n-fer activations, unlike ADIF SIG_INFO. Still named
|
||||
# "sig_refs" for API backwards compatibility.
|
||||
sig_refs: list = field(default_factory=list)
|
||||
sig_refs: list[ActivityRef] = field(default_factory=list)
|
||||
|
||||
# Timing info
|
||||
|
||||
@@ -87,7 +91,7 @@ class Alert:
|
||||
# Icon to use when displaying this alert in the web UI. Chosen from the Font Awesome set.
|
||||
icon: str | None = None
|
||||
|
||||
def infer_missing(self, credentials=None):
|
||||
def infer_missing(self, credentials: LookupCredentials | None = None) -> None:
|
||||
"""Infer missing parameters where possible"""
|
||||
|
||||
try:
|
||||
@@ -160,12 +164,12 @@ class Alert:
|
||||
except Exception:
|
||||
logger.exception("Exception while inferring missing data from spot")
|
||||
|
||||
def to_json(self):
|
||||
def to_json(self) -> str:
|
||||
"""JSON serialise"""
|
||||
|
||||
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True)
|
||||
|
||||
def expired(self):
|
||||
def expired(self) -> bool:
|
||||
"""Decide if this alert has expired (in which case it should not be added to the system in the first place, and not
|
||||
returned by the web server if later requested, and removed by the cleanup functions). "Expired" is defined as
|
||||
either having an end_time in the past, or if it only has a start_time, then that start time was more than 3 hours
|
||||
|
||||
+3
-1
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from core.enums import Continent, LocationSourceForCallsign
|
||||
@@ -40,7 +42,7 @@ class Callsign:
|
||||
# Location source
|
||||
location_source: LocationSourceForCallsign | None = None
|
||||
|
||||
def fully_populated(self):
|
||||
def fully_populated(self) -> bool:
|
||||
"""Utility method to indicate that the callsign data is fully populated. Multiple providers can return data for
|
||||
a callsign, and we try them in sequence until we have all the data we can, in which case there's no point
|
||||
querying any other providers."""
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from tornado.httputil import HTTPHeaders
|
||||
|
||||
|
||||
@dataclass
|
||||
class LookupCredentials:
|
||||
@@ -13,7 +17,7 @@ class LookupCredentials:
|
||||
hamqth_session_id: str = "" # alternative to username/password
|
||||
|
||||
|
||||
def extract_credentials(headers):
|
||||
def extract_credentials(headers: HTTPHeaders) -> LookupCredentials | None:
|
||||
"""Build a LookupCredentials from HTTP request headers; returns None if no usable credentials are present."""
|
||||
creds = LookupCredentials(
|
||||
qrz_username=headers.get("X-QRZ-Username", ""),
|
||||
|
||||
+16
-11
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, TypeVar
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
# Lookup tables for derived text descriptions.
|
||||
# Each threshold-based table is a list of (min_value, description) pairs in descending order;
|
||||
@@ -71,7 +76,7 @@ ELECTRON_FLUX_DESCRIPTIONS = [
|
||||
]
|
||||
|
||||
|
||||
def _xray_blackout_scale(xray):
|
||||
def _xray_blackout_scale(xray: str | None) -> int:
|
||||
"""Return the NOAA Radio Blackout scale number (R0-R5) for the given X-ray flux class string
|
||||
(e.g. "M4.5", "X12")."""
|
||||
|
||||
@@ -93,7 +98,7 @@ def _xray_blackout_scale(xray):
|
||||
return 0
|
||||
|
||||
|
||||
def _lookup_by_threshold(value, table, default=None):
|
||||
def _lookup_by_threshold(value: int | None, table: list[tuple[int, T]], default: T | None = None) -> T | None:
|
||||
"""Return the description from a threshold table for the given numeric value.
|
||||
The table is a list of (min_value, description) pairs in descending order."""
|
||||
|
||||
@@ -150,20 +155,20 @@ class SolarConditions:
|
||||
# Geomagnetic background noise level, e.g. "S0", "S1", "S2"
|
||||
geomag_noise: str | None = None
|
||||
# HF band propagation conditions, keyed by "{band}-{time}" e.g. "80m-40m-day"
|
||||
hf_conditions: dict | None = None
|
||||
hf_conditions: dict[str, str] | None = None
|
||||
# VHF propagation conditions, keyed by condition name
|
||||
vhf_conditions: dict | None = None
|
||||
vhf_conditions: dict[str, str | None] | None = None
|
||||
# NOAA Kp index 3-day forecast, keyed by UNIX timestamp of the start of each 3-hour UTC period
|
||||
k_index_forecast: dict | None = None
|
||||
k_index_forecast: dict[float, float] | None = None
|
||||
# NOAA Solar Radiation Storm (S1 or greater) probability forecast, keyed by UNIX timestamp of start of day UTC
|
||||
solar_storm_forecast: dict | None = None
|
||||
solar_storm_forecast: dict[float, int] | None = None
|
||||
# NOAA Radio Blackout (R1-R2) probability forecast, keyed by UNIX timestamp of start of day UTC
|
||||
blackout_forecast_r1r2: dict | None = None
|
||||
blackout_forecast_r1r2: dict[float, int] | None = None
|
||||
# NOAA Radio Blackout (R3 or greater) probability forecast, keyed by UNIX timestamp of start of day UTC
|
||||
blackout_forecast_r3_or_greater: dict | None = None
|
||||
blackout_forecast_r3_or_greater: dict[float, int] | None = None
|
||||
# Ionosonde measurements, dict keyed by URSI code, values are dicts with keys: ursi, name, fof2, muf, luf,
|
||||
# band_states. Populated by GIROIonosonde or KC2GProp providers.
|
||||
ionosonde_data: dict | None = None
|
||||
ionosonde_data: dict[str, Any] | None = None
|
||||
|
||||
# Derived values (populated by infer_descriptions())
|
||||
# HF radio blackout risk description, derived from xray
|
||||
@@ -183,7 +188,7 @@ class SolarConditions:
|
||||
# Electron flux description, derived from electron_flux
|
||||
electron_flux_desc: str | None = None
|
||||
|
||||
def infer_descriptions(self):
|
||||
def infer_descriptions(self) -> None:
|
||||
"""Populate derived text description fields from the current numeric/raw field values."""
|
||||
|
||||
if self.xray and len(self.xray) > 0:
|
||||
@@ -196,7 +201,7 @@ class SolarConditions:
|
||||
self.band_conditions_desc = _lookup_by_threshold(self.sfi, BAND_CONDITIONS_DESCRIPTIONS)
|
||||
self.electron_flux_desc = _lookup_by_threshold(self.electron_flux, ELECTRON_FLUX_DESCRIPTIONS)
|
||||
|
||||
def to_json(self):
|
||||
def to_json(self) -> str:
|
||||
"""JSON serialise. Dict key order is insertion order (Python 3.7+ guarantee), so callers receive
|
||||
fields in a predictable, logical sequence without relying on sort_keys."""
|
||||
|
||||
|
||||
+9
-6
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
@@ -32,6 +34,7 @@ from core.utils import (
|
||||
)
|
||||
from data.activities import ACTIVITIES
|
||||
from data.activity_ref import ActivityRef
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -129,7 +132,7 @@ class Spot:
|
||||
sig: str | None = None
|
||||
# Activity references. We allow multiple here for e.g. n-fer activations, unlike ADIF SIG_INFO. Still named
|
||||
# "sig_refs" for API backwards compatibility.
|
||||
sig_refs: list = field(default_factory=list)
|
||||
sig_refs: list[ActivityRef] = field(default_factory=list)
|
||||
|
||||
# Timing info
|
||||
|
||||
@@ -156,7 +159,7 @@ class Spot:
|
||||
# Icon to use when displaying this spot in the web UI. Chosen from the Font Awesome set.
|
||||
icon: str | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
def __post_init__(self) -> None:
|
||||
"""Normalise fields that don't survive a plain dict to Spot conversion. This is used in the "add spot" API
|
||||
endpoint where the client is submitting JSON, and we want to recreate a full Spot object, including nested
|
||||
objects such as the sig_refs list.."""
|
||||
@@ -167,7 +170,7 @@ class Spot:
|
||||
for activity_ref in self.sig_refs
|
||||
]
|
||||
|
||||
def infer_missing(self, credentials=None):
|
||||
def infer_missing(self, credentials: LookupCredentials | None = None) -> None:
|
||||
"""Infer missing parameters where possible"""
|
||||
|
||||
try:
|
||||
@@ -538,12 +541,12 @@ class Spot:
|
||||
except Exception:
|
||||
logger.exception("Exception while inferring missing data from spot")
|
||||
|
||||
def to_json(self):
|
||||
def to_json(self) -> str:
|
||||
"""JSON serialise"""
|
||||
|
||||
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True)
|
||||
|
||||
def _append_activity_ref_if_missing(self, new_activity_ref):
|
||||
def _append_activity_ref_if_missing(self, new_activity_ref: ActivityRef) -> None:
|
||||
"""Append an activity ref to the list, so long as it's not already there."""
|
||||
|
||||
new_activity_ref.id = new_activity_ref.id.strip().upper()
|
||||
@@ -555,7 +558,7 @@ class Spot:
|
||||
return
|
||||
self.sig_refs.append(new_activity_ref)
|
||||
|
||||
def expired(self):
|
||||
def expired(self) -> bool:
|
||||
"""Decide if this spot has expired (in which case it should not be added to the system in the first place, and not
|
||||
returned by the web server if later requested, and removed by the cleanup functions). "Expired" is defined as
|
||||
either having a time further ago than the server's MAX_SPOT_AGE. If it somehow doesn't have a time either, it is
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from data.activity_ref import ActivityRef
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -13,29 +17,29 @@ class ActivityRefDataProvider:
|
||||
"""Generic activity reference data provider class. Subclasses of this query the individual URLs or files for
|
||||
data."""
|
||||
|
||||
def __init__(self, sig_name, provider_config):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor. Note the parameter and attribute are still named "sig_name" for consistency with the API's
|
||||
"sig" field name."""
|
||||
|
||||
self.sig_name = sig_name
|
||||
self.enabled = provider_config["enabled"]
|
||||
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Not Started" if self.enabled else "Disabled"
|
||||
self.reference_count = 0
|
||||
self.enabled: bool = provider_config["enabled"]
|
||||
self.last_update_time: datetime = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status: str = "Not Started" if self.enabled else "Disabled"
|
||||
self.reference_count: int = 0
|
||||
self._stop_event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
|
||||
super()."""
|
||||
|
||||
self._stop_event.set()
|
||||
|
||||
def _add_data(self, new_data):
|
||||
def _add_data(self, new_data: list[ActivityRef]) -> None:
|
||||
"""Add all the provided reference data objects to the data store."""
|
||||
|
||||
# with transact() batches all writes together to save making thousands of individual sqlite writes. However,
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class ARLHS(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.ARLHS
|
||||
DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
if "ARLHS" in row and row["ARLHS"] != "":
|
||||
ref_id = row["ARLHS"]
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -14,11 +18,11 @@ class COTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.COTA
|
||||
DATA_URL = "https://www.cotagroup.org/cotagroup/map/data/castles-all-7d90ee2a5e1175e5dece1bbf9dc87504.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data[2]:
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class DCE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DCE
|
||||
DATA_URL = "https://www.acracb.org/dce/descargas/General/directorio_referencias_dce.xls"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
file_stream = io.BytesIO(http_response.content)
|
||||
df = pd.read_excel(file_stream, engine="xlrd", header=None)
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class DEFE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DEFE
|
||||
DATA_URL = "https://www.acracb.org/defe/descargas/General/directorio_referencias_defe.xls"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
file_stream = io.BytesIO(http_response.content)
|
||||
df = pd.read_excel(file_stream, engine="xlrd", header=None)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
@@ -16,11 +19,11 @@ class DME(LocalFileActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DME
|
||||
PATH = "datafiles/MUNICIPIOS.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.PATH)
|
||||
|
||||
def _file_to_data(self, path):
|
||||
new_data = []
|
||||
def _file_to_data(self, path: str) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
with open(path, encoding="latin-1") as _f:
|
||||
for row in csv.DictReader(_f, delimiter=";"):
|
||||
# Store reference IDs with the "DME-" prefix rather than just the number. This will prevent Spothole
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -13,11 +18,11 @@ class DMUE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DMUE
|
||||
DATA_URL = "https://dmue.radiogalena.es/nom_dmue.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
for row in csv.reader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
|
||||
if len(row) > 1 and row[0] and row[1]:
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class DMVE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DMVE
|
||||
DATA_URL = "https://www.acracb.org/dmve/descargas/General/directorio_referencias_dmve.xls"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
file_stream = io.BytesIO(http_response.content)
|
||||
# Despide the .xls extension this is actually an xlsx file, so we need openpyxl not xlrd
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -12,11 +17,11 @@ class DTMBA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DTMBA
|
||||
DATA_URL = "https://www.iu1fig.com/share/iz0eik/dtmba/export.php"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in http_response.content.decode("utf-8-sig").splitlines():
|
||||
split = row.split(";")
|
||||
ref_id = split[0]
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pdfplumber
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class FEA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.FEA
|
||||
DATA_URL = "http://ea5ol.net/Lista%20Faros.pdf"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
# Use PDFPlumber to extract the tables in the PDF
|
||||
with pdfplumber.open(BytesIO(http_response.content)) as pdf:
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from core.url_data_cache import URLDataCache
|
||||
from data.activity_ref import ActivityRef
|
||||
from providers.activityrefdata.activity_ref_data_provider import ActivityRefDataProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -16,35 +21,35 @@ class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
|
||||
"""Generic activity ref data provider class for providers that fetch their data from the web by downloading a
|
||||
file."""
|
||||
|
||||
def __init__(self, sig_name, provider_config, url, poll_interval):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
"""Set up the provider, note poll_interval is in *days*."""
|
||||
super().__init__(sig_name, provider_config)
|
||||
self._url = url
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._thread: Thread | None = None
|
||||
self._url_data_cache = URLDataCache(f"activity_ref_data_{sig_name}")
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
|
||||
# subsequent polls, so start() returns immediately and the application can continue starting.
|
||||
logger.info(f"Set up query of {self.sig_name} activity ref data every {self._poll_interval!s} days.")
|
||||
self._thread = Thread(target=self._run, name=f"FileDownloadActivityRefDataProvider-{self.sig_name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
super().stop()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.sig_name} activity ref data worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
# Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
|
||||
# caching, but just so continual restarts of the software during testing don't hammer the servers.
|
||||
@@ -76,7 +81,7 @@ class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
|
||||
logger.exception(f"Exception in HTTP Activity Ref Data Provider ({self.sig_name})")
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
"""Convert an HTTP response returned by the server into activity ref data. The whole response is provided here
|
||||
so the subclass implementations can check for HTTP status codes if necessary, and handle the response as
|
||||
JSON, CSV, whatever the remote file actually is."""
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class GMA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.GMA
|
||||
DATA_URL = "https://www.gma.rocks/download/summits.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
ref_id = row["Reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class ILLW(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.ILLW
|
||||
DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
if "ILLW" in row and row["ILLW"] != "":
|
||||
ref_id = row["ILLW"]
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -19,11 +23,11 @@ class IOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.IOTA
|
||||
DATA_URL = "https://www.iota-world.org/islands-on-the-air/downloads/download-file.html?path=groups.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data:
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.enums import ActivityName
|
||||
from providers.activityrefdata.pnp_kml_activity_ref_data_provider import (
|
||||
ParksNPeaksKMLActivityRefDataProvider,
|
||||
@@ -11,5 +15,5 @@ class KRMNPA(ParksNPeaksKMLActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.KRMNPA
|
||||
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=KRMNPA&poiFormat=4"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import locator_to_latlong
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -16,11 +20,11 @@ class LLOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.LLOTA
|
||||
DATA_URL = "https://llota.app/api/public/references"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data:
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
from data.activity_ref import ActivityRef
|
||||
from providers.activityrefdata.activity_ref_data_provider import ActivityRefDataProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -11,11 +15,11 @@ logger = logging.getLogger(__name__)
|
||||
class LocalFileActivityRefDataProvider(ActivityRefDataProvider):
|
||||
"""Generic activity ref data provider class for providers that fetch their data from a local file on startup."""
|
||||
|
||||
def __init__(self, sig_name, provider_config, path):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any], path: str) -> None:
|
||||
super().__init__(sig_name, provider_config)
|
||||
self._path = path
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.debug(f"Loading {self.sig_name} activity ref data from file.")
|
||||
try:
|
||||
new_data = self._file_to_data(self._path)
|
||||
@@ -30,7 +34,7 @@ class LocalFileActivityRefDataProvider(ActivityRefDataProvider):
|
||||
self.status = "Error"
|
||||
logger.exception(f"Exception in local file Activity Ref Data Provider ({self.sig_name})")
|
||||
|
||||
def _file_to_data(self, path):
|
||||
def _file_to_data(self, path: str) -> list[ActivityRef]:
|
||||
"""Load a file on the given path and turn it into activity ref data."""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class MOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.MOTA
|
||||
DATA_URL = "https://www.gma.rocks/download/mills.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
ref_id = row["Reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -14,11 +18,11 @@ class PGA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.PGA
|
||||
DATA_URL = "http://www.spga.pl/lista_pga2.php"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
soup = BeautifulSoup(http_response.text, "html.parser")
|
||||
|
||||
# Iterate through tables in the page
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from fastkml import kml
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
@@ -17,12 +21,12 @@ class ParksNPeaksKMLActivityRefDataProvider(FileDownloadActivityRefDataProvider)
|
||||
|
||||
REF_PATTERN = re.compile(r"VKFF-\d+")
|
||||
|
||||
def __init__(self, sig_name, provider_config, url, poll_interval):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
"""Set up the provider, note poll_interval is in *days*."""
|
||||
super().__init__(sig_name, provider_config, url, poll_interval)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
k = kml.KML.from_string(http_response.content)
|
||||
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class POTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.POTA
|
||||
DATA_URL = "https://pota.app/all_parks_ext.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.enums import ActivityName
|
||||
from providers.activityrefdata.pnp_kml_activity_ref_data_provider import (
|
||||
ParksNPeaksKMLActivityRefDataProvider,
|
||||
@@ -11,5 +15,5 @@ class SANPCPA(ParksNPeaksKMLActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.SANPCPA
|
||||
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=SANPCPA&poiFormat=4"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class SIOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.SIOTA
|
||||
DATA_URL = "https://www.silosontheair.com/data/silos.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["SILO_CODE"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -17,11 +21,11 @@ class SOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.SOTA
|
||||
DATA_URL = "https://storage.sota.org.uk/summitslist.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
ref_id = row["SummitCode"]
|
||||
latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from typing import Any
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -13,11 +16,11 @@ class Toilets(LocalFileActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.TOILETS
|
||||
PATH = "datafiles/toilets.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.PATH)
|
||||
|
||||
def _file_to_data(self, path):
|
||||
new_data = []
|
||||
def _file_to_data(self, path: str) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
with open(path) as _f:
|
||||
csv_data = _f.read()
|
||||
dr = csv.DictReader(csv_data.splitlines())
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class Towers(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.TOWERS
|
||||
DATA_URL = "https://wwtota.com/servis/generate_csv.php?ref=&filter=all"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
|
||||
ref_id = row["Ref"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -20,11 +24,11 @@ class WCA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WCA
|
||||
DATA_URL = "https://polo.ham2k.com/data/activities/wca/all-castles.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["REF"]
|
||||
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,11 +19,11 @@ class WOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WOTA
|
||||
DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for feature in http_response.json().get("features", []):
|
||||
ref_id = feature["properties"]["wotaId"]
|
||||
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class WWBOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WWBOTA
|
||||
DATA_URL = "https://api.wwbota.org/bunkers/?format=CSV"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["Reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class WWFF(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WWFF
|
||||
DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -16,11 +20,11 @@ class ZLOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.ZLOTA
|
||||
DATA_URL = "https://ontheair.nz/assets/assets.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data:
|
||||
|
||||
@@ -1,28 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from core.live_data_cache import LiveDataCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Deferred to avoid a circular import: data.alert imports core.call_lookup_helper, which imports
|
||||
# core.data_providers, which imports this module.
|
||||
from data.alert import Alert
|
||||
|
||||
|
||||
class AlertProvider:
|
||||
"""Generic alert provider class. Subclasses of this query the individual APIs for alerts."""
|
||||
|
||||
def __init__(self, name, provider_config):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor"""
|
||||
|
||||
self.name = name
|
||||
self.enabled = provider_config.get("enabled", True)
|
||||
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Not Started" if self.enabled else "Disabled"
|
||||
self._alerts = DATA_STORE.alerts
|
||||
self._alerts: LiveDataCache[Alert] = DATA_STORE.alerts
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def _submit_batch(self, alerts):
|
||||
def _submit_batch(self, alerts: list[Alert]) -> None:
|
||||
"""Submit a batch of alerts retrieved from the provider. There is no timestamp checking like there is for spots,
|
||||
because alerts could be created at any point for any time in the future. Rely on hashcode-based id matching
|
||||
to deal with duplicates."""
|
||||
@@ -35,11 +44,11 @@ class AlertProvider:
|
||||
alert.infer_missing()
|
||||
self._add_alert(alert)
|
||||
|
||||
def _add_alert(self, alert):
|
||||
def _add_alert(self, alert: Alert) -> None:
|
||||
if not alert.expired():
|
||||
self._alerts.set(alert.id, alert)
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from core.enums import ActivityName
|
||||
@@ -15,10 +19,10 @@ class BOTA(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://www.beachesontheair.com/"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("BOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Find the table of upcoming alerts
|
||||
bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,10 +18,10 @@ class Hamsat(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://hams.at/api/alerts"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("Hamsat", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Iterate through source data
|
||||
for source_alert in http_response.json()["data"]:
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, JSONDecodeError, ReadTimeout
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from data.alert import Alert
|
||||
from providers.alert.alert_provider import AlertProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -16,34 +20,34 @@ class HTTPAlertProvider(AlertProvider):
|
||||
"""Generic alert provider class for providers that request data via HTTP(S). Just for convenience to avoid code
|
||||
duplication. Subclasses of this query the individual APIs for data."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: int) -> None:
|
||||
super().__init__(name, provider_config)
|
||||
self._url = url
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
|
||||
# subsequent polls, so start() returns immediately and the application can continue starting.
|
||||
logger.info(f"Set up query of {self.name} alert API every {self._poll_interval!s} seconds.")
|
||||
self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} alert worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=self._poll_interval):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
# Request data from API
|
||||
logger.debug(f"Polling {self.name} alert API...")
|
||||
@@ -78,7 +82,7 @@ class HTTPAlertProvider(AlertProvider):
|
||||
# Brief pause on error before the next poll, but still respond promptly to stop()
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
"""Convert an HTTP response returned by the API into alert data. The whole response is provided here so the subclass
|
||||
implementations can check for HTTP status codes if necessary, and handle the response as JSON, XML, text, whatever
|
||||
the API actually provides."""
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
from datetime import datetime, time
|
||||
from typing import cast
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime, time
|
||||
from typing import Any, cast
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from icalendar import Calendar, Event
|
||||
|
||||
from data.alert import Alert
|
||||
@@ -12,10 +15,10 @@ class ICALAlertProvider(HTTPAlertProvider):
|
||||
"""Generic alert provider for iCal calendars. Defines an abstract method event_to_alert(event) that subclasses must
|
||||
implement, and use it to convert an iCal event to an Alert object based on whatever format their iCal events use."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: int) -> None:
|
||||
super().__init__(name, provider_config, url, poll_interval)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
cal = Calendar.from_ical(http_response.content)
|
||||
|
||||
@@ -34,7 +37,7 @@ class ICALAlertProvider(HTTPAlertProvider):
|
||||
"""Convert an ICal event to an Alert object. Subclasses must implement this method."""
|
||||
|
||||
@staticmethod
|
||||
def _to_utc_timestamp(value):
|
||||
def _to_utc_timestamp(value: datetime | date) -> float:
|
||||
"""Convert a date or datetime value from an iCal field into a UTC UNIX timestamp."""
|
||||
|
||||
# Datetime object so we can treat it as-is, check if it has a non-UTC tz and convert it if necessary
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import cast
|
||||
from typing import Any, cast
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from rss_parser import Parser
|
||||
from rss_parser.models.rss import RSS
|
||||
|
||||
@@ -18,10 +21,10 @@ class NG3K(HTTPAlertProvider):
|
||||
ALERTS_URL = "https://www.ng3k.com/adxo.xml"
|
||||
AS_CALL_PATTERN = re.compile("as ([a-z0-9/]+)", re.IGNORECASE)
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("NG3K", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
|
||||
# Iterate through source data
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -17,10 +21,10 @@ class ParksNPeaks(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://parksnpeaks.org/api/ALERTS/"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("ParksNPeaks", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Iterate through source data
|
||||
for source_alert in http_response.json():
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,10 +18,10 @@ class POTA(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://api.pota.app/activation"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("POTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Iterate through source data
|
||||
for source_alert in http_response.json():
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from icalendar import Event
|
||||
|
||||
@@ -12,7 +15,7 @@ class RSGBICALAlertProvider(ICALAlertProvider):
|
||||
handling specific to how RSGB's iCal events are formatted. This is still effectively an abstract class itself;
|
||||
RSGB has two contest calendars (HF & VHF) that each subclass this."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: int) -> None:
|
||||
super().__init__(name, provider_config, url, poll_interval)
|
||||
|
||||
FREQ_PATTERN = re.compile(r"([\d.]+(?:MHz|GHz))|SHF")
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from providers.alert.rsgb_ical_alert_provider import RSGBICALAlertProvider
|
||||
|
||||
|
||||
@@ -7,5 +11,5 @@ class RSGBHFContests(RSGBICALAlertProvider):
|
||||
POLL_INTERVAL_DAYS = 30
|
||||
ALERTS_URL = "https://calendar.google.com/calendar/ical/a5ff31ebb1b4834dc7fff4c5415ae8251c6a9aa11f98c6af6e472b6c552b1915%40group.calendar.google.com/public/basic.ics"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("RSGB HF Contests", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60)
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from providers.alert.rsgb_ical_alert_provider import RSGBICALAlertProvider
|
||||
|
||||
|
||||
@@ -7,5 +11,5 @@ class RSGBVHFContests(RSGBICALAlertProvider):
|
||||
POLL_INTERVAL_DAYS = 30
|
||||
ALERTS_URL = "https://calendar.google.com/calendar/ical/40f3552bff39a016f1cdca205864177070dcad68d55be17eb061cb021f39f96c%40group.calendar.google.com/public/basic.ics"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("RSGB VHF Contests", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60)
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,10 +18,10 @@ class SOTA(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://api-db2.sota.org.uk/api/alerts/365/all/all"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("SOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Iterate through source data
|
||||
for source_alert in http_response.json():
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from icalendar import Event
|
||||
|
||||
from core.enums import ActivityName
|
||||
@@ -11,7 +15,7 @@ class WA7BNM(ICALAlertProvider):
|
||||
POLL_INTERVAL_DAYS = 1
|
||||
ALERTS_URL = "https://contestcalendar.com/weeklycontcustom.php"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(
|
||||
"WA7BNM Contest Calendar", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60
|
||||
)
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import cast
|
||||
from typing import Any, cast
|
||||
from xml.parsers.expat import ExpatError
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from rss_parser import Parser as RSSParser
|
||||
from rss_parser.models.rss import RSS
|
||||
|
||||
@@ -22,10 +25,10 @@ class WOTA(HTTPAlertProvider):
|
||||
ALERTS_URL = "https://www.wota.org.uk/alerts_rss.php"
|
||||
RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("WOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,10 +18,10 @@ class WWFF(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://spots.wwff.co/static/agendas.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("WWFF", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Iterate through source data
|
||||
for source_alert in http_response.json():
|
||||
|
||||
@@ -1,18 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import diskcache
|
||||
|
||||
from providers.callsigndata.callsign_data_provider import CallsignDataProvider
|
||||
|
||||
|
||||
class APIQueryCallsignDataProvider(CallsignDataProvider):
|
||||
"""Generic callsign data provider class for providers that fetch their data from the web on-demand using an API."""
|
||||
|
||||
def __init__(self, name, provider_config, storage):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], storage: diskcache.Cache) -> None:
|
||||
"""Set up the provider."""
|
||||
super().__init__(name, provider_config, storage)
|
||||
|
||||
if self.enabled:
|
||||
self.status = "Ready"
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
pass
|
||||
|
||||
@@ -1,15 +1,21 @@
|
||||
from datetime import datetime
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import diskcache
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
|
||||
|
||||
class CallsignDataProvider:
|
||||
"""Generic callsign reference data provider class. Subclasses of this set up the various mechanisms via which
|
||||
Spothole can look up data for callsigns."""
|
||||
|
||||
def __init__(self, name, provider_config, storage):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], storage: diskcache.Cache) -> None:
|
||||
"""Constructor. As well as name and config, provide the storage object from DATA_STORE that will be used to
|
||||
store the result of lookups to speed up future access."""
|
||||
|
||||
@@ -21,18 +27,18 @@ class CallsignDataProvider:
|
||||
self.lookup_count = 0
|
||||
self._storage = storage
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access remote resources, if
|
||||
needed."""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def lookup(self, callsign, lookup_credentials):
|
||||
def lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
"""Looks up data for the provided callsign. Takes a LookupCredentials object, which provides any credentials
|
||||
that have been provided by the user for this session (QRZ.com/HamQTH) to allow us to look up using those
|
||||
services on the user's behalf. (Clublog is looked up using an API key owned by the server and provided in its
|
||||
@@ -57,7 +63,7 @@ class CallsignDataProvider:
|
||||
else:
|
||||
return None
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
"""Makes a new request to the data source for callsign data."""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
from pyhamtools import Callinfo, LookupLib
|
||||
@@ -7,6 +10,7 @@ from pyhamtools import Callinfo, LookupLib
|
||||
from core.data_store import DATA_STORE
|
||||
from core.utils import get_callsign_object_from_pyhamtools_callinfo
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
from providers.callsigndata.api_query_callsign_data_provider import (
|
||||
APIQueryCallsignDataProvider,
|
||||
)
|
||||
@@ -17,9 +21,9 @@ logger = logging.getLogger(__name__)
|
||||
class ClublogAPI(APIQueryCallsignDataProvider):
|
||||
"""Callsign data provider for Clublog's API."""
|
||||
|
||||
_callinfo = None
|
||||
_callinfo: Callinfo | None = None
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
# API key required for this provider
|
||||
self._api_key = provider_config.get("api_key", "")
|
||||
if self._api_key != "":
|
||||
@@ -33,7 +37,7 @@ class ClublogAPI(APIQueryCallsignDataProvider):
|
||||
|
||||
super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
callsign_data = Callsign(call=callsign)
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from pyhamtools import Callinfo, LookupLib
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from core.utils import get_callsign_object_from_pyhamtools_callinfo
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
from providers.callsigndata.file_download_callsign_data_provider import (
|
||||
FileDownloadCallsignDataProvider,
|
||||
)
|
||||
@@ -20,9 +24,9 @@ class ClublogXML(FileDownloadCallsignDataProvider):
|
||||
DATA_URL = "https://cdn.clublog.org/cty.php"
|
||||
CACHE_PATH_ZIPPED = "cache/cty.xml.gz"
|
||||
CACHE_PATH_UNZIPPED = "cache/cty.xml"
|
||||
_callinfo = None
|
||||
_callinfo: Callinfo | None = None
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
# API key required for this provider
|
||||
self._api_key = provider_config.get("api_key", "")
|
||||
if self._api_key == "":
|
||||
@@ -40,7 +44,7 @@ class ClublogXML(FileDownloadCallsignDataProvider):
|
||||
DATA_STORE.callsign_data_clublogxml,
|
||||
)
|
||||
|
||||
def _handle_file(self, path):
|
||||
def _handle_file(self, path: str) -> bool:
|
||||
try:
|
||||
# The download from Clublog is gzipped so we need to uncompress that and re-save as a separate file that
|
||||
# the LookupLib can actually use.
|
||||
@@ -60,7 +64,7 @@ class ClublogXML(FileDownloadCallsignDataProvider):
|
||||
logger.exception("Exception when loading Clublog XML.")
|
||||
return False
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
callsign_data = Callsign(call=callsign)
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from pyhamtools import Callinfo, LookupLib
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from core.utils import get_callsign_object_from_pyhamtools_callinfo
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
from providers.callsigndata.file_download_callsign_data_provider import (
|
||||
FileDownloadCallsignDataProvider,
|
||||
)
|
||||
@@ -18,9 +22,9 @@ class CountryFiles(FileDownloadCallsignDataProvider):
|
||||
POLL_INTERVAL_DAYS = 30
|
||||
DATA_URL = "https://www.country-files.com/cty/cty.plist"
|
||||
CACHE_PATH = "cache/cty.plist"
|
||||
_callinfo = None
|
||||
_callinfo: Callinfo | None = None
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(
|
||||
"CountryFiles.com",
|
||||
provider_config,
|
||||
@@ -30,7 +34,7 @@ class CountryFiles(FileDownloadCallsignDataProvider):
|
||||
DATA_STORE.callsign_data_countryfiles,
|
||||
)
|
||||
|
||||
def _handle_file(self, path):
|
||||
def _handle_file(self, path: str) -> bool:
|
||||
try:
|
||||
lookuplib = LookupLib(lookuptype="countryfile", filename=path)
|
||||
self._callinfo = Callinfo(lookuplib)
|
||||
@@ -40,7 +44,7 @@ class CountryFiles(FileDownloadCallsignDataProvider):
|
||||
logger.exception("Exception when loading Country Files cty.plist.")
|
||||
return False
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
callsign_data = Callsign(call=callsign)
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import diskcache
|
||||
import pytz
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
|
||||
|
||||
@@ -15,40 +19,48 @@ logger = logging.getLogger(__name__)
|
||||
class FileDownloadCallsignDataProvider(CallsignDataProvider):
|
||||
"""Generic callsign data provider class for providers that fetch their data from the web by downloading a file."""
|
||||
|
||||
def __init__(self, name, provider_config, url, cache_file_path, poll_interval, storage):
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
provider_config: dict[str, Any],
|
||||
url: str,
|
||||
cache_file_path: str,
|
||||
poll_interval: int,
|
||||
storage: diskcache.Cache,
|
||||
) -> None:
|
||||
"""Set up the provider, note poll_interval is in *days*."""
|
||||
super().__init__(name, provider_config, storage)
|
||||
self._url = url
|
||||
self._cache_file_path = cache_file_path
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event = Event()
|
||||
self._url_data_cache = URLDataCache(f"callsigndata_{name}")
|
||||
|
||||
if self.enabled:
|
||||
self.status = "Ready"
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
|
||||
# subsequent polls, so start() returns immediately and the application can continue starting.
|
||||
logger.info(f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
|
||||
self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} callsign data worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
# Request the file. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
|
||||
# caching, but just so continual restarts of the software during testing don't hammer the servers.
|
||||
@@ -87,7 +99,7 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
|
||||
logger.exception(f"Exception in callsign reference data provider ({self.name})")
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _handle_file(self, path):
|
||||
def _handle_file(self, path: str) -> bool:
|
||||
"""Handle an updated file downloaded from the server. Return true if successful, false otherwise."""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import xmltodict
|
||||
@@ -14,6 +17,7 @@ from core.data_store import CACHE_DIR, DATA_STORE
|
||||
from core.enums import Continent
|
||||
from core.url_data_cache import URLDataCache
|
||||
from data.callsign import Callsign, LocationSourceForCallsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
from providers.callsigndata.api_query_callsign_data_provider import (
|
||||
APIQueryCallsignDataProvider,
|
||||
)
|
||||
@@ -24,7 +28,7 @@ logger = logging.getLogger(__name__)
|
||||
class HamQTH(APIQueryCallsignDataProvider):
|
||||
"""Callsign data provider for HamQTH."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("HamQTH", provider_config, DATA_STORE.callsign_data_hamqth)
|
||||
self._HAMQTH_BASE_URL = "https://www.hamqth.com/xml.php"
|
||||
self._PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}".replace(" ", "_")
|
||||
@@ -33,7 +37,7 @@ class HamQTH(APIQueryCallsignDataProvider):
|
||||
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
|
||||
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/hamqth-creds", expire_after=timedelta(minutes=55))
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
# If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because
|
||||
# # someone might provide credentials next time around.
|
||||
if not lookup_credentials or not (
|
||||
@@ -117,7 +121,7 @@ class HamQTH(APIQueryCallsignDataProvider):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def hamqth_response_to_callsign(callsign, data):
|
||||
def hamqth_response_to_callsign(callsign: str, data: dict[str, Any]) -> Callsign:
|
||||
"""Convert the "Callsign" block in HamQTH's API response to our own Callsign object."""
|
||||
|
||||
# Check for sensible latitudes
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import xmltodict
|
||||
@@ -13,6 +16,7 @@ from core.data_store import CACHE_DIR, DATA_STORE
|
||||
from core.enums import Continent, LocationSourceForCallsign
|
||||
from core.url_data_cache import URLDataCache
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -21,7 +25,7 @@ logger = logging.getLogger(__name__)
|
||||
class QRZ(APIQueryCallsignDataProvider):
|
||||
"""Callsign data provider for QRZ.com."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("QRZ.com", provider_config, DATA_STORE.callsign_data_qrz)
|
||||
self._QRZ_BASE_URL = "https://xmldata.qrz.com/xml/current/"
|
||||
self._URL_DATA_CACHE = URLDataCache("qrz")
|
||||
@@ -29,7 +33,7 @@ class QRZ(APIQueryCallsignDataProvider):
|
||||
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
|
||||
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/qrz-creds", expire_after=timedelta(minutes=55))
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
# If we don't have QRZ credentials, skip this lookup. Return None so we don't *cache* the lack of data, because
|
||||
# someone might provide credentials next time around.
|
||||
if not lookup_credentials or not (
|
||||
@@ -125,7 +129,7 @@ class QRZ(APIQueryCallsignDataProvider):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def qrz_response_to_callsign(callsign, data):
|
||||
def qrz_response_to_callsign(callsign: str, data: dict[str, Any] | list[Any]) -> Callsign:
|
||||
"""Convert the "Callsign" block in QRZ's API response to our own Callsign object."""
|
||||
|
||||
# I have encountered a user passing multiple callsigns to the QRZ lookup function in a way that QRZ actually
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -31,17 +34,17 @@ class GIROIonosonde(SolarConditionsProvider):
|
||||
Designed to run alongside KC2GProp even though they produce similar data. GIRO has more stations and includes LUF
|
||||
data, but is less reliable and often offline."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("GIRO Ionosonde Data", provider_config)
|
||||
self._stations = self._load_stations()
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._stations: list[dict[str, str]] = self._load_stations()
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
# Pre-populate ionosonde_data with known station names for stations not already present,
|
||||
# so the station dropdown is available before the first poll. Does not overwrite existing
|
||||
# entries so KC2G cache data is preserved.
|
||||
existing = self._solar_conditions.ionosonde_data or {}
|
||||
new_entries = {
|
||||
existing: dict[str, Any] = self._solar_conditions.ionosonde_data or {}
|
||||
new_entries: dict[str, Any] = {
|
||||
s["ursi"]: {
|
||||
"ursi": s["ursi"],
|
||||
"name": s["name"],
|
||||
@@ -57,27 +60,27 @@ class GIROIonosonde(SolarConditionsProvider):
|
||||
self.update_data({"ionosonde_data": {**existing, **new_entries}})
|
||||
|
||||
@staticmethod
|
||||
def _load_stations():
|
||||
stations = []
|
||||
def _load_stations() -> list[dict[str, str]]:
|
||||
stations: list[dict[str, str]] = []
|
||||
with open(STATIONS_INDEX, newline="") as f:
|
||||
for row in csv.reader(f):
|
||||
if len(row) >= 2:
|
||||
stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
|
||||
return stations
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
|
||||
self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning("GIRO ionosonde worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
# Real interval at which we poll is the "once per hour" divided by the number of stations, so each one gets
|
||||
# polled once per hour, just not all at once
|
||||
interval = POLL_INTERVAL / len(self._stations)
|
||||
@@ -88,7 +91,7 @@ class GIROIonosonde(SolarConditionsProvider):
|
||||
if self._stop_event.wait(timeout=interval):
|
||||
break
|
||||
|
||||
def _poll_station(self, station):
|
||||
def _poll_station(self, station: dict[str, str]) -> None:
|
||||
ursi = station["ursi"]
|
||||
name = station["name"]
|
||||
try:
|
||||
@@ -139,7 +142,9 @@ class GIROIonosonde(SolarConditionsProvider):
|
||||
self.status = "Error"
|
||||
logger.exception(f"Exception fetching GIRO ionosonde data for {ursi} ({name})")
|
||||
|
||||
def _fetch_station_data(self, ursi, from_time, to_time):
|
||||
def _fetch_station_data(
|
||||
self, ursi: str, from_time: datetime, to_time: datetime
|
||||
) -> tuple[dict[float, float] | None, dict[float, float] | None, dict[float, float] | None]:
|
||||
"""Fetch foF2, MUF and LUF readings for a station. Returns (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""
|
||||
|
||||
from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S")
|
||||
@@ -159,12 +164,12 @@ class GIROIonosonde(SolarConditionsProvider):
|
||||
return None, None, None
|
||||
|
||||
@staticmethod
|
||||
def _parse_all(text):
|
||||
def _parse_all(text: str) -> tuple[dict[float, float], dict[float, float], dict[float, float]]:
|
||||
"""Parse web server response and return (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""
|
||||
|
||||
fof2_data = {}
|
||||
muf_data = {}
|
||||
luf_data = {}
|
||||
fof2_data: dict[float, float] = {}
|
||||
muf_data: dict[float, float] = {}
|
||||
luf_data: dict[float, float] = {}
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from xml.etree import ElementTree
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from dateutil import parser as dateutil_parser
|
||||
from dateutil import tz as dateutil_tz
|
||||
|
||||
@@ -19,10 +23,10 @@ class HamQSL(HTTPSolarConditionsProvider):
|
||||
"""Solar conditions provider using the HamQSL.com XML API (https://www.hamqsl.com/solarxml.php).
|
||||
Provides solar flux index, geomagnetic indices, and HF/VHF propagation condition summaries."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("HamQSL", provider_config, URL, POLL_INTERVAL)
|
||||
|
||||
def _http_response_to_solar_conditions(self, http_response):
|
||||
def _http_response_to_solar_conditions(self, http_response: requests.Response) -> dict[str, Any] | None:
|
||||
root = ElementTree.fromstring(http_response.text)
|
||||
sd = root.find("solardata")
|
||||
if sd is None:
|
||||
@@ -31,27 +35,27 @@ class HamQSL(HTTPSolarConditionsProvider):
|
||||
|
||||
# Some error checking functions in case the data is janky.
|
||||
|
||||
def text(tag, default=None):
|
||||
def text(tag: str, default: str | None = None) -> str | None:
|
||||
if sd is None:
|
||||
logger.warning("HamQSL solar conditions API returned unexpected XML structure")
|
||||
return default
|
||||
el = sd.find(tag)
|
||||
return el.text.strip() if el is not None and el.text else default
|
||||
|
||||
def float_val(tag, default=None):
|
||||
def float_val(tag: str, default: float | None = None) -> float | None:
|
||||
try:
|
||||
return float(text(tag))
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
def int_val(tag, default=None):
|
||||
def int_val(tag: str, default: int | None = None) -> int | None:
|
||||
try:
|
||||
return int(text(tag))
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
# Process HF band conditions
|
||||
hf_conditions = {}
|
||||
hf_conditions: dict[str, str] = {}
|
||||
calc = sd.find("calculatedconditions")
|
||||
if calc is not None:
|
||||
for band_el in calc.findall("band"):
|
||||
@@ -62,7 +66,7 @@ class HamQSL(HTTPSolarConditionsProvider):
|
||||
hf_conditions[f"{name}-{time}"] = condition
|
||||
|
||||
# Process VHF propagation conditions
|
||||
vhf_map = {}
|
||||
vhf_map: dict[tuple[str | None, str | None], str | None] = {}
|
||||
vhf = sd.find("calculatedvhfconditions")
|
||||
if vhf is not None:
|
||||
for ph_el in vhf.findall("phenomenon"):
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -16,32 +19,32 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
|
||||
"""Generic solar conditions provider for providers that request data via HTTP(S). Subclasses implement
|
||||
_http_response_to_solar_conditions() to parse the specific API response format."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
super().__init__(name, provider_config)
|
||||
self._url = url
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._url: str = url
|
||||
self._poll_interval: float = poll_interval
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.info(f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
|
||||
self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} solar conditions worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=self._poll_interval):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
logger.debug(f"Polling {self.name} solar conditions API...")
|
||||
http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
|
||||
@@ -66,7 +69,7 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
|
||||
logger.exception(f"Exception in HTTP Solar Conditions Provider ({self.name})")
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _http_response_to_solar_conditions(self, http_response):
|
||||
def _http_response_to_solar_conditions(self, http_response: requests.Response) -> dict[str, Any] | None:
|
||||
"""Convert an HTTP response into solar conditions data. Returns a dict mapping SolarConditions field
|
||||
names to their new values, or None if the response could not be parsed. Only the fields returned will
|
||||
be updated on the shared SolarConditions object; any fields not included will be left unchanged."""
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from core.constants import BANDS
|
||||
from data.band import Band
|
||||
|
||||
HF_BANDS = [b for b in BANDS if b.is_ham_hf]
|
||||
HF_BANDS: list[Band] = [b for b in BANDS if b.is_ham_hf]
|
||||
|
||||
|
||||
def _latest(d) -> float | None:
|
||||
def _latest(d: dict[float, float | str] | None) -> float | None:
|
||||
"""Given a map where the key is a timestamp and the value is a number represented as a string, find the latest
|
||||
timestamp and return the corresponding value as a float."""
|
||||
|
||||
@@ -11,7 +14,11 @@ def _latest(d) -> float | None:
|
||||
return float(val) if (val is not None and val != "None") else None
|
||||
|
||||
|
||||
def compute_band_states(fof2_dict, muf_dict, luf_dict):
|
||||
def compute_band_states(
|
||||
fof2_dict: dict[float, float | str] | None,
|
||||
muf_dict: dict[float, float | str] | None,
|
||||
luf_dict: dict[float, float | str] | None,
|
||||
) -> dict[str, str]:
|
||||
"""Compute HF band states from the latest foF2, MUF and LUF values.
|
||||
|
||||
Returns a map where the keys are HF bands and the values are as follows:
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -25,30 +28,30 @@ class KC2GProp(SolarConditionsProvider):
|
||||
Designed to run alongside GIROIonosonde even though they produce similar data. KC2G is more reliable and is always
|
||||
online, but has fewer stations and does not provide LUF data."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("KC2G Propagation Data", provider_config)
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
|
||||
self._thread = Thread(target=self._run, name="KC2GPropProvider", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning("KC2G ionosonde worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=POLL_INTERVAL):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
logger.debug("Polling KC2G ionosonde data...")
|
||||
http_response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
|
||||
@@ -61,7 +64,7 @@ class KC2GProp(SolarConditionsProvider):
|
||||
|
||||
# Start from existing ionosonde_data so the accumulated time series survives across polls and restarts and
|
||||
# stations provided only by GIROIonosonde are not discarded
|
||||
ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
|
||||
ionosonde_data: dict[str, Any] = dict(self._solar_conditions.ionosonde_data or {})
|
||||
updated_count = 0
|
||||
|
||||
for reading in http_response.json():
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from providers.solarconditions.http_solar_conditions_provider import (
|
||||
HTTPSolarConditionsProvider,
|
||||
@@ -16,11 +21,11 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
"""Solar conditions provider using the NOAA 3-day forecast text file. Parses the NOAA forecast and populates
|
||||
corresponding fields in the solar conditions object.."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("NOAA 3-day Forecast", provider_config, URL, POLL_INTERVAL)
|
||||
|
||||
@staticmethod
|
||||
def _parse_percentage_table(lines, section_header, year):
|
||||
def _parse_percentage_table(lines: list[str], section_header: str, year: int) -> dict[str, dict[float, int]] | None:
|
||||
"""Find and parse a forecast table using percentages, identified by section_header. This is common to the lookup
|
||||
of the solar storm and radio blackout forecast parsing."""
|
||||
|
||||
@@ -48,7 +53,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
return None
|
||||
|
||||
# Figure out the date based on the line found
|
||||
column_timestamps = []
|
||||
column_timestamps: list[float] = []
|
||||
for month_str, day_str in date_matches:
|
||||
try:
|
||||
dt = datetime.strptime(f"{day_str} {month_str} {year}", "%d %b %Y").replace(tzinfo=timezone.utc)
|
||||
@@ -58,7 +63,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
return None
|
||||
|
||||
# Parse data rows. Each non-empty line should have a text label followed by percentage values
|
||||
result = {}
|
||||
result: dict[str, dict[float, int]] = {}
|
||||
for line in lines[date_header_idx + 1 :]:
|
||||
line_stripped = line.strip()
|
||||
if not line_stripped:
|
||||
@@ -73,7 +78,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
|
||||
# Row label is everything before the first percentage value
|
||||
row_label = line_stripped[: line_stripped.index(pct_matches[0].group())].strip()
|
||||
row_data = {}
|
||||
row_data: dict[float, int] = {}
|
||||
for j, match in enumerate(pct_matches):
|
||||
if j >= len(column_timestamps):
|
||||
break
|
||||
@@ -83,7 +88,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
|
||||
return result if result else None
|
||||
|
||||
def _http_response_to_solar_conditions(self, http_response):
|
||||
def _http_response_to_solar_conditions(self, http_response: requests.Response) -> dict[str, Any] | None:
|
||||
lines = http_response.text.splitlines()
|
||||
|
||||
# Find the "NOAA Kp index breakdown" section header
|
||||
@@ -115,7 +120,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
logger.warning(f"NOAA K-index forecast: could not parse date headers from: {date_header_line}")
|
||||
return None
|
||||
|
||||
column_dates = []
|
||||
column_dates: list[date] = []
|
||||
for month_str, day_str in date_matches:
|
||||
try:
|
||||
column_dates.append(
|
||||
@@ -126,7 +131,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
return None
|
||||
|
||||
# Parse each data row, e.g. "00-03UT 2.00 3.00 2.00"
|
||||
k_index_forecast = {}
|
||||
k_index_forecast: dict[float, float] = {}
|
||||
for line in lines[start_idx + 3 :]:
|
||||
time_match = re.match(r"^(\d{2})-(\d{2})UT\s+(.*)", line.strip())
|
||||
if not time_match:
|
||||
@@ -167,14 +172,14 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
return None
|
||||
|
||||
# Parse Solar Radiation Storm Forecast (single row: "S1 or greater")
|
||||
solar_storm_forecast = None
|
||||
solar_storm_forecast: dict[float, int] | None = None
|
||||
radiation_table = self._parse_percentage_table(lines, "Solar Radiation Storm Forecast", year)
|
||||
if radiation_table:
|
||||
solar_storm_forecast = radiation_table.get("S1 or greater")
|
||||
|
||||
# Parse Radio Blackout Forecast (two rows: "R1-R2" and "R3 or greater")
|
||||
blackout_forecast_r1r2 = None
|
||||
blackout_forecast_r3_or_greater = None
|
||||
blackout_forecast_r1r2: dict[float, int] | None = None
|
||||
blackout_forecast_r3_or_greater: dict[float, int] | None = None
|
||||
blackout_table = self._parse_percentage_table(lines, "Radio Blackout Forecast", year)
|
||||
if blackout_table:
|
||||
blackout_forecast_r1r2 = blackout_table.get("R1-R2")
|
||||
|
||||
@@ -1,34 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from data.solar_conditions import SolarConditions
|
||||
|
||||
|
||||
class SolarConditionsProvider:
|
||||
"""Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
|
||||
propagation data."""
|
||||
|
||||
def __init__(self, name, provider_config):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor"""
|
||||
|
||||
self.name = name
|
||||
self.enabled = provider_config.get("enabled", True)
|
||||
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Not Started" if self.enabled else "Disabled"
|
||||
self._solar_conditions = DATA_STORE.solar_conditions.get()
|
||||
self.name: str = name
|
||||
self.enabled: bool = provider_config.get("enabled", True)
|
||||
self.last_update_time: datetime = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status: str = "Not Started" if self.enabled else "Disabled"
|
||||
self._solar_conditions: SolarConditions = DATA_STORE.solar_conditions.get()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def update_data(self, new_data):
|
||||
def update_data(self, new_data: dict[str, Any] | None) -> None:
|
||||
"""Update the solar conditions object with new data"""
|
||||
|
||||
if new_data:
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import aprslib
|
||||
import pytz
|
||||
@@ -15,17 +18,17 @@ logger = logging.getLogger(__name__)
|
||||
class APRSIS(SpotProvider):
|
||||
"""Spot provider for the APRS-IS."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("APRS-IS", provider_config)
|
||||
self._thread = None
|
||||
self._aprsis = None
|
||||
self._stop_event = Event()
|
||||
self._thread: Thread | None = None
|
||||
self._aprsis: aprslib.IS | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
self._thread = Thread(target=self._run, name="APRSISSpotProvider", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self._aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN)
|
||||
@@ -43,7 +46,7 @@ class APRSIS(SpotProvider):
|
||||
if not self._stop_event.is_set():
|
||||
self._stop_event.wait(timeout=5)
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self.status = "Shutting down"
|
||||
self._stop_event.set()
|
||||
if self._aprsis:
|
||||
@@ -53,7 +56,7 @@ class APRSIS(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning("APRS-IS worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _handle(self, data):
|
||||
def _handle(self, data: dict[str, Any]) -> None:
|
||||
try:
|
||||
# Split SSID in "from" call and store separately
|
||||
from_parts = str(data["from"]).split("-")
|
||||
|
||||
+18
-15
@@ -1,16 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
from datetime import datetime
|
||||
from threading import Event, Lock, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import telnetlib3
|
||||
|
||||
from core.config import SERVER_OWNER_CALLSIGN
|
||||
from core.utils import decode_telnet_bytes
|
||||
from data.spot import Spot
|
||||
from providers.spot.spot_provider import SpotProvider
|
||||
from core.utils import decode_telnet_bytes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,29 +31,29 @@ class DXCluster(SpotProvider):
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor requires hostname and port"""
|
||||
|
||||
name = provider_config.get("name", "Cluster")
|
||||
super().__init__(name, provider_config)
|
||||
self._hostname = provider_config["host"]
|
||||
self._port = provider_config["port"]
|
||||
self._login_prompt = provider_config.get("login_prompt", "login:")
|
||||
self._login_callsign = provider_config.get("login_callsign", SERVER_OWNER_CALLSIGN)
|
||||
self._allow_rbn_spots = provider_config.get("allow_rbn_spots", False)
|
||||
self._spot_line_pattern = (
|
||||
self._hostname: str = provider_config["host"]
|
||||
self._port: int = provider_config["port"]
|
||||
self._login_prompt: str = provider_config.get("login_prompt", "login:")
|
||||
self._login_callsign: str = provider_config.get("login_callsign", SERVER_OWNER_CALLSIGN)
|
||||
self._allow_rbn_spots: bool = provider_config.get("allow_rbn_spots", False)
|
||||
self._spot_line_pattern: re.Pattern[str] = (
|
||||
self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
|
||||
)
|
||||
self._telnet = None
|
||||
self._telnet_lock = Lock()
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._telnet: telnetlib3.Telnet | None = None
|
||||
self._telnet_lock: Lock = Lock()
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
with self._telnet_lock:
|
||||
if self._telnet:
|
||||
@@ -64,7 +67,7 @@ class DXCluster(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"DX Cluster {self._hostname} worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _handle(self):
|
||||
def _handle(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
connected = False
|
||||
while not connected and not self._stop_event.is_set():
|
||||
|
||||
+11
-7
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from core.enums import ActivityName, ActivityRefType, Mode
|
||||
@@ -21,14 +25,14 @@ class GMA(HTTPSpotProvider):
|
||||
# GMA spots don't contain the details of the programme they are for, we need a separate lookup for that
|
||||
REF_INFO_URL_ROOT = "https://www.gma.rocks/api/ref/?"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
# Ensure there is an API key in our config, and set up the query URL using it. If no key is provided,
|
||||
# disable this spot provider.
|
||||
self._api_key = provider_config.get("api_key", "")
|
||||
self._api_key: str = provider_config.get("api_key", "")
|
||||
if self._api_key == "":
|
||||
provider_config["enabled"] = False
|
||||
logger.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
|
||||
self._url_data_cache = URLDataCache("GMA")
|
||||
self._url_data_cache: URLDataCache = URLDataCache("GMA")
|
||||
|
||||
super().__init__(
|
||||
"GMA",
|
||||
@@ -37,8 +41,8 @@ class GMA(HTTPSpotProvider):
|
||||
self.POLL_INTERVAL_SEC,
|
||||
)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
if "RCD" in http_response.json():
|
||||
for source_spot in http_response.json()["RCD"]:
|
||||
@@ -172,10 +176,10 @@ class GMA(HTTPSpotProvider):
|
||||
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.GMA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO: Implement.
|
||||
# Spotting to GMA is documented: https://www.cqgma.org/api/doc/apigma_spot.pdf We (or the user) need a GMA account, and to send the password in plaintext(!!)
|
||||
raise NotImplementedError("GMA upstream spot submission is not yet implemented")
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -27,17 +30,17 @@ class HEMA(HTTPSpotProvider):
|
||||
FREQ_MODE_PATTERN = re.compile("^([\\d.]*) \\((.*)\\)$")
|
||||
SPOTTER_COMMENT_PATTERN = re.compile("^\\((.*)\\) (.*)$")
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("HEMA", provider_config, self.SPOT_SEED_URL, self.POLL_INTERVAL_SEC)
|
||||
self._spot_seed = ""
|
||||
self._spot_seed: str = ""
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
# OK, source data is actually just the spot seed at this point. We'll then go on to fetch real data if we know
|
||||
# this has changed.
|
||||
spot_seed_changed = http_response.text != self._spot_seed
|
||||
self._spot_seed = http_response.text
|
||||
|
||||
new_spots = []
|
||||
new_spots: list[Spot] = []
|
||||
# OK, if the spot seed actually changed, now we make the real request for data.
|
||||
if spot_seed_changed:
|
||||
try:
|
||||
@@ -89,10 +92,10 @@ class HEMA(HTTPSpotProvider):
|
||||
logger.warning("Connection error when accessing HEMA spots API.")
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.HEMA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO: Implement. Currently blocked awaiting their API team to make a change to allow us to spot with a
|
||||
# reference and not a reference *number*.
|
||||
raise NotImplementedError("HEMA upstream spot submission is not yet implemented")
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, JSONDecodeError, ReadTimeout
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from data.spot import Spot
|
||||
from providers.spot.spot_provider import SpotProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -16,22 +20,22 @@ class HTTPSpotProvider(SpotProvider):
|
||||
"""Generic spot provider class for providers that request data via HTTP(S). Just for convenience to avoid code
|
||||
duplication. Subclasses of this query the individual APIs for data."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
super().__init__(name, provider_config)
|
||||
self._url = url
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._wakeup_event = Event()
|
||||
self._url: str = url
|
||||
self._poll_interval: float = poll_interval
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
self._wakeup_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
|
||||
# subsequent polls, so start() returns immediately and the application can continue starting.
|
||||
logger.info(f"Set up query of {self.name} spot API every {self._poll_interval!s} seconds.")
|
||||
self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
self._wakeup_event.set()
|
||||
if self._thread:
|
||||
@@ -39,12 +43,12 @@ class HTTPSpotProvider(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} spot worker thread did not exit on time and will be killed.")
|
||||
|
||||
def force_poll(self):
|
||||
def force_poll(self) -> None:
|
||||
"""Trigger an immediate poll without waiting for the normal interval."""
|
||||
|
||||
self._wakeup_event.set()
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._wakeup_event.clear()
|
||||
self._poll()
|
||||
@@ -52,7 +56,7 @@ class HTTPSpotProvider(SpotProvider):
|
||||
if self._stop_event.is_set():
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
# Request data from API
|
||||
logger.debug(f"Polling {self.name} spot API...")
|
||||
@@ -86,7 +90,7 @@ class HTTPSpotProvider(SpotProvider):
|
||||
logger.exception(f"Exception in HTTP Spot Provider ({self.name})")
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot] | None:
|
||||
"""Convert an HTTP response returned by the API into spot data. The whole response is provided here so the subclass
|
||||
implementations can check for HTTP status codes if necessary, and handle the response as JSON, XML, text, whatever
|
||||
the API actually provides."""
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType, Mode
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -12,11 +17,11 @@ class LLOTA(HTTPSpotProvider):
|
||||
POLL_INTERVAL_SEC = 120
|
||||
SPOTS_URL = "https://llota.app/api/public/spots"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("LLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
for source_spot in http_response.json():
|
||||
# Find the most recent spotter and comment from the history array
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import ClassVar
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -33,11 +35,11 @@ class ParksNPeaks(HTTPSpotProvider):
|
||||
ActivityName.SANPCPA,
|
||||
]
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
if http_response and http_response != "":
|
||||
for source_spot in http_response.json():
|
||||
@@ -117,10 +119,10 @@ class ParksNPeaks(HTTPSpotProvider):
|
||||
new_spots.append(spot)
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity in self.SUBMITTABLE_ACTIVITIES
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO test this works
|
||||
user_id = credentials.get("user_id", "")
|
||||
api_key = credentials.get("api_key", "")
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -17,11 +20,11 @@ class POTA(HTTPSpotProvider):
|
||||
SPOTS_URL = "https://api.pota.app/spot/activator"
|
||||
SUBMIT_URL = "https://api.pota.app/spot"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("POTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
for source_spot in http_response.json():
|
||||
# Convert to our spot format
|
||||
@@ -57,10 +60,10 @@ class POTA(HTTPSpotProvider):
|
||||
new_spots.append(spot)
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.POTA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
sig_ref = spot.sig_refs[0].id if spot.sig_refs else None
|
||||
if sig_ref:
|
||||
body = {
|
||||
|
||||
+13
-10
@@ -1,16 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
from datetime import datetime
|
||||
from threading import Event, Lock, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import telnetlib3
|
||||
|
||||
from core.config import SERVER_OWNER_CALLSIGN
|
||||
from core.utils import decode_telnet_bytes
|
||||
from data.spot import Spot
|
||||
from providers.spot.spot_provider import SpotProvider
|
||||
from core.utils import decode_telnet_bytes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -24,22 +27,22 @@ class RBN(SpotProvider):
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor requires port number."""
|
||||
|
||||
name = provider_config.get("name", "RBN")
|
||||
super().__init__(name, provider_config)
|
||||
self._port = provider_config["port"]
|
||||
self._telnet = None
|
||||
self._telnet_lock = Lock()
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._port: int = provider_config["port"]
|
||||
self._telnet: telnetlib3.Telnet | None = None
|
||||
self._telnet_lock: Lock = Lock()
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
self._thread = Thread(target=self._handle, name=f"RBNSpotProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
with self._telnet_lock:
|
||||
if self._telnet:
|
||||
@@ -53,7 +56,7 @@ class RBN(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"RBN (port {self._port!s}) worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _handle(self):
|
||||
def _handle(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
connected = False
|
||||
while not connected and not self._stop_event.is_set():
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import ClassVar
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
|
||||
@@ -27,17 +29,17 @@ class SOTA(HTTPSpotProvider):
|
||||
SUBMIT_URL = "https://api-db2.sota.org.uk/api/spots"
|
||||
VALID_MODES: ClassVar[list[str]] = ["AM", "CW", "Data", "DV", "FM", "SSB"]
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("SOTA", provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)
|
||||
self._api_epoch = ""
|
||||
self._api_epoch: str = ""
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
# OK, source data is actually just the epoch at this point. We'll then go on to fetch real data if we know this
|
||||
# has changed.
|
||||
epoch_changed = http_response.text != self._api_epoch
|
||||
self._api_epoch = http_response.text
|
||||
|
||||
new_spots = []
|
||||
new_spots: list[Spot] = []
|
||||
# OK, if the epoch actually changed, now we make the real request for data.
|
||||
if epoch_changed:
|
||||
try:
|
||||
@@ -83,10 +85,10 @@ class SOTA(HTTPSpotProvider):
|
||||
logger.warning("Timeout when accessing SOTA spots API.")
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.SOTA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO test this method works
|
||||
access_token = credentials.get("access_token", "")
|
||||
id_token = credentials.get("id_token", "")
|
||||
|
||||
@@ -1,30 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from core.live_data_cache import LiveDataCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Deferred to avoid a circular import: data.spot imports core.call_lookup_helper, which imports
|
||||
# core.data_providers, which imports this module.
|
||||
from data.spot import Spot
|
||||
|
||||
|
||||
class SpotProvider:
|
||||
"""Generic spot provider class. Subclasses of this query the individual APIs for data."""
|
||||
|
||||
def __init__(self, name, provider_config):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor"""
|
||||
|
||||
self.name = name
|
||||
self.enabled = provider_config.get("enabled", True)
|
||||
self.enabled_by_default_in_web_ui = provider_config.get("enabled_by_default_in_web_ui", True)
|
||||
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Not Started" if self.enabled else "Disabled"
|
||||
self._spots = DATA_STORE.spots
|
||||
self.name: str = name
|
||||
self.enabled: bool = provider_config.get("enabled", True)
|
||||
self.enabled_by_default_in_web_ui: bool = provider_config.get("enabled_by_default_in_web_ui", True)
|
||||
self.last_update_time: datetime = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.last_spot_time: datetime = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status: str = "Not Started" if self.enabled else "Disabled"
|
||||
self._spots: LiveDataCache[Spot] = DATA_STORE.spots
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def _submit_batch(self, spots):
|
||||
def _submit_batch(self, spots: list[Spot]) -> None:
|
||||
"""Submit a batch of spots retrieved from the provider. Only spots that are newer than the last spot retrieved
|
||||
by this provider will be added to the spot list, to prevent duplications. Spots passing the check will also have
|
||||
their infer_missing() method called to complete their data set. This is called by the API-querying
|
||||
@@ -41,7 +50,7 @@ class SpotProvider:
|
||||
if spots:
|
||||
self.last_spot_time = datetime.fromtimestamp(max(s.time for s in spots), pytz.UTC)
|
||||
|
||||
def _submit(self, spot):
|
||||
def _submit(self, spot: Spot) -> None:
|
||||
"""Submit a single spot retrieved from the provider. This will be added to the list regardless of its age. Spots
|
||||
passing the check will also have their infer_missing() method called to complete their data set. This is called by
|
||||
the data streaming subclasses, which can be relied upon not to re-provide old spots."""
|
||||
@@ -51,27 +60,27 @@ class SpotProvider:
|
||||
self._add_spot(spot)
|
||||
self.last_spot_time = datetime.fromtimestamp(spot.time, pytz.UTC)
|
||||
|
||||
def _add_spot(self, spot):
|
||||
def _add_spot(self, spot: Spot) -> None:
|
||||
if not spot.expired():
|
||||
self._spots.set(spot.id, spot)
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
"""Return True if this provider supports submitting spots upstream for the given activity."""
|
||||
|
||||
return False
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
"""Submit a spot upstream to this provider's API. credentials is a dict with provider-specific keys.
|
||||
Raises an exception with a descriptive message on failure."""
|
||||
|
||||
raise NotImplementedError("This provider does not support spot submission")
|
||||
|
||||
def force_poll(self):
|
||||
def force_poll(self) -> None:
|
||||
"""Trigger an immediate poll without waiting for the normal interval. Default implementation here does nothing
|
||||
because not all spot providers have a polling mechanism. Providers that do should override this method."""
|
||||
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Lock, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
from requests_sse import EventSource, InvalidStatusCodeError
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from data.spot import Spot
|
||||
from providers.spot.spot_provider import SpotProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -14,23 +18,23 @@ logger = logging.getLogger(__name__)
|
||||
class SSESpotProvider(SpotProvider):
|
||||
"""Spot provider using Server-Sent Events."""
|
||||
|
||||
def __init__(self, name, provider_config, url):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str) -> None:
|
||||
super().__init__(name, provider_config)
|
||||
self._url = url
|
||||
self._thread = None
|
||||
self._last_event_id = None
|
||||
self._stop_event = Event()
|
||||
self._event_source_lock = Lock()
|
||||
self._event_source = None
|
||||
self._url: str = url
|
||||
self._thread: Thread | None = None
|
||||
self._last_event_id: str | None = None
|
||||
self._stop_event: Event = Event()
|
||||
self._event_source_lock: Lock = Lock()
|
||||
self._event_source: EventSource | None = None
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.info(f"Set up SSE connection to {self.name} spot API.")
|
||||
self._stop_event.clear()
|
||||
self._thread = Thread(target=self._run, name=f"SSESpotProvider-{self.name}")
|
||||
self._thread.daemon = True
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
|
||||
with self._event_source_lock:
|
||||
@@ -46,17 +50,17 @@ class SSESpotProvider(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} SSE worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _on_open(self):
|
||||
def _on_open(self) -> None:
|
||||
self.status = "Waiting for Data"
|
||||
|
||||
def _on_error(self):
|
||||
def _on_error(self) -> None:
|
||||
self.status = "Connecting"
|
||||
|
||||
def _set_event_source(self, event_source):
|
||||
def _set_event_source(self, event_source: EventSource | None) -> None:
|
||||
with self._event_source_lock:
|
||||
self._event_source = event_source
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
logger.debug(f"Connecting to {self.name} spot API...")
|
||||
@@ -102,7 +106,7 @@ class SSESpotProvider(SpotProvider):
|
||||
self.status = "Disconnected"
|
||||
self._stop_event.wait(timeout=5) # Wait before trying to reconnect
|
||||
|
||||
def _sse_message_to_spot(self, message_data):
|
||||
def _sse_message_to_spot(self, message_data: str) -> Spot | None:
|
||||
"""Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass
|
||||
implementations can handle the message as JSON, XML, text, whatever the API actually provides."""
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import ClassVar
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import requests
|
||||
|
||||
@@ -36,11 +38,11 @@ class Tiles(HTTPSpotProvider):
|
||||
"Other",
|
||||
]
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
for source_spot in http_response.json()["spots"]:
|
||||
# Convert to our spot format
|
||||
@@ -84,10 +86,10 @@ class Tiles(HTTPSpotProvider):
|
||||
new_spots.append(spot)
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.TILES
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# Tiles on the air currently only supports *self* spots
|
||||
if spot.dx_call == spot.de_call:
|
||||
# Figure out a valid mode. Borrowed this from PoLo :)
|
||||
@@ -127,7 +129,7 @@ class Tiles(HTTPSpotProvider):
|
||||
|
||||
# Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles'
|
||||
# strange frequency format where we can sometimes have e.g. "14.123.5".
|
||||
def strip_extra_decimal_points(s):
|
||||
def strip_extra_decimal_points(s: str) -> str:
|
||||
parts = s.split(".", 1)
|
||||
if len(parts) == 1:
|
||||
return s
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class Towers(HTTPSpotProvider):
|
||||
POLL_INTERVAL_SEC = 120
|
||||
SPOTS_URL = "https://wwtota.com/api/cluster_live.php"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("Towers", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
response_fixed = http_response.text.replace("\\/", "/")
|
||||
response_json = json.loads(response_fixed)
|
||||
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import Mode
|
||||
from data.spot import Spot
|
||||
@@ -14,11 +18,11 @@ class UKPacketNet(HTTPSpotProvider):
|
||||
POLL_INTERVAL_SEC = 600
|
||||
SPOTS_URL = "https://nodes.ukpacketradio.network/api/nodedata"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("UK Packet Net", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
nodes = http_response.json()["nodes"]
|
||||
for node in nodes.values():
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user