mirror of
https://git.ianrenton.com/ian/spothole.git
synced 2026-09-21 06:47:42 +00:00
Autogenerated type safety parameterisation of all methods
This commit is contained in:
@@ -1,10 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from data.activity_ref import ActivityRef
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -13,29 +17,29 @@ class ActivityRefDataProvider:
|
||||
"""Generic activity reference data provider class. Subclasses of this query the individual URLs or files for
|
||||
data."""
|
||||
|
||||
def __init__(self, sig_name, provider_config):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor. Note the parameter and attribute are still named "sig_name" for consistency with the API's
|
||||
"sig" field name."""
|
||||
|
||||
self.sig_name = sig_name
|
||||
self.enabled = provider_config["enabled"]
|
||||
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Not Started" if self.enabled else "Disabled"
|
||||
self.reference_count = 0
|
||||
self.enabled: bool = provider_config["enabled"]
|
||||
self.last_update_time: datetime = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status: str = "Not Started" if self.enabled else "Disabled"
|
||||
self.reference_count: int = 0
|
||||
self._stop_event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
|
||||
super()."""
|
||||
|
||||
self._stop_event.set()
|
||||
|
||||
def _add_data(self, new_data):
|
||||
def _add_data(self, new_data: list[ActivityRef]) -> None:
|
||||
"""Add all the provided reference data objects to the data store."""
|
||||
|
||||
# with transact() batches all writes together to save making thousands of individual sqlite writes. However,
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class ARLHS(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.ARLHS
|
||||
DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
if "ARLHS" in row and row["ARLHS"] != "":
|
||||
ref_id = row["ARLHS"]
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -14,11 +18,11 @@ class COTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.COTA
|
||||
DATA_URL = "https://www.cotagroup.org/cotagroup/map/data/castles-all-7d90ee2a5e1175e5dece1bbf9dc87504.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data[2]:
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class DCE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DCE
|
||||
DATA_URL = "https://www.acracb.org/dce/descargas/General/directorio_referencias_dce.xls"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
file_stream = io.BytesIO(http_response.content)
|
||||
df = pd.read_excel(file_stream, engine="xlrd", header=None)
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class DEFE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DEFE
|
||||
DATA_URL = "https://www.acracb.org/defe/descargas/General/directorio_referencias_defe.xls"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
file_stream = io.BytesIO(http_response.content)
|
||||
df = pd.read_excel(file_stream, engine="xlrd", header=None)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
@@ -16,11 +19,11 @@ class DME(LocalFileActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DME
|
||||
PATH = "datafiles/MUNICIPIOS.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.PATH)
|
||||
|
||||
def _file_to_data(self, path):
|
||||
new_data = []
|
||||
def _file_to_data(self, path: str) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
with open(path, encoding="latin-1") as _f:
|
||||
for row in csv.DictReader(_f, delimiter=";"):
|
||||
# Store reference IDs with the "DME-" prefix rather than just the number. This will prevent Spothole
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -13,11 +18,11 @@ class DMUE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DMUE
|
||||
DATA_URL = "https://dmue.radiogalena.es/nom_dmue.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
for row in csv.reader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
|
||||
if len(row) > 1 and row[0] and row[1]:
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class DMVE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DMVE
|
||||
DATA_URL = "https://www.acracb.org/dmve/descargas/General/directorio_referencias_dmve.xls"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
file_stream = io.BytesIO(http_response.content)
|
||||
# Despide the .xls extension this is actually an xlsx file, so we need openpyxl not xlrd
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -12,11 +17,11 @@ class DTMBA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DTMBA
|
||||
DATA_URL = "https://www.iu1fig.com/share/iz0eik/dtmba/export.php"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in http_response.content.decode("utf-8-sig").splitlines():
|
||||
split = row.split(";")
|
||||
ref_id = split[0]
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pdfplumber
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class FEA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.FEA
|
||||
DATA_URL = "http://ea5ol.net/Lista%20Faros.pdf"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
# Use PDFPlumber to extract the tables in the PDF
|
||||
with pdfplumber.open(BytesIO(http_response.content)) as pdf:
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from core.url_data_cache import URLDataCache
|
||||
from data.activity_ref import ActivityRef
|
||||
from providers.activityrefdata.activity_ref_data_provider import ActivityRefDataProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -16,35 +21,35 @@ class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
|
||||
"""Generic activity ref data provider class for providers that fetch their data from the web by downloading a
|
||||
file."""
|
||||
|
||||
def __init__(self, sig_name, provider_config, url, poll_interval):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
"""Set up the provider, note poll_interval is in *days*."""
|
||||
super().__init__(sig_name, provider_config)
|
||||
self._url = url
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._thread: Thread | None = None
|
||||
self._url_data_cache = URLDataCache(f"activity_ref_data_{sig_name}")
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
|
||||
# subsequent polls, so start() returns immediately and the application can continue starting.
|
||||
logger.info(f"Set up query of {self.sig_name} activity ref data every {self._poll_interval!s} days.")
|
||||
self._thread = Thread(target=self._run, name=f"FileDownloadActivityRefDataProvider-{self.sig_name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
super().stop()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.sig_name} activity ref data worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
# Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
|
||||
# caching, but just so continual restarts of the software during testing don't hammer the servers.
|
||||
@@ -76,7 +81,7 @@ class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
|
||||
logger.exception(f"Exception in HTTP Activity Ref Data Provider ({self.sig_name})")
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
"""Convert an HTTP response returned by the server into activity ref data. The whole response is provided here
|
||||
so the subclass implementations can check for HTTP status codes if necessary, and handle the response as
|
||||
JSON, CSV, whatever the remote file actually is."""
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class GMA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.GMA
|
||||
DATA_URL = "https://www.gma.rocks/download/summits.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
ref_id = row["Reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class ILLW(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.ILLW
|
||||
DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
if "ILLW" in row and row["ILLW"] != "":
|
||||
ref_id = row["ILLW"]
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -19,11 +23,11 @@ class IOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.IOTA
|
||||
DATA_URL = "https://www.iota-world.org/islands-on-the-air/downloads/download-file.html?path=groups.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data:
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.enums import ActivityName
|
||||
from providers.activityrefdata.pnp_kml_activity_ref_data_provider import (
|
||||
ParksNPeaksKMLActivityRefDataProvider,
|
||||
@@ -11,5 +15,5 @@ class KRMNPA(ParksNPeaksKMLActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.KRMNPA
|
||||
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=KRMNPA&poiFormat=4"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import locator_to_latlong
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -16,11 +20,11 @@ class LLOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.LLOTA
|
||||
DATA_URL = "https://llota.app/api/public/references"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data:
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
from data.activity_ref import ActivityRef
|
||||
from providers.activityrefdata.activity_ref_data_provider import ActivityRefDataProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -11,11 +15,11 @@ logger = logging.getLogger(__name__)
|
||||
class LocalFileActivityRefDataProvider(ActivityRefDataProvider):
|
||||
"""Generic activity ref data provider class for providers that fetch their data from a local file on startup."""
|
||||
|
||||
def __init__(self, sig_name, provider_config, path):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any], path: str) -> None:
|
||||
super().__init__(sig_name, provider_config)
|
||||
self._path = path
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.debug(f"Loading {self.sig_name} activity ref data from file.")
|
||||
try:
|
||||
new_data = self._file_to_data(self._path)
|
||||
@@ -30,7 +34,7 @@ class LocalFileActivityRefDataProvider(ActivityRefDataProvider):
|
||||
self.status = "Error"
|
||||
logger.exception(f"Exception in local file Activity Ref Data Provider ({self.sig_name})")
|
||||
|
||||
def _file_to_data(self, path):
|
||||
def _file_to_data(self, path: str) -> list[ActivityRef]:
|
||||
"""Load a file on the given path and turn it into activity ref data."""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class MOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.MOTA
|
||||
DATA_URL = "https://www.gma.rocks/download/mills.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
ref_id = row["Reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -14,11 +18,11 @@ class PGA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.PGA
|
||||
DATA_URL = "http://www.spga.pl/lista_pga2.php"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
soup = BeautifulSoup(http_response.text, "html.parser")
|
||||
|
||||
# Iterate through tables in the page
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from fastkml import kml
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
@@ -17,12 +21,12 @@ class ParksNPeaksKMLActivityRefDataProvider(FileDownloadActivityRefDataProvider)
|
||||
|
||||
REF_PATTERN = re.compile(r"VKFF-\d+")
|
||||
|
||||
def __init__(self, sig_name, provider_config, url, poll_interval):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
"""Set up the provider, note poll_interval is in *days*."""
|
||||
super().__init__(sig_name, provider_config, url, poll_interval)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
k = kml.KML.from_string(http_response.content)
|
||||
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class POTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.POTA
|
||||
DATA_URL = "https://pota.app/all_parks_ext.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.enums import ActivityName
|
||||
from providers.activityrefdata.pnp_kml_activity_ref_data_provider import (
|
||||
ParksNPeaksKMLActivityRefDataProvider,
|
||||
@@ -11,5 +15,5 @@ class SANPCPA(ParksNPeaksKMLActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.SANPCPA
|
||||
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=SANPCPA&poiFormat=4"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class SIOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.SIOTA
|
||||
DATA_URL = "https://www.silosontheair.com/data/silos.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["SILO_CODE"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -17,11 +21,11 @@ class SOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.SOTA
|
||||
DATA_URL = "https://storage.sota.org.uk/summitslist.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
ref_id = row["SummitCode"]
|
||||
latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from typing import Any
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -13,11 +16,11 @@ class Toilets(LocalFileActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.TOILETS
|
||||
PATH = "datafiles/toilets.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.PATH)
|
||||
|
||||
def _file_to_data(self, path):
|
||||
new_data = []
|
||||
def _file_to_data(self, path: str) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
with open(path) as _f:
|
||||
csv_data = _f.read()
|
||||
dr = csv.DictReader(csv_data.splitlines())
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class Towers(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.TOWERS
|
||||
DATA_URL = "https://wwtota.com/servis/generate_csv.php?ref=&filter=all"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
|
||||
ref_id = row["Ref"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -20,11 +24,11 @@ class WCA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WCA
|
||||
DATA_URL = "https://polo.ham2k.com/data/activities/wca/all-castles.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["REF"]
|
||||
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,11 +19,11 @@ class WOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WOTA
|
||||
DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for feature in http_response.json().get("features", []):
|
||||
ref_id = feature["properties"]["wotaId"]
|
||||
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class WWBOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WWBOTA
|
||||
DATA_URL = "https://api.wwbota.org/bunkers/?format=CSV"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["Reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class WWFF(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WWFF
|
||||
DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -16,11 +20,11 @@ class ZLOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.ZLOTA
|
||||
DATA_URL = "https://ontheair.nz/assets/assets.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data:
|
||||
|
||||
@@ -1,28 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from core.live_data_cache import LiveDataCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Deferred to avoid a circular import: data.alert imports core.call_lookup_helper, which imports
|
||||
# core.data_providers, which imports this module.
|
||||
from data.alert import Alert
|
||||
|
||||
|
||||
class AlertProvider:
|
||||
"""Generic alert provider class. Subclasses of this query the individual APIs for alerts."""
|
||||
|
||||
def __init__(self, name, provider_config):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor"""
|
||||
|
||||
self.name = name
|
||||
self.enabled = provider_config.get("enabled", True)
|
||||
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Not Started" if self.enabled else "Disabled"
|
||||
self._alerts = DATA_STORE.alerts
|
||||
self._alerts: LiveDataCache[Alert] = DATA_STORE.alerts
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def _submit_batch(self, alerts):
|
||||
def _submit_batch(self, alerts: list[Alert]) -> None:
|
||||
"""Submit a batch of alerts retrieved from the provider. There is no timestamp checking like there is for spots,
|
||||
because alerts could be created at any point for any time in the future. Rely on hashcode-based id matching
|
||||
to deal with duplicates."""
|
||||
@@ -35,11 +44,11 @@ class AlertProvider:
|
||||
alert.infer_missing()
|
||||
self._add_alert(alert)
|
||||
|
||||
def _add_alert(self, alert):
|
||||
def _add_alert(self, alert: Alert) -> None:
|
||||
if not alert.expired():
|
||||
self._alerts.set(alert.id, alert)
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from core.enums import ActivityName
|
||||
@@ -15,10 +19,10 @@ class BOTA(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://www.beachesontheair.com/"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("BOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Find the table of upcoming alerts
|
||||
bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,10 +18,10 @@ class Hamsat(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://hams.at/api/alerts"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("Hamsat", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Iterate through source data
|
||||
for source_alert in http_response.json()["data"]:
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, JSONDecodeError, ReadTimeout
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from data.alert import Alert
|
||||
from providers.alert.alert_provider import AlertProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -16,34 +20,34 @@ class HTTPAlertProvider(AlertProvider):
|
||||
"""Generic alert provider class for providers that request data via HTTP(S). Just for convenience to avoid code
|
||||
duplication. Subclasses of this query the individual APIs for data."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: int) -> None:
|
||||
super().__init__(name, provider_config)
|
||||
self._url = url
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
|
||||
# subsequent polls, so start() returns immediately and the application can continue starting.
|
||||
logger.info(f"Set up query of {self.name} alert API every {self._poll_interval!s} seconds.")
|
||||
self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} alert worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=self._poll_interval):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
# Request data from API
|
||||
logger.debug(f"Polling {self.name} alert API...")
|
||||
@@ -78,7 +82,7 @@ class HTTPAlertProvider(AlertProvider):
|
||||
# Brief pause on error before the next poll, but still respond promptly to stop()
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
"""Convert an HTTP response returned by the API into alert data. The whole response is provided here so the subclass
|
||||
implementations can check for HTTP status codes if necessary, and handle the response as JSON, XML, text, whatever
|
||||
the API actually provides."""
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
from datetime import datetime, time
|
||||
from typing import cast
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime, time
|
||||
from typing import Any, cast
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from icalendar import Calendar, Event
|
||||
|
||||
from data.alert import Alert
|
||||
@@ -12,10 +15,10 @@ class ICALAlertProvider(HTTPAlertProvider):
|
||||
"""Generic alert provider for iCal calendars. Defines an abstract method event_to_alert(event) that subclasses must
|
||||
implement, and use it to convert an iCal event to an Alert object based on whatever format their iCal events use."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: int) -> None:
|
||||
super().__init__(name, provider_config, url, poll_interval)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
cal = Calendar.from_ical(http_response.content)
|
||||
|
||||
@@ -34,7 +37,7 @@ class ICALAlertProvider(HTTPAlertProvider):
|
||||
"""Convert an ICal event to an Alert object. Subclasses must implement this method."""
|
||||
|
||||
@staticmethod
|
||||
def _to_utc_timestamp(value):
|
||||
def _to_utc_timestamp(value: datetime | date) -> float:
|
||||
"""Convert a date or datetime value from an iCal field into a UTC UNIX timestamp."""
|
||||
|
||||
# Datetime object so we can treat it as-is, check if it has a non-UTC tz and convert it if necessary
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import cast
|
||||
from typing import Any, cast
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from rss_parser import Parser
|
||||
from rss_parser.models.rss import RSS
|
||||
|
||||
@@ -18,10 +21,10 @@ class NG3K(HTTPAlertProvider):
|
||||
ALERTS_URL = "https://www.ng3k.com/adxo.xml"
|
||||
AS_CALL_PATTERN = re.compile("as ([a-z0-9/]+)", re.IGNORECASE)
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("NG3K", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
|
||||
# Iterate through source data
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -17,10 +21,10 @@ class ParksNPeaks(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://parksnpeaks.org/api/ALERTS/"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("ParksNPeaks", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Iterate through source data
|
||||
for source_alert in http_response.json():
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,10 +18,10 @@ class POTA(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://api.pota.app/activation"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("POTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Iterate through source data
|
||||
for source_alert in http_response.json():
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from icalendar import Event
|
||||
|
||||
@@ -12,7 +15,7 @@ class RSGBICALAlertProvider(ICALAlertProvider):
|
||||
handling specific to how RSGB's iCal events are formatted. This is still effectively an abstract class itself;
|
||||
RSGB has two contest calendars (HF & VHF) that each subclass this."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: int) -> None:
|
||||
super().__init__(name, provider_config, url, poll_interval)
|
||||
|
||||
FREQ_PATTERN = re.compile(r"([\d.]+(?:MHz|GHz))|SHF")
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from providers.alert.rsgb_ical_alert_provider import RSGBICALAlertProvider
|
||||
|
||||
|
||||
@@ -7,5 +11,5 @@ class RSGBHFContests(RSGBICALAlertProvider):
|
||||
POLL_INTERVAL_DAYS = 30
|
||||
ALERTS_URL = "https://calendar.google.com/calendar/ical/a5ff31ebb1b4834dc7fff4c5415ae8251c6a9aa11f98c6af6e472b6c552b1915%40group.calendar.google.com/public/basic.ics"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("RSGB HF Contests", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60)
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from providers.alert.rsgb_ical_alert_provider import RSGBICALAlertProvider
|
||||
|
||||
|
||||
@@ -7,5 +11,5 @@ class RSGBVHFContests(RSGBICALAlertProvider):
|
||||
POLL_INTERVAL_DAYS = 30
|
||||
ALERTS_URL = "https://calendar.google.com/calendar/ical/40f3552bff39a016f1cdca205864177070dcad68d55be17eb061cb021f39f96c%40group.calendar.google.com/public/basic.ics"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("RSGB VHF Contests", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60)
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,10 +18,10 @@ class SOTA(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://api-db2.sota.org.uk/api/alerts/365/all/all"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("SOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Iterate through source data
|
||||
for source_alert in http_response.json():
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from icalendar import Event
|
||||
|
||||
from core.enums import ActivityName
|
||||
@@ -11,7 +15,7 @@ class WA7BNM(ICALAlertProvider):
|
||||
POLL_INTERVAL_DAYS = 1
|
||||
ALERTS_URL = "https://contestcalendar.com/weeklycontcustom.php"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(
|
||||
"WA7BNM Contest Calendar", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_DAYS * 24 * 60 * 60
|
||||
)
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import cast
|
||||
from typing import Any, cast
|
||||
from xml.parsers.expat import ExpatError
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from rss_parser import Parser as RSSParser
|
||||
from rss_parser.models.rss import RSS
|
||||
|
||||
@@ -22,10 +25,10 @@ class WOTA(HTTPAlertProvider):
|
||||
ALERTS_URL = "https://www.wota.org.uk/alerts_rss.php"
|
||||
RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("WOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,10 +18,10 @@ class WWFF(HTTPAlertProvider):
|
||||
POLL_INTERVAL_SEC = 1800
|
||||
ALERTS_URL = "https://spots.wwff.co/static/agendas.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("WWFF", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_alerts(self, http_response):
|
||||
def _http_response_to_alerts(self, http_response: requests.Response) -> list[Alert]:
|
||||
new_alerts = []
|
||||
# Iterate through source data
|
||||
for source_alert in http_response.json():
|
||||
|
||||
@@ -1,18 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import diskcache
|
||||
|
||||
from providers.callsigndata.callsign_data_provider import CallsignDataProvider
|
||||
|
||||
|
||||
class APIQueryCallsignDataProvider(CallsignDataProvider):
|
||||
"""Generic callsign data provider class for providers that fetch their data from the web on-demand using an API."""
|
||||
|
||||
def __init__(self, name, provider_config, storage):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], storage: diskcache.Cache) -> None:
|
||||
"""Set up the provider."""
|
||||
super().__init__(name, provider_config, storage)
|
||||
|
||||
if self.enabled:
|
||||
self.status = "Ready"
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
pass
|
||||
|
||||
@@ -1,15 +1,21 @@
|
||||
from datetime import datetime
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import diskcache
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
|
||||
|
||||
class CallsignDataProvider:
|
||||
"""Generic callsign reference data provider class. Subclasses of this set up the various mechanisms via which
|
||||
Spothole can look up data for callsigns."""
|
||||
|
||||
def __init__(self, name, provider_config, storage):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], storage: diskcache.Cache) -> None:
|
||||
"""Constructor. As well as name and config, provide the storage object from DATA_STORE that will be used to
|
||||
store the result of lookups to speed up future access."""
|
||||
|
||||
@@ -21,18 +27,18 @@ class CallsignDataProvider:
|
||||
self.lookup_count = 0
|
||||
self._storage = storage
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access remote resources, if
|
||||
needed."""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def lookup(self, callsign, lookup_credentials):
|
||||
def lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
"""Looks up data for the provided callsign. Takes a LookupCredentials object, which provides any credentials
|
||||
that have been provided by the user for this session (QRZ.com/HamQTH) to allow us to look up using those
|
||||
services on the user's behalf. (Clublog is looked up using an API key owned by the server and provided in its
|
||||
@@ -57,7 +63,7 @@ class CallsignDataProvider:
|
||||
else:
|
||||
return None
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
"""Makes a new request to the data source for callsign data."""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
from pyhamtools import Callinfo, LookupLib
|
||||
@@ -7,6 +10,7 @@ from pyhamtools import Callinfo, LookupLib
|
||||
from core.data_store import DATA_STORE
|
||||
from core.utils import get_callsign_object_from_pyhamtools_callinfo
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
from providers.callsigndata.api_query_callsign_data_provider import (
|
||||
APIQueryCallsignDataProvider,
|
||||
)
|
||||
@@ -17,9 +21,9 @@ logger = logging.getLogger(__name__)
|
||||
class ClublogAPI(APIQueryCallsignDataProvider):
|
||||
"""Callsign data provider for Clublog's API."""
|
||||
|
||||
_callinfo = None
|
||||
_callinfo: Callinfo | None = None
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
# API key required for this provider
|
||||
self._api_key = provider_config.get("api_key", "")
|
||||
if self._api_key != "":
|
||||
@@ -33,7 +37,7 @@ class ClublogAPI(APIQueryCallsignDataProvider):
|
||||
|
||||
super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
callsign_data = Callsign(call=callsign)
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from pyhamtools import Callinfo, LookupLib
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from core.utils import get_callsign_object_from_pyhamtools_callinfo
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
from providers.callsigndata.file_download_callsign_data_provider import (
|
||||
FileDownloadCallsignDataProvider,
|
||||
)
|
||||
@@ -20,9 +24,9 @@ class ClublogXML(FileDownloadCallsignDataProvider):
|
||||
DATA_URL = "https://cdn.clublog.org/cty.php"
|
||||
CACHE_PATH_ZIPPED = "cache/cty.xml.gz"
|
||||
CACHE_PATH_UNZIPPED = "cache/cty.xml"
|
||||
_callinfo = None
|
||||
_callinfo: Callinfo | None = None
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
# API key required for this provider
|
||||
self._api_key = provider_config.get("api_key", "")
|
||||
if self._api_key == "":
|
||||
@@ -40,7 +44,7 @@ class ClublogXML(FileDownloadCallsignDataProvider):
|
||||
DATA_STORE.callsign_data_clublogxml,
|
||||
)
|
||||
|
||||
def _handle_file(self, path):
|
||||
def _handle_file(self, path: str) -> bool:
|
||||
try:
|
||||
# The download from Clublog is gzipped so we need to uncompress that and re-save as a separate file that
|
||||
# the LookupLib can actually use.
|
||||
@@ -60,7 +64,7 @@ class ClublogXML(FileDownloadCallsignDataProvider):
|
||||
logger.exception("Exception when loading Clublog XML.")
|
||||
return False
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
callsign_data = Callsign(call=callsign)
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from pyhamtools import Callinfo, LookupLib
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from core.utils import get_callsign_object_from_pyhamtools_callinfo
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
from providers.callsigndata.file_download_callsign_data_provider import (
|
||||
FileDownloadCallsignDataProvider,
|
||||
)
|
||||
@@ -18,9 +22,9 @@ class CountryFiles(FileDownloadCallsignDataProvider):
|
||||
POLL_INTERVAL_DAYS = 30
|
||||
DATA_URL = "https://www.country-files.com/cty/cty.plist"
|
||||
CACHE_PATH = "cache/cty.plist"
|
||||
_callinfo = None
|
||||
_callinfo: Callinfo | None = None
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(
|
||||
"CountryFiles.com",
|
||||
provider_config,
|
||||
@@ -30,7 +34,7 @@ class CountryFiles(FileDownloadCallsignDataProvider):
|
||||
DATA_STORE.callsign_data_countryfiles,
|
||||
)
|
||||
|
||||
def _handle_file(self, path):
|
||||
def _handle_file(self, path: str) -> bool:
|
||||
try:
|
||||
lookuplib = LookupLib(lookuptype="countryfile", filename=path)
|
||||
self._callinfo = Callinfo(lookuplib)
|
||||
@@ -40,7 +44,7 @@ class CountryFiles(FileDownloadCallsignDataProvider):
|
||||
logger.exception("Exception when loading Country Files cty.plist.")
|
||||
return False
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
callsign_data = Callsign(call=callsign)
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import diskcache
|
||||
import pytz
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
|
||||
|
||||
@@ -15,40 +19,48 @@ logger = logging.getLogger(__name__)
|
||||
class FileDownloadCallsignDataProvider(CallsignDataProvider):
|
||||
"""Generic callsign data provider class for providers that fetch their data from the web by downloading a file."""
|
||||
|
||||
def __init__(self, name, provider_config, url, cache_file_path, poll_interval, storage):
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
provider_config: dict[str, Any],
|
||||
url: str,
|
||||
cache_file_path: str,
|
||||
poll_interval: int,
|
||||
storage: diskcache.Cache,
|
||||
) -> None:
|
||||
"""Set up the provider, note poll_interval is in *days*."""
|
||||
super().__init__(name, provider_config, storage)
|
||||
self._url = url
|
||||
self._cache_file_path = cache_file_path
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event = Event()
|
||||
self._url_data_cache = URLDataCache(f"callsigndata_{name}")
|
||||
|
||||
if self.enabled:
|
||||
self.status = "Ready"
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
|
||||
# subsequent polls, so start() returns immediately and the application can continue starting.
|
||||
logger.info(f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
|
||||
self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} callsign data worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
# Request the file. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
|
||||
# caching, but just so continual restarts of the software during testing don't hammer the servers.
|
||||
@@ -87,7 +99,7 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
|
||||
logger.exception(f"Exception in callsign reference data provider ({self.name})")
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _handle_file(self, path):
|
||||
def _handle_file(self, path: str) -> bool:
|
||||
"""Handle an updated file downloaded from the server. Return true if successful, false otherwise."""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import xmltodict
|
||||
@@ -14,6 +17,7 @@ from core.data_store import CACHE_DIR, DATA_STORE
|
||||
from core.enums import Continent
|
||||
from core.url_data_cache import URLDataCache
|
||||
from data.callsign import Callsign, LocationSourceForCallsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
from providers.callsigndata.api_query_callsign_data_provider import (
|
||||
APIQueryCallsignDataProvider,
|
||||
)
|
||||
@@ -24,7 +28,7 @@ logger = logging.getLogger(__name__)
|
||||
class HamQTH(APIQueryCallsignDataProvider):
|
||||
"""Callsign data provider for HamQTH."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("HamQTH", provider_config, DATA_STORE.callsign_data_hamqth)
|
||||
self._HAMQTH_BASE_URL = "https://www.hamqth.com/xml.php"
|
||||
self._PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}".replace(" ", "_")
|
||||
@@ -33,7 +37,7 @@ class HamQTH(APIQueryCallsignDataProvider):
|
||||
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
|
||||
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/hamqth-creds", expire_after=timedelta(minutes=55))
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
# If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because
|
||||
# # someone might provide credentials next time around.
|
||||
if not lookup_credentials or not (
|
||||
@@ -117,7 +121,7 @@ class HamQTH(APIQueryCallsignDataProvider):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def hamqth_response_to_callsign(callsign, data):
|
||||
def hamqth_response_to_callsign(callsign: str, data: dict[str, Any]) -> Callsign:
|
||||
"""Convert the "Callsign" block in HamQTH's API response to our own Callsign object."""
|
||||
|
||||
# Check for sensible latitudes
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import xmltodict
|
||||
@@ -13,6 +16,7 @@ from core.data_store import CACHE_DIR, DATA_STORE
|
||||
from core.enums import Continent, LocationSourceForCallsign
|
||||
from core.url_data_cache import URLDataCache
|
||||
from data.callsign import Callsign
|
||||
from data.lookup_credentials import LookupCredentials
|
||||
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -21,7 +25,7 @@ logger = logging.getLogger(__name__)
|
||||
class QRZ(APIQueryCallsignDataProvider):
|
||||
"""Callsign data provider for QRZ.com."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("QRZ.com", provider_config, DATA_STORE.callsign_data_qrz)
|
||||
self._QRZ_BASE_URL = "https://xmldata.qrz.com/xml/current/"
|
||||
self._URL_DATA_CACHE = URLDataCache("qrz")
|
||||
@@ -29,7 +33,7 @@ class QRZ(APIQueryCallsignDataProvider):
|
||||
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
|
||||
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/qrz-creds", expire_after=timedelta(minutes=55))
|
||||
|
||||
def _perform_new_lookup(self, callsign, lookup_credentials):
|
||||
def _perform_new_lookup(self, callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign | None:
|
||||
# If we don't have QRZ credentials, skip this lookup. Return None so we don't *cache* the lack of data, because
|
||||
# someone might provide credentials next time around.
|
||||
if not lookup_credentials or not (
|
||||
@@ -125,7 +129,7 @@ class QRZ(APIQueryCallsignDataProvider):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def qrz_response_to_callsign(callsign, data):
|
||||
def qrz_response_to_callsign(callsign: str, data: dict[str, Any] | list[Any]) -> Callsign:
|
||||
"""Convert the "Callsign" block in QRZ's API response to our own Callsign object."""
|
||||
|
||||
# I have encountered a user passing multiple callsigns to the QRZ lookup function in a way that QRZ actually
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -31,17 +34,17 @@ class GIROIonosonde(SolarConditionsProvider):
|
||||
Designed to run alongside KC2GProp even though they produce similar data. GIRO has more stations and includes LUF
|
||||
data, but is less reliable and often offline."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("GIRO Ionosonde Data", provider_config)
|
||||
self._stations = self._load_stations()
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._stations: list[dict[str, str]] = self._load_stations()
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
# Pre-populate ionosonde_data with known station names for stations not already present,
|
||||
# so the station dropdown is available before the first poll. Does not overwrite existing
|
||||
# entries so KC2G cache data is preserved.
|
||||
existing = self._solar_conditions.ionosonde_data or {}
|
||||
new_entries = {
|
||||
existing: dict[str, Any] = self._solar_conditions.ionosonde_data or {}
|
||||
new_entries: dict[str, Any] = {
|
||||
s["ursi"]: {
|
||||
"ursi": s["ursi"],
|
||||
"name": s["name"],
|
||||
@@ -57,27 +60,27 @@ class GIROIonosonde(SolarConditionsProvider):
|
||||
self.update_data({"ionosonde_data": {**existing, **new_entries}})
|
||||
|
||||
@staticmethod
|
||||
def _load_stations():
|
||||
stations = []
|
||||
def _load_stations() -> list[dict[str, str]]:
|
||||
stations: list[dict[str, str]] = []
|
||||
with open(STATIONS_INDEX, newline="") as f:
|
||||
for row in csv.reader(f):
|
||||
if len(row) >= 2:
|
||||
stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
|
||||
return stations
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
|
||||
self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning("GIRO ionosonde worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
# Real interval at which we poll is the "once per hour" divided by the number of stations, so each one gets
|
||||
# polled once per hour, just not all at once
|
||||
interval = POLL_INTERVAL / len(self._stations)
|
||||
@@ -88,7 +91,7 @@ class GIROIonosonde(SolarConditionsProvider):
|
||||
if self._stop_event.wait(timeout=interval):
|
||||
break
|
||||
|
||||
def _poll_station(self, station):
|
||||
def _poll_station(self, station: dict[str, str]) -> None:
|
||||
ursi = station["ursi"]
|
||||
name = station["name"]
|
||||
try:
|
||||
@@ -139,7 +142,9 @@ class GIROIonosonde(SolarConditionsProvider):
|
||||
self.status = "Error"
|
||||
logger.exception(f"Exception fetching GIRO ionosonde data for {ursi} ({name})")
|
||||
|
||||
def _fetch_station_data(self, ursi, from_time, to_time):
|
||||
def _fetch_station_data(
|
||||
self, ursi: str, from_time: datetime, to_time: datetime
|
||||
) -> tuple[dict[float, float] | None, dict[float, float] | None, dict[float, float] | None]:
|
||||
"""Fetch foF2, MUF and LUF readings for a station. Returns (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""
|
||||
|
||||
from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S")
|
||||
@@ -159,12 +164,12 @@ class GIROIonosonde(SolarConditionsProvider):
|
||||
return None, None, None
|
||||
|
||||
@staticmethod
|
||||
def _parse_all(text):
|
||||
def _parse_all(text: str) -> tuple[dict[float, float], dict[float, float], dict[float, float]]:
|
||||
"""Parse web server response and return (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""
|
||||
|
||||
fof2_data = {}
|
||||
muf_data = {}
|
||||
luf_data = {}
|
||||
fof2_data: dict[float, float] = {}
|
||||
muf_data: dict[float, float] = {}
|
||||
luf_data: dict[float, float] = {}
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from xml.etree import ElementTree
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from dateutil import parser as dateutil_parser
|
||||
from dateutil import tz as dateutil_tz
|
||||
|
||||
@@ -19,10 +23,10 @@ class HamQSL(HTTPSolarConditionsProvider):
|
||||
"""Solar conditions provider using the HamQSL.com XML API (https://www.hamqsl.com/solarxml.php).
|
||||
Provides solar flux index, geomagnetic indices, and HF/VHF propagation condition summaries."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("HamQSL", provider_config, URL, POLL_INTERVAL)
|
||||
|
||||
def _http_response_to_solar_conditions(self, http_response):
|
||||
def _http_response_to_solar_conditions(self, http_response: requests.Response) -> dict[str, Any] | None:
|
||||
root = ElementTree.fromstring(http_response.text)
|
||||
sd = root.find("solardata")
|
||||
if sd is None:
|
||||
@@ -31,27 +35,27 @@ class HamQSL(HTTPSolarConditionsProvider):
|
||||
|
||||
# Some error checking functions in case the data is janky.
|
||||
|
||||
def text(tag, default=None):
|
||||
def text(tag: str, default: str | None = None) -> str | None:
|
||||
if sd is None:
|
||||
logger.warning("HamQSL solar conditions API returned unexpected XML structure")
|
||||
return default
|
||||
el = sd.find(tag)
|
||||
return el.text.strip() if el is not None and el.text else default
|
||||
|
||||
def float_val(tag, default=None):
|
||||
def float_val(tag: str, default: float | None = None) -> float | None:
|
||||
try:
|
||||
return float(text(tag))
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
def int_val(tag, default=None):
|
||||
def int_val(tag: str, default: int | None = None) -> int | None:
|
||||
try:
|
||||
return int(text(tag))
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
# Process HF band conditions
|
||||
hf_conditions = {}
|
||||
hf_conditions: dict[str, str] = {}
|
||||
calc = sd.find("calculatedconditions")
|
||||
if calc is not None:
|
||||
for band_el in calc.findall("band"):
|
||||
@@ -62,7 +66,7 @@ class HamQSL(HTTPSolarConditionsProvider):
|
||||
hf_conditions[f"{name}-{time}"] = condition
|
||||
|
||||
# Process VHF propagation conditions
|
||||
vhf_map = {}
|
||||
vhf_map: dict[tuple[str | None, str | None], str | None] = {}
|
||||
vhf = sd.find("calculatedvhfconditions")
|
||||
if vhf is not None:
|
||||
for ph_el in vhf.findall("phenomenon"):
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -16,32 +19,32 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
|
||||
"""Generic solar conditions provider for providers that request data via HTTP(S). Subclasses implement
|
||||
_http_response_to_solar_conditions() to parse the specific API response format."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
super().__init__(name, provider_config)
|
||||
self._url = url
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._url: str = url
|
||||
self._poll_interval: float = poll_interval
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.info(f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
|
||||
self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} solar conditions worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=self._poll_interval):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
logger.debug(f"Polling {self.name} solar conditions API...")
|
||||
http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
|
||||
@@ -66,7 +69,7 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
|
||||
logger.exception(f"Exception in HTTP Solar Conditions Provider ({self.name})")
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _http_response_to_solar_conditions(self, http_response):
|
||||
def _http_response_to_solar_conditions(self, http_response: requests.Response) -> dict[str, Any] | None:
|
||||
"""Convert an HTTP response into solar conditions data. Returns a dict mapping SolarConditions field
|
||||
names to their new values, or None if the response could not be parsed. Only the fields returned will
|
||||
be updated on the shared SolarConditions object; any fields not included will be left unchanged."""
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from core.constants import BANDS
|
||||
from data.band import Band
|
||||
|
||||
HF_BANDS = [b for b in BANDS if b.is_ham_hf]
|
||||
HF_BANDS: list[Band] = [b for b in BANDS if b.is_ham_hf]
|
||||
|
||||
|
||||
def _latest(d) -> float | None:
|
||||
def _latest(d: dict[float, float | str] | None) -> float | None:
|
||||
"""Given a map where the key is a timestamp and the value is a number represented as a string, find the latest
|
||||
timestamp and return the corresponding value as a float."""
|
||||
|
||||
@@ -11,7 +14,11 @@ def _latest(d) -> float | None:
|
||||
return float(val) if (val is not None and val != "None") else None
|
||||
|
||||
|
||||
def compute_band_states(fof2_dict, muf_dict, luf_dict):
|
||||
def compute_band_states(
|
||||
fof2_dict: dict[float, float | str] | None,
|
||||
muf_dict: dict[float, float | str] | None,
|
||||
luf_dict: dict[float, float | str] | None,
|
||||
) -> dict[str, str]:
|
||||
"""Compute HF band states from the latest foF2, MUF and LUF values.
|
||||
|
||||
Returns a map where the keys are HF bands and the values are as follows:
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -25,30 +28,30 @@ class KC2GProp(SolarConditionsProvider):
|
||||
Designed to run alongside GIROIonosonde even though they produce similar data. KC2G is more reliable and is always
|
||||
online, but has fewer stations and does not provide LUF data."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("KC2G Propagation Data", provider_config)
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
|
||||
self._thread = Thread(target=self._run, name="KC2GPropProvider", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning("KC2G ionosonde worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=POLL_INTERVAL):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
logger.debug("Polling KC2G ionosonde data...")
|
||||
http_response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
|
||||
@@ -61,7 +64,7 @@ class KC2GProp(SolarConditionsProvider):
|
||||
|
||||
# Start from existing ionosonde_data so the accumulated time series survives across polls and restarts and
|
||||
# stations provided only by GIROIonosonde are not discarded
|
||||
ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
|
||||
ionosonde_data: dict[str, Any] = dict(self._solar_conditions.ionosonde_data or {})
|
||||
updated_count = 0
|
||||
|
||||
for reading in http_response.json():
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from providers.solarconditions.http_solar_conditions_provider import (
|
||||
HTTPSolarConditionsProvider,
|
||||
@@ -16,11 +21,11 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
"""Solar conditions provider using the NOAA 3-day forecast text file. Parses the NOAA forecast and populates
|
||||
corresponding fields in the solar conditions object.."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("NOAA 3-day Forecast", provider_config, URL, POLL_INTERVAL)
|
||||
|
||||
@staticmethod
|
||||
def _parse_percentage_table(lines, section_header, year):
|
||||
def _parse_percentage_table(lines: list[str], section_header: str, year: int) -> dict[str, dict[float, int]] | None:
|
||||
"""Find and parse a forecast table using percentages, identified by section_header. This is common to the lookup
|
||||
of the solar storm and radio blackout forecast parsing."""
|
||||
|
||||
@@ -48,7 +53,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
return None
|
||||
|
||||
# Figure out the date based on the line found
|
||||
column_timestamps = []
|
||||
column_timestamps: list[float] = []
|
||||
for month_str, day_str in date_matches:
|
||||
try:
|
||||
dt = datetime.strptime(f"{day_str} {month_str} {year}", "%d %b %Y").replace(tzinfo=timezone.utc)
|
||||
@@ -58,7 +63,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
return None
|
||||
|
||||
# Parse data rows. Each non-empty line should have a text label followed by percentage values
|
||||
result = {}
|
||||
result: dict[str, dict[float, int]] = {}
|
||||
for line in lines[date_header_idx + 1 :]:
|
||||
line_stripped = line.strip()
|
||||
if not line_stripped:
|
||||
@@ -73,7 +78,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
|
||||
# Row label is everything before the first percentage value
|
||||
row_label = line_stripped[: line_stripped.index(pct_matches[0].group())].strip()
|
||||
row_data = {}
|
||||
row_data: dict[float, int] = {}
|
||||
for j, match in enumerate(pct_matches):
|
||||
if j >= len(column_timestamps):
|
||||
break
|
||||
@@ -83,7 +88,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
|
||||
return result if result else None
|
||||
|
||||
def _http_response_to_solar_conditions(self, http_response):
|
||||
def _http_response_to_solar_conditions(self, http_response: requests.Response) -> dict[str, Any] | None:
|
||||
lines = http_response.text.splitlines()
|
||||
|
||||
# Find the "NOAA Kp index breakdown" section header
|
||||
@@ -115,7 +120,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
logger.warning(f"NOAA K-index forecast: could not parse date headers from: {date_header_line}")
|
||||
return None
|
||||
|
||||
column_dates = []
|
||||
column_dates: list[date] = []
|
||||
for month_str, day_str in date_matches:
|
||||
try:
|
||||
column_dates.append(
|
||||
@@ -126,7 +131,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
return None
|
||||
|
||||
# Parse each data row, e.g. "00-03UT 2.00 3.00 2.00"
|
||||
k_index_forecast = {}
|
||||
k_index_forecast: dict[float, float] = {}
|
||||
for line in lines[start_idx + 3 :]:
|
||||
time_match = re.match(r"^(\d{2})-(\d{2})UT\s+(.*)", line.strip())
|
||||
if not time_match:
|
||||
@@ -167,14 +172,14 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
|
||||
return None
|
||||
|
||||
# Parse Solar Radiation Storm Forecast (single row: "S1 or greater")
|
||||
solar_storm_forecast = None
|
||||
solar_storm_forecast: dict[float, int] | None = None
|
||||
radiation_table = self._parse_percentage_table(lines, "Solar Radiation Storm Forecast", year)
|
||||
if radiation_table:
|
||||
solar_storm_forecast = radiation_table.get("S1 or greater")
|
||||
|
||||
# Parse Radio Blackout Forecast (two rows: "R1-R2" and "R3 or greater")
|
||||
blackout_forecast_r1r2 = None
|
||||
blackout_forecast_r3_or_greater = None
|
||||
blackout_forecast_r1r2: dict[float, int] | None = None
|
||||
blackout_forecast_r3_or_greater: dict[float, int] | None = None
|
||||
blackout_table = self._parse_percentage_table(lines, "Radio Blackout Forecast", year)
|
||||
if blackout_table:
|
||||
blackout_forecast_r1r2 = blackout_table.get("R1-R2")
|
||||
|
||||
@@ -1,34 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from data.solar_conditions import SolarConditions
|
||||
|
||||
|
||||
class SolarConditionsProvider:
|
||||
"""Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
|
||||
propagation data."""
|
||||
|
||||
def __init__(self, name, provider_config):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor"""
|
||||
|
||||
self.name = name
|
||||
self.enabled = provider_config.get("enabled", True)
|
||||
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Not Started" if self.enabled else "Disabled"
|
||||
self._solar_conditions = DATA_STORE.solar_conditions.get()
|
||||
self.name: str = name
|
||||
self.enabled: bool = provider_config.get("enabled", True)
|
||||
self.last_update_time: datetime = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status: str = "Not Started" if self.enabled else "Disabled"
|
||||
self._solar_conditions: SolarConditions = DATA_STORE.solar_conditions.get()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def update_data(self, new_data):
|
||||
def update_data(self, new_data: dict[str, Any] | None) -> None:
|
||||
"""Update the solar conditions object with new data"""
|
||||
|
||||
if new_data:
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import aprslib
|
||||
import pytz
|
||||
@@ -15,17 +18,17 @@ logger = logging.getLogger(__name__)
|
||||
class APRSIS(SpotProvider):
|
||||
"""Spot provider for the APRS-IS."""
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("APRS-IS", provider_config)
|
||||
self._thread = None
|
||||
self._aprsis = None
|
||||
self._stop_event = Event()
|
||||
self._thread: Thread | None = None
|
||||
self._aprsis: aprslib.IS | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
self._thread = Thread(target=self._run, name="APRSISSpotProvider", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self._aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN)
|
||||
@@ -43,7 +46,7 @@ class APRSIS(SpotProvider):
|
||||
if not self._stop_event.is_set():
|
||||
self._stop_event.wait(timeout=5)
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self.status = "Shutting down"
|
||||
self._stop_event.set()
|
||||
if self._aprsis:
|
||||
@@ -53,7 +56,7 @@ class APRSIS(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning("APRS-IS worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _handle(self, data):
|
||||
def _handle(self, data: dict[str, Any]) -> None:
|
||||
try:
|
||||
# Split SSID in "from" call and store separately
|
||||
from_parts = str(data["from"]).split("-")
|
||||
|
||||
+18
-15
@@ -1,16 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
from datetime import datetime
|
||||
from threading import Event, Lock, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import telnetlib3
|
||||
|
||||
from core.config import SERVER_OWNER_CALLSIGN
|
||||
from core.utils import decode_telnet_bytes
|
||||
from data.spot import Spot
|
||||
from providers.spot.spot_provider import SpotProvider
|
||||
from core.utils import decode_telnet_bytes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,29 +31,29 @@ class DXCluster(SpotProvider):
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor requires hostname and port"""
|
||||
|
||||
name = provider_config.get("name", "Cluster")
|
||||
super().__init__(name, provider_config)
|
||||
self._hostname = provider_config["host"]
|
||||
self._port = provider_config["port"]
|
||||
self._login_prompt = provider_config.get("login_prompt", "login:")
|
||||
self._login_callsign = provider_config.get("login_callsign", SERVER_OWNER_CALLSIGN)
|
||||
self._allow_rbn_spots = provider_config.get("allow_rbn_spots", False)
|
||||
self._spot_line_pattern = (
|
||||
self._hostname: str = provider_config["host"]
|
||||
self._port: int = provider_config["port"]
|
||||
self._login_prompt: str = provider_config.get("login_prompt", "login:")
|
||||
self._login_callsign: str = provider_config.get("login_callsign", SERVER_OWNER_CALLSIGN)
|
||||
self._allow_rbn_spots: bool = provider_config.get("allow_rbn_spots", False)
|
||||
self._spot_line_pattern: re.Pattern[str] = (
|
||||
self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
|
||||
)
|
||||
self._telnet = None
|
||||
self._telnet_lock = Lock()
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._telnet: telnetlib3.Telnet | None = None
|
||||
self._telnet_lock: Lock = Lock()
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
with self._telnet_lock:
|
||||
if self._telnet:
|
||||
@@ -64,7 +67,7 @@ class DXCluster(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"DX Cluster {self._hostname} worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _handle(self):
|
||||
def _handle(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
connected = False
|
||||
while not connected and not self._stop_event.is_set():
|
||||
|
||||
+11
-7
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from core.enums import ActivityName, ActivityRefType, Mode
|
||||
@@ -21,14 +25,14 @@ class GMA(HTTPSpotProvider):
|
||||
# GMA spots don't contain the details of the programme they are for, we need a separate lookup for that
|
||||
REF_INFO_URL_ROOT = "https://www.gma.rocks/api/ref/?"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
# Ensure there is an API key in our config, and set up the query URL using it. If no key is provided,
|
||||
# disable this spot provider.
|
||||
self._api_key = provider_config.get("api_key", "")
|
||||
self._api_key: str = provider_config.get("api_key", "")
|
||||
if self._api_key == "":
|
||||
provider_config["enabled"] = False
|
||||
logger.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
|
||||
self._url_data_cache = URLDataCache("GMA")
|
||||
self._url_data_cache: URLDataCache = URLDataCache("GMA")
|
||||
|
||||
super().__init__(
|
||||
"GMA",
|
||||
@@ -37,8 +41,8 @@ class GMA(HTTPSpotProvider):
|
||||
self.POLL_INTERVAL_SEC,
|
||||
)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
if "RCD" in http_response.json():
|
||||
for source_spot in http_response.json()["RCD"]:
|
||||
@@ -172,10 +176,10 @@ class GMA(HTTPSpotProvider):
|
||||
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.GMA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO: Implement.
|
||||
# Spotting to GMA is documented: https://www.cqgma.org/api/doc/apigma_spot.pdf We (or the user) need a GMA account, and to send the password in plaintext(!!)
|
||||
raise NotImplementedError("GMA upstream spot submission is not yet implemented")
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -27,17 +30,17 @@ class HEMA(HTTPSpotProvider):
|
||||
FREQ_MODE_PATTERN = re.compile("^([\\d.]*) \\((.*)\\)$")
|
||||
SPOTTER_COMMENT_PATTERN = re.compile("^\\((.*)\\) (.*)$")
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("HEMA", provider_config, self.SPOT_SEED_URL, self.POLL_INTERVAL_SEC)
|
||||
self._spot_seed = ""
|
||||
self._spot_seed: str = ""
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
# OK, source data is actually just the spot seed at this point. We'll then go on to fetch real data if we know
|
||||
# this has changed.
|
||||
spot_seed_changed = http_response.text != self._spot_seed
|
||||
self._spot_seed = http_response.text
|
||||
|
||||
new_spots = []
|
||||
new_spots: list[Spot] = []
|
||||
# OK, if the spot seed actually changed, now we make the real request for data.
|
||||
if spot_seed_changed:
|
||||
try:
|
||||
@@ -89,10 +92,10 @@ class HEMA(HTTPSpotProvider):
|
||||
logger.warning("Connection error when accessing HEMA spots API.")
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.HEMA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO: Implement. Currently blocked awaiting their API team to make a change to allow us to spot with a
|
||||
# reference and not a reference *number*.
|
||||
raise NotImplementedError("HEMA upstream spot submission is not yet implemented")
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, JSONDecodeError, ReadTimeout
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from data.spot import Spot
|
||||
from providers.spot.spot_provider import SpotProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -16,22 +20,22 @@ class HTTPSpotProvider(SpotProvider):
|
||||
"""Generic spot provider class for providers that request data via HTTP(S). Just for convenience to avoid code
|
||||
duplication. Subclasses of this query the individual APIs for data."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
super().__init__(name, provider_config)
|
||||
self._url = url
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._wakeup_event = Event()
|
||||
self._url: str = url
|
||||
self._poll_interval: float = poll_interval
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
self._wakeup_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
|
||||
# subsequent polls, so start() returns immediately and the application can continue starting.
|
||||
logger.info(f"Set up query of {self.name} spot API every {self._poll_interval!s} seconds.")
|
||||
self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
self._wakeup_event.set()
|
||||
if self._thread:
|
||||
@@ -39,12 +43,12 @@ class HTTPSpotProvider(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} spot worker thread did not exit on time and will be killed.")
|
||||
|
||||
def force_poll(self):
|
||||
def force_poll(self) -> None:
|
||||
"""Trigger an immediate poll without waiting for the normal interval."""
|
||||
|
||||
self._wakeup_event.set()
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._wakeup_event.clear()
|
||||
self._poll()
|
||||
@@ -52,7 +56,7 @@ class HTTPSpotProvider(SpotProvider):
|
||||
if self._stop_event.is_set():
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
# Request data from API
|
||||
logger.debug(f"Polling {self.name} spot API...")
|
||||
@@ -86,7 +90,7 @@ class HTTPSpotProvider(SpotProvider):
|
||||
logger.exception(f"Exception in HTTP Spot Provider ({self.name})")
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot] | None:
|
||||
"""Convert an HTTP response returned by the API into spot data. The whole response is provided here so the subclass
|
||||
implementations can check for HTTP status codes if necessary, and handle the response as JSON, XML, text, whatever
|
||||
the API actually provides."""
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType, Mode
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -12,11 +17,11 @@ class LLOTA(HTTPSpotProvider):
|
||||
POLL_INTERVAL_SEC = 120
|
||||
SPOTS_URL = "https://llota.app/api/public/spots"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("LLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
for source_spot in http_response.json():
|
||||
# Find the most recent spotter and comment from the history array
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import ClassVar
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -33,11 +35,11 @@ class ParksNPeaks(HTTPSpotProvider):
|
||||
ActivityName.SANPCPA,
|
||||
]
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
if http_response and http_response != "":
|
||||
for source_spot in http_response.json():
|
||||
@@ -117,10 +119,10 @@ class ParksNPeaks(HTTPSpotProvider):
|
||||
new_spots.append(spot)
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity in self.SUBMITTABLE_ACTIVITIES
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO test this works
|
||||
user_id = credentials.get("user_id", "")
|
||||
api_key = credentials.get("api_key", "")
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
@@ -17,11 +20,11 @@ class POTA(HTTPSpotProvider):
|
||||
SPOTS_URL = "https://api.pota.app/spot/activator"
|
||||
SUBMIT_URL = "https://api.pota.app/spot"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("POTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
for source_spot in http_response.json():
|
||||
# Convert to our spot format
|
||||
@@ -57,10 +60,10 @@ class POTA(HTTPSpotProvider):
|
||||
new_spots.append(spot)
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.POTA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
sig_ref = spot.sig_refs[0].id if spot.sig_refs else None
|
||||
if sig_ref:
|
||||
body = {
|
||||
|
||||
+13
-10
@@ -1,16 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
from datetime import datetime
|
||||
from threading import Event, Lock, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import telnetlib3
|
||||
|
||||
from core.config import SERVER_OWNER_CALLSIGN
|
||||
from core.utils import decode_telnet_bytes
|
||||
from data.spot import Spot
|
||||
from providers.spot.spot_provider import SpotProvider
|
||||
from core.utils import decode_telnet_bytes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -24,22 +27,22 @@ class RBN(SpotProvider):
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor requires port number."""
|
||||
|
||||
name = provider_config.get("name", "RBN")
|
||||
super().__init__(name, provider_config)
|
||||
self._port = provider_config["port"]
|
||||
self._telnet = None
|
||||
self._telnet_lock = Lock()
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._port: int = provider_config["port"]
|
||||
self._telnet: telnetlib3.Telnet | None = None
|
||||
self._telnet_lock: Lock = Lock()
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
self._thread = Thread(target=self._handle, name=f"RBNSpotProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
with self._telnet_lock:
|
||||
if self._telnet:
|
||||
@@ -53,7 +56,7 @@ class RBN(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"RBN (port {self._port!s}) worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _handle(self):
|
||||
def _handle(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
connected = False
|
||||
while not connected and not self._stop_event.is_set():
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import ClassVar
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
|
||||
@@ -27,17 +29,17 @@ class SOTA(HTTPSpotProvider):
|
||||
SUBMIT_URL = "https://api-db2.sota.org.uk/api/spots"
|
||||
VALID_MODES: ClassVar[list[str]] = ["AM", "CW", "Data", "DV", "FM", "SSB"]
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("SOTA", provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)
|
||||
self._api_epoch = ""
|
||||
self._api_epoch: str = ""
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
# OK, source data is actually just the epoch at this point. We'll then go on to fetch real data if we know this
|
||||
# has changed.
|
||||
epoch_changed = http_response.text != self._api_epoch
|
||||
self._api_epoch = http_response.text
|
||||
|
||||
new_spots = []
|
||||
new_spots: list[Spot] = []
|
||||
# OK, if the epoch actually changed, now we make the real request for data.
|
||||
if epoch_changed:
|
||||
try:
|
||||
@@ -83,10 +85,10 @@ class SOTA(HTTPSpotProvider):
|
||||
logger.warning("Timeout when accessing SOTA spots API.")
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.SOTA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO test this method works
|
||||
access_token = credentials.get("access_token", "")
|
||||
id_token = credentials.get("id_token", "")
|
||||
|
||||
@@ -1,30 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from core.live_data_cache import LiveDataCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Deferred to avoid a circular import: data.spot imports core.call_lookup_helper, which imports
|
||||
# core.data_providers, which imports this module.
|
||||
from data.spot import Spot
|
||||
|
||||
|
||||
class SpotProvider:
|
||||
"""Generic spot provider class. Subclasses of this query the individual APIs for data."""
|
||||
|
||||
def __init__(self, name, provider_config):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor"""
|
||||
|
||||
self.name = name
|
||||
self.enabled = provider_config.get("enabled", True)
|
||||
self.enabled_by_default_in_web_ui = provider_config.get("enabled_by_default_in_web_ui", True)
|
||||
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Not Started" if self.enabled else "Disabled"
|
||||
self._spots = DATA_STORE.spots
|
||||
self.name: str = name
|
||||
self.enabled: bool = provider_config.get("enabled", True)
|
||||
self.enabled_by_default_in_web_ui: bool = provider_config.get("enabled_by_default_in_web_ui", True)
|
||||
self.last_update_time: datetime = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.last_spot_time: datetime = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status: str = "Not Started" if self.enabled else "Disabled"
|
||||
self._spots: LiveDataCache[Spot] = DATA_STORE.spots
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def _submit_batch(self, spots):
|
||||
def _submit_batch(self, spots: list[Spot]) -> None:
|
||||
"""Submit a batch of spots retrieved from the provider. Only spots that are newer than the last spot retrieved
|
||||
by this provider will be added to the spot list, to prevent duplications. Spots passing the check will also have
|
||||
their infer_missing() method called to complete their data set. This is called by the API-querying
|
||||
@@ -41,7 +50,7 @@ class SpotProvider:
|
||||
if spots:
|
||||
self.last_spot_time = datetime.fromtimestamp(max(s.time for s in spots), pytz.UTC)
|
||||
|
||||
def _submit(self, spot):
|
||||
def _submit(self, spot: Spot) -> None:
|
||||
"""Submit a single spot retrieved from the provider. This will be added to the list regardless of its age. Spots
|
||||
passing the check will also have their infer_missing() method called to complete their data set. This is called by
|
||||
the data streaming subclasses, which can be relied upon not to re-provide old spots."""
|
||||
@@ -51,27 +60,27 @@ class SpotProvider:
|
||||
self._add_spot(spot)
|
||||
self.last_spot_time = datetime.fromtimestamp(spot.time, pytz.UTC)
|
||||
|
||||
def _add_spot(self, spot):
|
||||
def _add_spot(self, spot: Spot) -> None:
|
||||
if not spot.expired():
|
||||
self._spots.set(spot.id, spot)
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
"""Return True if this provider supports submitting spots upstream for the given activity."""
|
||||
|
||||
return False
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
"""Submit a spot upstream to this provider's API. credentials is a dict with provider-specific keys.
|
||||
Raises an exception with a descriptive message on failure."""
|
||||
|
||||
raise NotImplementedError("This provider does not support spot submission")
|
||||
|
||||
def force_poll(self):
|
||||
def force_poll(self) -> None:
|
||||
"""Trigger an immediate poll without waiting for the normal interval. Default implementation here does nothing
|
||||
because not all spot providers have a polling mechanism. Providers that do should override this method."""
|
||||
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Lock, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
from requests_sse import EventSource, InvalidStatusCodeError
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from data.spot import Spot
|
||||
from providers.spot.spot_provider import SpotProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -14,23 +18,23 @@ logger = logging.getLogger(__name__)
|
||||
class SSESpotProvider(SpotProvider):
|
||||
"""Spot provider using Server-Sent Events."""
|
||||
|
||||
def __init__(self, name, provider_config, url):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str) -> None:
|
||||
super().__init__(name, provider_config)
|
||||
self._url = url
|
||||
self._thread = None
|
||||
self._last_event_id = None
|
||||
self._stop_event = Event()
|
||||
self._event_source_lock = Lock()
|
||||
self._event_source = None
|
||||
self._url: str = url
|
||||
self._thread: Thread | None = None
|
||||
self._last_event_id: str | None = None
|
||||
self._stop_event: Event = Event()
|
||||
self._event_source_lock: Lock = Lock()
|
||||
self._event_source: EventSource | None = None
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.info(f"Set up SSE connection to {self.name} spot API.")
|
||||
self._stop_event.clear()
|
||||
self._thread = Thread(target=self._run, name=f"SSESpotProvider-{self.name}")
|
||||
self._thread.daemon = True
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
|
||||
with self._event_source_lock:
|
||||
@@ -46,17 +50,17 @@ class SSESpotProvider(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} SSE worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _on_open(self):
|
||||
def _on_open(self) -> None:
|
||||
self.status = "Waiting for Data"
|
||||
|
||||
def _on_error(self):
|
||||
def _on_error(self) -> None:
|
||||
self.status = "Connecting"
|
||||
|
||||
def _set_event_source(self, event_source):
|
||||
def _set_event_source(self, event_source: EventSource | None) -> None:
|
||||
with self._event_source_lock:
|
||||
self._event_source = event_source
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
logger.debug(f"Connecting to {self.name} spot API...")
|
||||
@@ -102,7 +106,7 @@ class SSESpotProvider(SpotProvider):
|
||||
self.status = "Disconnected"
|
||||
self._stop_event.wait(timeout=5) # Wait before trying to reconnect
|
||||
|
||||
def _sse_message_to_spot(self, message_data):
|
||||
def _sse_message_to_spot(self, message_data: str) -> Spot | None:
|
||||
"""Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass
|
||||
implementations can handle the message as JSON, XML, text, whatever the API actually provides."""
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import ClassVar
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import requests
|
||||
|
||||
@@ -36,11 +38,11 @@ class Tiles(HTTPSpotProvider):
|
||||
"Other",
|
||||
]
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
for source_spot in http_response.json()["spots"]:
|
||||
# Convert to our spot format
|
||||
@@ -84,10 +86,10 @@ class Tiles(HTTPSpotProvider):
|
||||
new_spots.append(spot)
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.TILES
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# Tiles on the air currently only supports *self* spots
|
||||
if spot.dx_call == spot.de_call:
|
||||
# Figure out a valid mode. Borrowed this from PoLo :)
|
||||
@@ -127,7 +129,7 @@ class Tiles(HTTPSpotProvider):
|
||||
|
||||
# Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles'
|
||||
# strange frequency format where we can sometimes have e.g. "14.123.5".
|
||||
def strip_extra_decimal_points(s):
|
||||
def strip_extra_decimal_points(s: str) -> str:
|
||||
parts = s.split(".", 1)
|
||||
if len(parts) == 1:
|
||||
return s
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class Towers(HTTPSpotProvider):
|
||||
POLL_INTERVAL_SEC = 120
|
||||
SPOTS_URL = "https://wwtota.com/api/cluster_live.php"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("Towers", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
response_fixed = http_response.text.replace("\\/", "/")
|
||||
response_json = json.loads(response_fixed)
|
||||
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import Mode
|
||||
from data.spot import Spot
|
||||
@@ -14,11 +18,11 @@ class UKPacketNet(HTTPSpotProvider):
|
||||
POLL_INTERVAL_SEC = 600
|
||||
SPOTS_URL = "https://nodes.ukpacketradio.network/api/nodedata"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("UK Packet Net", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
nodes = http_response.json()["nodes"]
|
||||
for node in nodes.values():
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
from websocket import create_connection
|
||||
from websocket import WebSocket, create_connection
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from data.spot import Spot
|
||||
from providers.spot.spot_provider import SpotProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -14,22 +18,22 @@ logger = logging.getLogger(__name__)
|
||||
class WebsocketSpotProvider(SpotProvider):
|
||||
"""Spot provider using websockets."""
|
||||
|
||||
def __init__(self, name, provider_config, url):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str) -> None:
|
||||
super().__init__(name, provider_config)
|
||||
self._url = url
|
||||
self._ws = None
|
||||
self._thread = None
|
||||
self._stop_event = Event()
|
||||
self._last_event_id = None
|
||||
self._url: str = url
|
||||
self._ws: WebSocket | None = None
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event: Event = Event()
|
||||
self._last_event_id: str | None = None
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.info(f"Set up websocket connection to {self.name} spot API.")
|
||||
self._stop_event.clear()
|
||||
self._thread = Thread(target=self._run, name=f"WebsocketSpotProvider-{self.name}")
|
||||
self._thread.daemon = True
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._ws:
|
||||
self._ws.close()
|
||||
@@ -38,13 +42,13 @@ class WebsocketSpotProvider(SpotProvider):
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} websocket worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _on_open(self):
|
||||
def _on_open(self) -> None:
|
||||
self.status = "Waiting for Data"
|
||||
|
||||
def _on_error(self):
|
||||
def _on_error(self) -> None:
|
||||
self.status = "Connecting"
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
logger.debug(f"Connecting to {self.name} spot API...")
|
||||
@@ -86,7 +90,7 @@ class WebsocketSpotProvider(SpotProvider):
|
||||
if not self._stop_event.is_set():
|
||||
self._stop_event.wait(timeout=5) # Wait before trying to reconnect
|
||||
|
||||
def _ws_message_to_spot(self, b):
|
||||
def _ws_message_to_spot(self, b: str | bytes) -> Spot | None:
|
||||
"""Convert a WS message received from the API into a spot. The exact message data (in bytes) is provided here so the
|
||||
subclass implementations can handle the message as string, JSON, XML, whatever the API actually provides."""
|
||||
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import cast
|
||||
from typing import Any, cast
|
||||
from xml.parsers.expat import ExpatError
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from rss_parser import Parser
|
||||
from rss_parser.models.rss import RSS
|
||||
|
||||
@@ -23,11 +26,11 @@ class WOTA(HTTPSpotProvider):
|
||||
SPOTS_URL = "https://www.wota.org.uk/spots_rss.php"
|
||||
RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("WOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
try:
|
||||
rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
|
||||
# Iterate through source data
|
||||
@@ -110,9 +113,9 @@ class WOTA(HTTPSpotProvider):
|
||||
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.WOTA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO Ask M5TEA if he's happy to share how this is done from his app
|
||||
raise NotImplementedError("WOTA upstream spot submission is not yet implemented")
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType, Mode
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -12,14 +15,14 @@ class WWBOTA(SSESpotProvider):
|
||||
|
||||
SPOTS_URL = "https://api.wwbota.net/spots/"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("WWBOTA", provider_config, self.SPOTS_URL)
|
||||
|
||||
def _sse_message_to_spot(self, message_data):
|
||||
def _sse_message_to_spot(self, message_data: str) -> Spot | None:
|
||||
source_spot = json.loads(message_data)
|
||||
# Convert to our spot format. First we unpack references, because WWBOTA spots can have more than one for
|
||||
# n-fer activations.
|
||||
refs = []
|
||||
refs: list[ActivityRef] = []
|
||||
for ref in source_spot["references"]:
|
||||
activity_ref = ActivityRef(
|
||||
id=ref["reference"],
|
||||
@@ -52,9 +55,9 @@ class WWBOTA(SSESpotProvider):
|
||||
# WWBOTA does support a special "Test" spot type, we need to avoid adding that.
|
||||
return spot if source_spot["type"] != "Test" else None
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.WWBOTA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO: Implement. WWBOTA API docs cover this: https://api.wwbota.org/#tag/Spots/operation/create_spot_spots__post
|
||||
raise NotImplementedError("WWBOTA upstream spot submission is not yet implemented")
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType, Mode
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,11 +18,11 @@ class WWFF(HTTPSpotProvider):
|
||||
POLL_INTERVAL_SEC = 120
|
||||
SPOTS_URL = "https://spots.wwff.co/static/spots.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("WWFF", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
for source_spot in http_response.json():
|
||||
# Convert to our spot format
|
||||
@@ -51,10 +55,10 @@ class WWFF(HTTPSpotProvider):
|
||||
new_spots.append(spot)
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.WWFF
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO: Implement. Spotting to WWFF should be possible, need to look up the Spotline docs or copy approach from
|
||||
# PoLo. Either way I think we need an API key for the app (but maybe not for the user?)
|
||||
raise NotImplementedError("WWFF upstream spot submission is not yet implemented")
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
@@ -17,15 +20,17 @@ class XOTA(WebsocketSpotProvider):
|
||||
is why we also provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3
|
||||
might become "C3 T-01". This allows us to provide location lookups for TOTA at several conferences."""
|
||||
|
||||
ACTIVITY = None
|
||||
ACTIVITY: str | None = None
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
name = provider_config.get("name", "xOTA")
|
||||
super().__init__(name, provider_config, provider_config["url"])
|
||||
self.ACTIVITY = str(provider_config["sig"]) if "sig" in provider_config else None
|
||||
self._activity_ref_prefix = str(provider_config["sig_ref_prefix"]) if "sig_ref_prefix" in provider_config else ""
|
||||
self._activity_ref_prefix: str = (
|
||||
str(provider_config["sig_ref_prefix"]) if "sig_ref_prefix" in provider_config else ""
|
||||
)
|
||||
|
||||
def _ws_message_to_spot(self, b):
|
||||
def _ws_message_to_spot(self, b: str | bytes) -> Spot | None:
|
||||
string = b.decode("utf-8")
|
||||
source_spot = json.loads(string)
|
||||
ref_id = f"{self._activity_ref_prefix} {source_spot['reference']['title']}"
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, Mode
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,11 +18,11 @@ class ZLOTA(HTTPSpotProvider):
|
||||
POLL_INTERVAL_SEC = 120
|
||||
SPOTS_URL = "https://ontheair.nz/api/spots?zlota_only=true"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("ZLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
|
||||
|
||||
def _http_response_to_spots(self, http_response):
|
||||
new_spots = []
|
||||
def _http_response_to_spots(self, http_response: requests.Response) -> list[Spot]:
|
||||
new_spots: list[Spot] = []
|
||||
# Iterate through source data
|
||||
for source_spot in http_response.json():
|
||||
# Frequency is often inconsistent as to whether it's in Hz or kHz. Make a guess.
|
||||
@@ -51,9 +55,9 @@ class ZLOTA(HTTPSpotProvider):
|
||||
new_spots.append(spot)
|
||||
return new_spots
|
||||
|
||||
def can_submit_spot(self, activity):
|
||||
def can_submit_spot(self, activity: str) -> bool:
|
||||
return activity == ActivityName.ZLOTA
|
||||
|
||||
def submit_spot(self, spot, credentials):
|
||||
def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None:
|
||||
# TODO: Implement. Spotting to ZLOTA is supported via POST, see https://ontheair.nz/api
|
||||
raise NotImplementedError("ZLOTA upstream spot submission is not yet implemented")
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import geopandas
|
||||
from shapely import prepare
|
||||
@@ -17,10 +20,10 @@ class CQZoneData(LocalFileStaticDataProvider):
|
||||
|
||||
PATH = "datafiles/cqzones.geojson"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("CQ Zone Data", provider_config, self.PATH)
|
||||
|
||||
def _load_data(self, path):
|
||||
def _load_data(self, path: str) -> bool:
|
||||
try:
|
||||
with open(path) as f:
|
||||
cq_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
from requests import Response
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
@@ -16,36 +20,36 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
|
||||
"""Generic static reference data provider class for providers that fetch their data from the web by downloading a
|
||||
file."""
|
||||
|
||||
def __init__(self, name, provider_config, url, poll_interval):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
"""Set up the provider, note poll_interval is in *days*."""
|
||||
super().__init__(name, provider_config)
|
||||
self._url = url
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._thread: Thread | None = None
|
||||
self._stop_event = Event()
|
||||
self._url_data_cache = URLDataCache(f"staticdata_{name}")
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
|
||||
# subsequent polls, so start() returns immediately and the application can continue starting.
|
||||
logger.info(f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.")
|
||||
self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.name} static data worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
# Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
|
||||
# caching, but just so continual restarts of the software during testing don't hammer the servers.
|
||||
@@ -76,7 +80,7 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
|
||||
logger.exception(f"Exception in HTTP static reference data provider ({self.name})")
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _handle_http_response(self, http_response):
|
||||
def _handle_http_response(self, http_response: Response) -> bool:
|
||||
"""Handle an HTTP response returned by the server and load the data from it. Return true if successful,
|
||||
false otherwise."""
|
||||
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import geopandas
|
||||
from shapely import prepare
|
||||
@@ -17,10 +20,10 @@ class ITUZoneData(LocalFileStaticDataProvider):
|
||||
|
||||
PATH = "datafiles/ituzones.geojson"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("ITU Zone Data", provider_config, self.PATH)
|
||||
|
||||
def _load_data(self, path):
|
||||
def _load_data(self, path: str) -> bool:
|
||||
try:
|
||||
with open(path) as f:
|
||||
itu_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from requests import Response
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from providers.staticdata.file_download_static_data_provider import (
|
||||
@@ -15,14 +20,14 @@ class K0SWE(FileDownloadStaticDataProvider):
|
||||
POLL_INTERVAL_DAYS = 7
|
||||
DATA_URL = "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__("K0SWE DXCC JSON", provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _handle_http_response(self, http_response):
|
||||
def _handle_http_response(self, http_response: Response) -> bool:
|
||||
try:
|
||||
dxcc_list = http_response.json()["dxcc"]
|
||||
# Reformat as a map for to place in the data store
|
||||
dxcc_map = {}
|
||||
dxcc_map: dict[int, Any] = {}
|
||||
for dxcc in dxcc_list:
|
||||
dxcc_map[dxcc["entityCode"]] = dxcc
|
||||
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
@@ -11,12 +14,12 @@ logger = logging.getLogger(__name__)
|
||||
class LocalFileStaticDataProvider(StaticDataProvider):
|
||||
"""Generic static reference data provider class for providers that fetch their data from a local file on startup."""
|
||||
|
||||
def __init__(self, name, provider_config, path):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any], path: str) -> None:
|
||||
super().__init__(name, provider_config)
|
||||
self._path = path
|
||||
self._stop = False
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.debug(f"Loading {self.name} static reference data from file.")
|
||||
try:
|
||||
ok = self._load_data(self._path)
|
||||
@@ -31,10 +34,10 @@ class LocalFileStaticDataProvider(StaticDataProvider):
|
||||
self.status = "Error"
|
||||
logger.exception(f"Exception in local file Static Data Provider ({self.name})")
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
self._stop = True
|
||||
|
||||
def _load_data(self, path):
|
||||
def _load_data(self, path: str) -> bool:
|
||||
"""Load data from the given file path. Return true if successful, false otherwise."""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
@@ -6,21 +9,21 @@ import pytz
|
||||
class StaticDataProvider:
|
||||
"""Generic static reference data provider class. Subclasses of this query the individual URLs or files for data."""
|
||||
|
||||
def __init__(self, name, provider_config):
|
||||
def __init__(self, name: str, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor"""
|
||||
|
||||
self.name = name
|
||||
self.enabled = provider_config["enabled"]
|
||||
self.enabled: bool = provider_config["enabled"]
|
||||
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Not Started" if self.enabled else "Disabled"
|
||||
self.reference_count = 0
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
Reference in New Issue
Block a user