w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-09-20 14:27:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Use ruff linter to fix issues and provide consistent formatting

Browse Source
This commit is contained in:
Ian Renton
2026-08-15 08:25:54 +01:00
parent 7391c28cd0
commit af3f82c14d
121 changed files with 1989 additions and 996 deletions
Download Patch File Download Diff File Expand all files Collapse all files
providers/alert/alert_provider.py
+1 -1
View File
@@ -29,7 +29,7 @@ class AlertProvider:
# Sort the batch so that earliest ones go in first. This helps keep the ordering correct when alerts are fired
# off to SSE listeners.
alerts = sorted(alerts, key=lambda a: (a.start_time if a and a.start_time else 0))
alerts = sorted(alerts, key=lambda a: a.start_time if a and a.start_time else 0)
for alert in alerts:
# Fill in any blanks and add to the list
alert.infer_missing()
providers/alert/bota.py
+16 -14
View File
@@ -3,9 +3,9 @@ from datetime import datetime, timedelta
import pytz
from bs4 import BeautifulSoup
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class BOTA(HTTPAlertProvider):
@@ -23,17 +23,17 @@ class BOTA(HTTPAlertProvider):
bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
if not bs.body:
return new_alerts
div = bs.body.find('div', attrs={'class': 'view-activations-public'})
div = bs.body.find("div", attrs={"class": "view-activations-public"})
if div:
table = div.find('table', attrs={'class': 'views-table'})
table = div.find("table", attrs={"class": "views-table"})
if table:
tbody = table.find('tbody')
tbody = table.find("tbody")
if not tbody:
return new_alerts
for row in tbody.find_all('tr'):
cells = row.find_all('td')
first_cell_anchor = cells[0].find('a') if len(cells) > 0 else None
second_cell_anchor = cells[1].find('a') if len(cells) > 1 else None
for row in tbody.find_all("tr"):
cells = row.find_all("td")
first_cell_anchor = cells[0].find("a") if len(cells) > 0 else None
second_cell_anchor = cells[1].find("a") if len(cells) > 1 else None
if not first_cell_anchor or not second_cell_anchor:
continue
first_cell_text = first_cell_anchor.get_text().strip()
@@ -41,7 +41,7 @@ class BOTA(HTTPAlertProvider):
dx_call = second_cell_anchor.get_text().strip().upper()
# Get the date, dealing with the fact we get no year so have to figure out if it's last year or next year
date_span = cells[2].find('span') if len(cells) > 2 else None
date_span = cells[2].find("span") if len(cells) > 2 else None
if not date_span:
continue
date_text = date_span.get_text().strip()
@@ -52,11 +52,13 @@ class BOTA(HTTPAlertProvider):
date_time = date_time.replace(year=datetime.now(pytz.UTC).year + 1)
# Convert to our alert format
alert = Alert(source=self.name,
dx_calls=[dx_call],
sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
start_time=date_time.timestamp(),
is_dxpedition=False)
alert = Alert(
source=self.name,
dx_calls=[dx_call],
sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
start_time=date_time.timestamp(),
is_dxpedition=False,
)
new_alerts.append(alert)
return new_alerts
providers/alert/http_alert_provider.py
+3 -3
View File
@@ -1,13 +1,13 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from providers.alert.alert_provider import AlertProvider
from core.constants import HTTP_HEADERS
from providers.alert.alert_provider import AlertProvider
class HTTPAlertProvider(AlertProvider):
providers/alert/ng3k.py
+21 -14
View File
@@ -6,8 +6,8 @@ import pytz
from rss_parser import Parser
from rss_parser.models.rss import RSS
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from providers.alert.http_alert_provider import HTTPAlertProvider
class NG3K(HTTPAlertProvider):
@@ -49,11 +49,16 @@ class NG3K(HTTPAlertProvider):
end_day = end_string.split(", ")[0].strip()
end_mon = start_mon
start_timestamp = datetime.strptime(f"{start_year} {start_mon} {start_day}", "%Y %b %d").replace(
tzinfo=pytz.UTC).timestamp()
end_timestamp = datetime.strptime(f"{end_year} {end_mon} {end_day} 23:59",
"%Y %b %d %H:%M").replace(
tzinfo=pytz.UTC).timestamp()
start_timestamp = (
datetime.strptime(f"{start_year} {start_mon} {start_day}", "%Y %b %d")
.replace(tzinfo=pytz.UTC)
.timestamp()
)
end_timestamp = (
datetime.strptime(f"{end_year} {end_mon} {end_day} 23:59", "%Y %b %d %H:%M")
.replace(tzinfo=pytz.UTC)
.timestamp()
)
# Sometimes the DX callsign is "real", sometimes you just get a prefix with the real working callsigns being
# provided in the "by" field. e.g. call="JW", by="By LA7XK as JW7XK, LA6VM as JW6VM, LA9DL as JW9DL". So
@@ -75,14 +80,16 @@ class NG3K(HTTPAlertProvider):
comment = extra_parts[3] if len(extra_parts) > 3 else ""
# Convert to our alert format
alert = Alert(source=self.name,
dx_calls=dx_calls,
dx_country=dx_country,
freqs_modes=bands + (f"; {modes}" if modes != "" else ""),
comment=f"{by}; {comment}; {qsl_info}",
start_time=start_timestamp,
end_time=end_timestamp,
is_dxpedition=True)
alert = Alert(
source=self.name,
dx_calls=dx_calls,
dx_country=dx_country,
freqs_modes=bands + (f"; {modes}" if modes != "" else ""),
comment=f"{by}; {comment}; {qsl_info}",
start_time=start_timestamp,
end_time=end_timestamp,
is_dxpedition=True,
)
# Add to our list.
new_alerts.append(alert)
providers/alert/parksnpeaks.py
+25 -12
View File
@@ -3,9 +3,9 @@ from datetime import datetime
import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class ParksNPeaks(HTTPAlertProvider):
@@ -30,8 +30,9 @@ class ParksNPeaks(HTTPAlertProvider):
else:
sig_ref = source_alert["WWFFID"]
sig_ref_name = source_alert["Location"]
start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp()
start_time = (
datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp()
)
sigrefs = []
# PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list,
@@ -40,17 +41,29 @@ class ParksNPeaks(HTTPAlertProvider):
sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)]
# Convert to our alert format
alert = Alert(source=self.name,
source_id=source_alert["alID"],
dx_calls=[source_alert["CallSign"].upper()],
freqs_modes=f"{source_alert['Freq']} {source_alert['MODE']}",
comment=source_alert["Comments"],
sig_refs=sigrefs,
start_time=start_time,
is_dxpedition=False)
alert = Alert(
source=self.name,
source_id=source_alert["alID"],
dx_calls=[source_alert["CallSign"].upper()],
freqs_modes=f"{source_alert['Freq']} {source_alert['MODE']}",
comment=source_alert["Comments"],
sig_refs=sigrefs,
start_time=start_time,
is_dxpedition=False,
)
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig and sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "SANPCPA", "LLOTA", "QRP"]:
if sig and sig not in [
"POTA",
"SOTA",
"WWFF",
"SIOTA",
"ZLOTA",
"KRMNPA",
"SANPCPA",
"LLOTA",
"QRP",
]:
logging.warning(f"PNP alert found with sig {sig}, developer needs to add support for this!")
# If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
providers/alert/pota.py
+26 -13
View File
@@ -2,9 +2,9 @@ from datetime import datetime
import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class POTA(HTTPAlertProvider):
@@ -21,18 +21,31 @@ class POTA(HTTPAlertProvider):
# Iterate through source data
for source_alert in http_response.json():
# Convert to our alert format
alert = Alert(source=self.name,
source_id=source_alert["scheduledActivitiesId"],
dx_calls=[source_alert["activator"].upper()],
freqs_modes=source_alert["frequencies"],
comment=source_alert["comments"],
sig_refs=[SIGRef(id=source_alert["reference"], sig="POTA", name=source_alert["name"],
url=f"https://pota.app/#/park/{source_alert['reference']}")],
start_time=datetime.strptime(source_alert["startDate"] + source_alert["startTime"],
"%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"],
"%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
is_dxpedition=False)
alert = Alert(
source=self.name,
source_id=source_alert["scheduledActivitiesId"],
dx_calls=[source_alert["activator"].upper()],
freqs_modes=source_alert["frequencies"],
comment=source_alert["comments"],
sig_refs=[
SIGRef(
id=source_alert["reference"],
sig="POTA",
name=source_alert["name"],
url=f"https://pota.app/#/park/{source_alert['reference']}",
)
],
start_time=datetime.strptime(
source_alert["startDate"] + source_alert["startTime"],
"%Y-%m-%d%H:%M",
)
.replace(tzinfo=pytz.UTC)
.timestamp(),
end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"], "%Y-%m-%d%H:%M")
.replace(tzinfo=pytz.UTC)
.timestamp(),
is_dxpedition=False,
)
# Add to our list, but exclude any old spots that POTA can sometimes give us where even the end time is
# in the past. Don't worry about de-duping, removing old alerts etc. at this point; other code will do
providers/alert/sota.py
+21 -13
View File
@@ -2,9 +2,9 @@ from datetime import datetime
import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class SOTA(HTTPAlertProvider):
@@ -26,18 +26,26 @@ class SOTA(HTTPAlertProvider):
summit_points = None
if len(details) > 2:
summit_points = int(details[-1].split(" ")[0])
alert = Alert(source=self.name,
source_id=source_alert["id"],
dx_calls=[source_alert["activatingCallsign"].upper()],
dx_names=[source_alert["activatorName"].upper()],
freqs_modes=source_alert["frequency"],
comment=source_alert["comments"],
sig_refs=[
SIGRef(id=f"{source_alert['associationCode']}/{source_alert['summitCode']}", sig="SOTA",
name=summit_name, activation_score=summit_points)],
start_time=datetime.strptime(source_alert["dateActivated"],
"%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=pytz.UTC).timestamp(),
is_dxpedition=False)
alert = Alert(
source=self.name,
source_id=source_alert["id"],
dx_calls=[source_alert["activatingCallsign"].upper()],
dx_names=[source_alert["activatorName"].upper()],
freqs_modes=source_alert["frequency"],
comment=source_alert["comments"],
sig_refs=[
SIGRef(
id=f"{source_alert['associationCode']}/{source_alert['summitCode']}",
sig="SOTA",
name=summit_name,
activation_score=summit_points,
)
],
start_time=datetime.strptime(source_alert["dateActivated"], "%Y-%m-%dT%H:%M:%SZ")
.replace(tzinfo=pytz.UTC)
.timestamp(),
is_dxpedition=False,
)
# Add to our list
new_alerts.append(alert)
providers/alert/wota.py
+15 -10
View File
@@ -5,9 +5,9 @@ import pytz
from rss_parser import Parser as RSSParser
from rss_parser.models.rss import RSS
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class WOTA(HTTPAlertProvider):
@@ -25,9 +25,12 @@ class WOTA(HTTPAlertProvider):
rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig")))
# Iterate through source data
for source_alert in rss.channel.items:
# Reject GUID missing or zero
if not source_alert.guid or not source_alert.guid.content or source_alert.guid.content == "http://www.wota.org.uk/alerts/0":
if (
not source_alert.guid
or not source_alert.guid.content
or source_alert.guid.content == "http://www.wota.org.uk/alerts/0"
):
continue
# Pick apart the title
@@ -51,13 +54,15 @@ class WOTA(HTTPAlertProvider):
time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
# Convert to our alert format
alert = Alert(source=self.name,
source_id=source_alert.guid.content,
dx_calls=[dx_call],
freqs_modes=freqs_modes,
comment=comment,
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
start_time=time.timestamp())
alert = Alert(
source=self.name,
source_id=source_alert.guid.content,
dx_calls=[dx_call],
freqs_modes=freqs_modes,
comment=comment,
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
start_time=time.timestamp(),
)
# Add to our list.
new_alerts.append(alert)
providers/alert/wwff.py
+16 -12
View File
@@ -2,9 +2,9 @@ from datetime import datetime
import pytz
from providers.alert.http_alert_provider import HTTPAlertProvider
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
class WWFF(HTTPAlertProvider):
@@ -21,17 +21,21 @@ class WWFF(HTTPAlertProvider):
# Iterate through source data
for source_alert in http_response.json():
# Convert to our alert format
alert = Alert(source=self.name,
source_id=source_alert["id"],
dx_calls=[source_alert["activator_call"].upper()],
freqs_modes=f"{source_alert['band']} {source_alert['mode']}",
comment=source_alert["remarks"],
sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")],
start_time=datetime.strptime(source_alert["utc_start"],
"%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
end_time=datetime.strptime(source_alert["utc_end"],
"%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
is_dxpedition=False)
alert = Alert(
source=self.name,
source_id=source_alert["id"],
dx_calls=[source_alert["activator_call"].upper()],
freqs_modes=f"{source_alert['band']} {source_alert['mode']}",
comment=source_alert["remarks"],
sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")],
start_time=datetime.strptime(source_alert["utc_start"], "%Y-%m-%d %H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
end_time=datetime.strptime(source_alert["utc_end"], "%Y-%m-%d %H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
is_dxpedition=False,
)
# Add to our list
new_alerts.append(alert)
providers/callsigndata/api_query_callsign_data_provider.py
+1 -1
View File
@@ -5,7 +5,7 @@ class APIQueryCallsignDataProvider(CallsignDataProvider):
"""Generic callsign data provider class for providers that fetch their data from the web on-demand using an API."""
def __init__(self, name, provider_config, storage):
""" Set up the provider."""
"""Set up the provider."""
super().__init__(name, provider_config, storage)
if self.enabled:
providers/callsigndata/callsign_data_provider.py
-1
View File
@@ -51,7 +51,6 @@ class CallsignDataProvider:
else:
return None
def _perform_new_lookup(self, callsign, lookup_credentials):
"""Makes a new request to the data source for callsign data."""
providers/callsigndata/clublogapi.py
+6 -4
View File
@@ -2,12 +2,14 @@ import logging
from datetime import datetime
import pytz
from pyhamtools import LookupLib, Callinfo
from pyhamtools import Callinfo, LookupLib
from core.data_store import DATA_STORE
from core.utils import get_callsign_object_from_pyhamtools_callinfo
from data.callsign import Callsign
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
from providers.callsigndata.api_query_callsign_data_provider import (
APIQueryCallsignDataProvider,
)
class ClublogAPI(APIQueryCallsignDataProvider):
@@ -24,11 +26,11 @@ class ClublogAPI(APIQueryCallsignDataProvider):
else:
provider_config["enabled"] = False
logging.warning(
"Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
"Clublog XML callsign data provider configured but no api key was provided, this has been disabled."
)
super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
def _perform_new_lookup(self, callsign, lookup_credentials):
callsign_data = Callsign(call=callsign)
providers/callsigndata/clublogxml.py
+14 -5
View File
@@ -1,12 +1,14 @@
import gzip
import logging
from pyhamtools import LookupLib, Callinfo
from pyhamtools import Callinfo, LookupLib
from core.data_store import DATA_STORE
from core.utils import get_callsign_object_from_pyhamtools_callinfo
from data.callsign import Callsign
from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
from providers.callsigndata.file_download_callsign_data_provider import (
FileDownloadCallsignDataProvider,
)
class ClublogXML(FileDownloadCallsignDataProvider):
@@ -24,10 +26,17 @@ class ClublogXML(FileDownloadCallsignDataProvider):
if self._api_key == "":
provider_config["enabled"] = False
logging.warning(
"Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
"Clublog XML callsign data provider configured but no api key was provided, this has been disabled."
)
super().__init__("Clublog XML", provider_config, f"{self.DATA_URL}?api={self._api_key}",
self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS, DATA_STORE.callsign_data_clublogxml)
super().__init__(
"Clublog XML",
provider_config,
f"{self.DATA_URL}?api={self._api_key}",
self.CACHE_PATH_ZIPPED,
self.POLL_INTERVAL_DAYS,
DATA_STORE.callsign_data_clublogxml,
)
def _handle_file(self, path):
try:
providers/callsigndata/countryfiles.py
+12 -4
View File
@@ -1,11 +1,13 @@
import logging
from pyhamtools import LookupLib, Callinfo
from pyhamtools import Callinfo, LookupLib
from core.data_store import DATA_STORE
from core.utils import get_callsign_object_from_pyhamtools_callinfo
from data.callsign import Callsign
from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
from providers.callsigndata.file_download_callsign_data_provider import (
FileDownloadCallsignDataProvider,
)
class CountryFiles(FileDownloadCallsignDataProvider):
@@ -17,8 +19,14 @@ class CountryFiles(FileDownloadCallsignDataProvider):
_callinfo = None
def __init__(self, provider_config):
super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS,
DATA_STORE.callsign_data_countryfiles)
super().__init__(
"CountryFiles.com",
provider_config,
self.DATA_URL,
self.CACHE_PATH,
self.POLL_INTERVAL_DAYS,
DATA_STORE.callsign_data_countryfiles,
)
def _handle_file(self, path):
try:
providers/callsigndata/file_download_callsign_data_provider.py
+6 -5
View File
@@ -1,6 +1,6 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
from requests import ReadTimeout
@@ -15,7 +15,7 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
"""Generic callsign data provider class for providers that fetch their data from the web by downloading a file."""
def __init__(self, name, provider_config, url, cache_file_path, poll_interval, storage):
""" Set up the provider, note poll_interval is in *days*."""
"""Set up the provider, note poll_interval is in *days*."""
super().__init__(name, provider_config, storage)
self._url = url
self._cache_file_path = cache_file_path
@@ -30,8 +30,7 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting.
logging.info(
f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
logging.info(f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}")
self._thread.start()
@@ -68,7 +67,9 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
else:
self.status = "Error"
logging.warning(f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}.")
logging.warning(
f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}."
)
except ConnectionError:
self.status = "Error"
providers/callsigndata/hamqth.py
+42 -31
View File
@@ -1,6 +1,6 @@
import logging
import urllib.parse
from datetime import timedelta, datetime
from datetime import datetime, timedelta
import pytz
import xmltodict
@@ -10,10 +10,12 @@ from requests_cache import CachedSession
from core.config import SERVER_OWNER_CALLSIGN
from core.constants import HTTP_HEADERS, SOFTWARE_VERSION
from core.data_store import DATA_STORE, CACHE_DIR
from core.data_store import CACHE_DIR, DATA_STORE
from core.url_data_cache import URLDataCache
from data.callsign import Callsign
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
from providers.callsigndata.api_query_callsign_data_provider import (
APIQueryCallsignDataProvider,
)
class HamQTH(APIQueryCallsignDataProvider):
@@ -26,14 +28,15 @@ class HamQTH(APIQueryCallsignDataProvider):
self._URL_DATA_CACHE = URLDataCache("hamqth")
# Separate URL cache for session key lookups. Once a session key is returned from logging in with a username
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/hamqth-creds",
expire_after=timedelta(minutes=55))
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/hamqth-creds", expire_after=timedelta(minutes=55))
def _perform_new_lookup(self, callsign, lookup_credentials):
# If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because
# # someone might provide credentials next time around.
if not lookup_credentials or not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
or lookup_credentials.hamqth_session_id):
if not lookup_credentials or not (
(lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
or lookup_credentials.hamqth_session_id
):
return None
try:
@@ -45,7 +48,8 @@ class HamQTH(APIQueryCallsignDataProvider):
try:
session_data = self._CREDENTIALS_CACHE.get(
f"{self._HAMQTH_BASE_URL}?u={urllib.parse.quote_plus(lookup_credentials.hamqth_username)}&p={urllib.parse.quote_plus(lookup_credentials.hamqth_password)}",
headers=HTTP_HEADERS).content
headers=HTTP_HEADERS,
).content
dict_data = xmltodict.parse(session_data)
if "session_id" in dict_data["HamQTH"]["session"]:
session_id = str(dict_data["HamQTH"]["session"]["session_id"])
@@ -67,13 +71,16 @@ class HamQTH(APIQueryCallsignDataProvider):
if home_call != callsign:
calls_to_try.append(home_call)
except ValueError:
logging.debug("Could not look up home call for callsign %s", callsign)
logging.debug(f"Could not look up home call for callsign {callsign}")
# Try looking up each call using the API
for lookup_call in calls_to_try:
try:
response = self._URL_DATA_CACHE.get(
f"{self._HAMQTH_BASE_URL}?id={session_id}&callsign={urllib.parse.quote_plus(lookup_call)}&prg={self._PRG}", headers=HTTP_HEADERS, timeout=10)
f"{self._HAMQTH_BASE_URL}?id={session_id}&callsign={urllib.parse.quote_plus(lookup_call)}&prg={self._PRG}",
headers=HTTP_HEADERS,
timeout=10,
)
if response.ok:
# Found data, convert it to our object and return it
data = xmltodict.parse(response.content)["HamQTH"]["search"]
@@ -83,19 +90,18 @@ class HamQTH(APIQueryCallsignDataProvider):
return self.hamqth_response_to_callsign(callsign, data)
elif not response.from_cache:
logging.warning("HTTP %d looking up callsign %s using HamQTH", response.status_code,
lookup_call)
logging.warning(f"HTTP {response.status_code} looking up callsign {lookup_call} using HamQTH")
except (KeyError, ValueError):
continue
except ConnectionError:
logging.warning(f"Connection error when looking up callsign %s using HamQTH", lookup_call)
logging.warning(f"Connection error when looking up callsign {lookup_call} using HamQTH")
continue
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when looking up callsign %s using HamQTH", lookup_call)
logging.warning(f"Timeout when looking up callsign {lookup_call} using HamQTH")
continue
except Exception:
logging.exception("Exception when looking up callsign %s using HamQTH", lookup_call)
logging.exception(f"Exception when looking up callsign {lookup_call} using HamQTH")
continue
# Not found in HamQTH; return a Callsign object with no data so we cache that and don't keep retrying
@@ -114,9 +120,12 @@ class HamQTH(APIQueryCallsignDataProvider):
# Check for sensible latitudes
lat = None
lon = None
if "latitude" in data and "longitude" in data and (
float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float(
data["latitude"]) < 89.9:
if (
"latitude" in data
and "longitude" in data
and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
and -89.9 < float(data["latitude"]) < 89.9
):
lat = float(data["latitude"])
lon = float(data["longitude"])
@@ -125,16 +134,18 @@ class HamQTH(APIQueryCallsignDataProvider):
if "grid" in data and not data["grid"].startswith("AA00"):
grid = data["grid"]
return Callsign(call=callsign,
home_call=callinfo.Callinfo.get_homecall(callsign),
name=data["nick"] if "nick" in data else None,
qth=data["qth"] if "qth" in data else None,
country=data["country"] if "country" in data else None,
continent=data["continent"] if "continent" in data else None,
latitude=lat,
longitude=lon,
grid=grid,
dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cq"]) if "cq" in data else None,
itu_zone=int(data["itu"]) if "itu" in data else None,
location_source="HOME QTH")
return Callsign(
call=callsign,
home_call=callinfo.Callinfo.get_homecall(callsign),
name=data["nick"] if "nick" in data else None,
qth=data["qth"] if "qth" in data else None,
country=data["country"] if "country" in data else None,
continent=data["continent"] if "continent" in data else None,
latitude=lat,
longitude=lon,
grid=grid,
dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cq"]) if "cq" in data else None,
itu_zone=int(data["itu"]) if "itu" in data else None,
location_source="HOME QTH",
)
providers/callsigndata/qrz.py
+45 -34
View File
@@ -1,6 +1,6 @@
import logging
import urllib.parse
from datetime import timedelta, datetime
from datetime import datetime, timedelta
import pytz
import xmltodict
@@ -9,10 +9,12 @@ from requests import ConnectTimeout, ReadTimeout
from requests_cache import CachedSession
from core.constants import HTTP_HEADERS
from core.data_store import DATA_STORE, CACHE_DIR
from core.data_store import CACHE_DIR, DATA_STORE
from core.url_data_cache import URLDataCache
from data.callsign import Callsign
from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
from providers.callsigndata.api_query_callsign_data_provider import (
APIQueryCallsignDataProvider,
)
class QRZ(APIQueryCallsignDataProvider):
@@ -24,14 +26,14 @@ class QRZ(APIQueryCallsignDataProvider):
self._URL_DATA_CACHE = URLDataCache("qrz")
# Separate URL cache for session key lookups. Once a session key is returned from logging in with a username
# and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/qrz-creds",
expire_after=timedelta(minutes=55))
self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/qrz-creds", expire_after=timedelta(minutes=55))
def _perform_new_lookup(self, callsign, lookup_credentials):
# If we don't have QRZ credentials, skip this lookup. Return None so we don't *cache* the lack of data, because
# someone might provide credentials next time around.
if not lookup_credentials or not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
or lookup_credentials.qrz_session_key):
if not lookup_credentials or not (
(lookup_credentials.qrz_username and lookup_credentials.qrz_password) or lookup_credentials.qrz_session_key
):
return None
try:
@@ -43,7 +45,8 @@ class QRZ(APIQueryCallsignDataProvider):
try:
login_response = self._CREDENTIALS_CACHE.get(
f"{self._QRZ_BASE_URL}?username={urllib.parse.quote_plus(lookup_credentials.qrz_username)}&password={urllib.parse.quote_plus(lookup_credentials.qrz_password)}&agent=spothole",
headers=HTTP_HEADERS).content
headers=HTTP_HEADERS,
).content
login_data = xmltodict.parse(login_response)
session = login_data.get("QRZDatabase", {}).get("Session", {})
if "Key" in session:
@@ -66,14 +69,16 @@ class QRZ(APIQueryCallsignDataProvider):
if home_call != callsign:
calls_to_try.append(home_call)
except ValueError:
logging.debug("Could not look up home call for callsign %s", callsign)
logging.debug(f"Could not look up home call for callsign {callsign}")
# Try looking up each call using the API
for lookup_call in calls_to_try:
try:
response = self._URL_DATA_CACHE.get(
f"{self._QRZ_BASE_URL}?s={session_key}&callsign={urllib.parse.quote_plus(lookup_call)}",
headers=HTTP_HEADERS, timeout=10)
headers=HTTP_HEADERS,
timeout=10,
)
if response.ok:
qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
if qrz_response:
@@ -88,24 +93,25 @@ class QRZ(APIQueryCallsignDataProvider):
elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
# Errors here are normally just "callsign not in database", no need to log that ourselves
# above debug level.
logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call,
qrz_response.get("Session").get("Error"))
logging.debug(
f"QRZ returned an error looking up callsign {lookup_call}: {qrz_response.get('Session').get('Error')}"
)
elif not response.from_cache:
logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call)
logging.warning(f"QRZ returned a malformed response looking up callsign {lookup_call}")
elif not response.from_cache:
logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call)
logging.warning(f"HTTP {response.status_code} looking up callsign {lookup_call} using QRZ")
except (KeyError, ValueError):
continue
except ConnectionError:
logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call)
logging.warning(f"Connection error when looking up callsign {lookup_call} using QRZ")
continue
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call)
logging.warning(f"Timeout when looking up callsign {lookup_call} using QRZ.")
continue
except Exception:
logging.exception("Exception when looking up callsign %s using QRZ", lookup_call)
logging.exception(f"Exception when looking up callsign {lookup_call} using QRZ")
continue
# Not found in QRZ; return a Callsign object with no data so we cache that and don't keep retrying
@@ -128,16 +134,19 @@ class QRZ(APIQueryCallsignDataProvider):
if "fname" in data:
name = data["fname"]
if "nick" in data:
name = f"{name} \"{data['nick']}\""
name = f'{name} "{data["nick"]}"'
if "name" in data:
name = f"{name} {data['name']}"
# Check for sensible latitudes
lat = None
lon = None
if "latitude" in data and "longitude" in data and (
float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float(
data["latitude"]) < 89.9:
if (
"latitude" in data
and "longitude" in data
and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
and -89.9 < float(data["latitude"]) < 89.9
):
lat = float(data["latitude"])
lon = float(data["longitude"])
@@ -146,16 +155,18 @@ class QRZ(APIQueryCallsignDataProvider):
if "grid" in data and not data["grid"].startswith("AA00"):
grid = data["grid"]
return Callsign(call=callsign,
home_call=callinfo.Callinfo.get_homecall(callsign),
name=name,
qth=data["addr2"] if "addr2" in data else None,
country=data["country"] if "country" in data else None,
continent=data["continent"] if "continent" in data else None,
latitude=lat,
longitude=lon,
grid=grid,
dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
location_source="HOME QTH")
return Callsign(
call=callsign,
home_call=callinfo.Callinfo.get_homecall(callsign),
name=name,
qth=data["addr2"] if "addr2" in data else None,
country=data["country"] if "country" in data else None,
continent=data["continent"] if "continent" in data else None,
latitude=lat,
longitude=lon,
grid=grid,
dxcc_id=int(data["adif"]) if "adif" in data else None,
cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
location_source="HOME QTH",
)
providers/sigrefdata/arlhs.py
+15 -9
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ARLHS(FileDownloadSIGRefDataProvider):
@@ -20,14 +22,18 @@ class ARLHS(FileDownloadSIGRefDataProvider):
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
if "ARLHS" in row and row["ARLHS"] != "":
ref_id = row["ARLHS"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
ref_type="Lighthouse",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Lighthouse",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
grid=row["Maidenhead Locator"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/dme.py
+21 -10
View File
@@ -4,7 +4,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
from providers.sigrefdata.local_file_sig_ref_data_provider import (
LocalFileSIGRefDataProvider,
)
class DME(LocalFileSIGRefDataProvider):
@@ -21,16 +23,25 @@ class DME(LocalFileSIGRefDataProvider):
with open(path, encoding="latin-1") as _f:
for row in csv.DictReader(_f, delimiter=";"):
ref_id = row["COD_INE"][:5]
latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
"LATITUD_ETRS89_REGCAN95") else None
longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
"LONGITUD_ETRS89_REGCAN95") else None
latitude = (
float(row["LATITUD_ETRS89_REGCAN95"].replace(",", "."))
if row.get("LATITUD_ETRS89_REGCAN95")
else None
)
longitude = (
float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", "."))
if row.get("LONGITUD_ETRS89_REGCAN95")
else None
)
ref = SIGRef(sig=self.SIG, id=ref_id,
ref_type="Town",
name=f"{row['NOMBRE_ACTUAL']}, {row['PROVINCIA']}",
latitude=latitude,
longitude=longitude)
ref = SIGRef(
sig=self.SIG,
id=ref_id,
ref_type="Town",
name=f"{row['NOMBRE_ACTUAL']}, {row['PROVINCIA']}",
latitude=latitude,
longitude=longitude,
)
if latitude and longitude:
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
providers/sigrefdata/file_download_sig_ref_data_provider.py
+3 -4
View File
@@ -1,6 +1,6 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
from requests import ReadTimeout
@@ -15,7 +15,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
"""Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
def __init__(self, sig_name, provider_config, url, poll_interval):
""" Set up the provider, note poll_interval is in *days*."""
"""Set up the provider, note poll_interval is in *days*."""
super().__init__(sig_name, provider_config)
self._url = url
self._poll_interval = poll_interval
@@ -26,8 +26,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting.
logging.info(
f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
logging.info(f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}")
self._thread.start()
providers/sigrefdata/gma.py
+18 -11
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class GMA(FileDownloadSIGRefDataProvider):
@@ -19,16 +21,21 @@ class GMA(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
ref_type="Summit",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
altitude=float(row["Height (m)"].replace("m", "")) if "Height (m)" in row and row[
"Height (m)"] != "" else None,
grid=row["Maidenhead Locator"]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Summit",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
altitude=float(row["Height (m)"].replace("m", ""))
if "Height (m)" in row and row["Height (m)"] != ""
else None,
grid=row["Maidenhead Locator"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/illw.py
+15 -9
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ILLW(FileDownloadSIGRefDataProvider):
@@ -20,14 +22,18 @@ class ILLW(FileDownloadSIGRefDataProvider):
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
if "ILLW" in row and row["ILLW"] != "":
ref_id = row["ILLW"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
ref_type="Lighthouse",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Lighthouse",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
grid=row["Maidenhead Locator"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/iota.py
+19 -4
View File
@@ -4,7 +4,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class IOTA(FileDownloadSIGRefDataProvider):
@@ -29,10 +31,23 @@ class IOTA(FileDownloadSIGRefDataProvider):
try:
grid = latlong_to_locator(latitude, longitude, 6)
except ValueError:
logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
logging.debug(
"Error converting lat/lon to locator for an IOTA reference %f %f",
latitude,
longitude,
)
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
ref_type="Island", grid=grid, latitude=latitude, longitude=longitude))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=ref["name"],
ref_type="Island",
grid=grid,
latitude=latitude,
longitude=longitude,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
# of the data in this case
providers/sigrefdata/krmnpa.py
+4 -2
View File
@@ -1,4 +1,6 @@
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import ParksNPeaksKMLSIGRefDataProvider
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import (
ParksNPeaksKMLSIGRefDataProvider,
)
class KRMNPA(ParksNPeaksKMLSIGRefDataProvider):
@@ -9,4 +11,4 @@ class KRMNPA(ParksNPeaksKMLSIGRefDataProvider):
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=KRMNPA&poiFormat=4"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
providers/sigrefdata/llota.py
+15 -7
View File
@@ -3,7 +3,9 @@ from time import sleep
from pyhamtools.locator import locator_to_latlong
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class LLOTA(FileDownloadSIGRefDataProvider):
@@ -25,12 +27,18 @@ class LLOTA(FileDownloadSIGRefDataProvider):
grid = str(ref["grid_locator"])
ll = locator_to_latlong(grid)
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
ref_type="Lake",
url=f"https://llota.app/list/ref/{ref_id}",
grid=grid,
latitude=ll[0],
longitude=ll[1]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=str(ref["name"]),
ref_type="Lake",
url=f"https://llota.app/list/ref/{ref_id}",
grid=grid,
latitude=ll[0],
longitude=ll[1],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
# of the data in this case
providers/sigrefdata/mota.py
+15 -9
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class MOTA(FileDownloadSIGRefDataProvider):
@@ -19,14 +21,18 @@ class MOTA(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
ref_type="Mill",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Mill",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
grid=row["Maidenhead Locator"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/pnp_kml_sig_ref_data_provider.py
+13 -7
View File
@@ -5,7 +5,9 @@ from fastkml import kml
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
@@ -15,7 +17,7 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
REF_PATTERN = re.compile(r"VKFF-\d+")
def __init__(self, sig_name, provider_config, url, poll_interval):
""" Set up the provider, note poll_interval is in *days*."""
"""Set up the provider, note poll_interval is in *days*."""
super().__init__(sig_name, provider_config, url, poll_interval)
def _http_response_to_data(self, http_response):
@@ -37,11 +39,15 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
longitude, latitude = placemark.geometry.x, placemark.geometry.y
ref = SIGRef(sig=self.sig_name, id=ref_id, name=placemark.name,
ref_type="Park",
url=f"https://parksnpeaks.org/getPark.php?actPark={ref_id}",
latitude=latitude,
longitude=longitude)
ref = SIGRef(
sig=self.sig_name,
id=ref_id,
name=placemark.name,
ref_type="Park",
url=f"https://parksnpeaks.org/getPark.php?actPark={ref_id}",
latitude=latitude,
longitude=longitude,
)
if latitude and longitude:
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
providers/sigrefdata/pota.py
+15 -9
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class POTA(FileDownloadSIGRefDataProvider):
@@ -19,14 +21,18 @@ class POTA(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
ref_type="Park",
url=f"https://pota.app/#/park/{ref_id}",
grid=row["grid"] if "grid" in row else None,
latitude=float(row["latitude"]) if "latitude" in row and row[
"latitude"] != "" else None,
longitude=float(row["longitude"]) if "longitude" in row and row[
"longitude"] != "" else None))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["name"] if "name" in row else None,
ref_type="Park",
url=f"https://pota.app/#/park/{ref_id}",
grid=row["grid"] if "grid" in row else None,
latitude=float(row["latitude"]) if "latitude" in row and row["latitude"] != "" else None,
longitude=float(row["longitude"]) if "longitude" in row and row["longitude"] != "" else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/sanpcpa.py
+3 -1
View File
@@ -1,4 +1,6 @@
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import ParksNPeaksKMLSIGRefDataProvider
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import (
ParksNPeaksKMLSIGRefDataProvider,
)
class SANPCPA(ParksNPeaksKMLSIGRefDataProvider):
providers/sigrefdata/sig_ref_data_provider.py
+1 -3
View File
@@ -19,13 +19,11 @@ class SIGRefDataProvider:
self.reference_count = 0
self._stop = False
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
raise NotImplementedError("Subclasses must implement this method")
def stop(self):
"""Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
super()."""
@@ -47,4 +45,4 @@ class SIGRefDataProvider:
break
self.reference_count = len(new_data)
logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)
logging.info(f"Loaded {self.reference_count} references for {self.sig_name} into the data store.")
providers/sigrefdata/siota.py
+14 -6
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class SIOTA(FileDownloadSIGRefDataProvider):
@@ -19,11 +21,17 @@ class SIOTA(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["SILO_CODE"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
ref_type="Silo",
grid=row["LOCATOR"] if "LOCATOR" in row else None,
latitude=float(row["LAT"]) if "LAT" in row else None,
longitude=float(row["LNG"]) if "LNG" in row else None))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["NAME"] if "NAME" in row else None,
ref_type="Silo",
grid=row["LOCATOR"] if "LOCATOR" in row else None,
latitude=float(row["LAT"]) if "LAT" in row else None,
longitude=float(row["LNG"]) if "LNG" in row else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/sota.py
+14 -8
View File
@@ -4,7 +4,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class SOTA(FileDownloadSIGRefDataProvider):
@@ -24,13 +26,17 @@ class SOTA(FileDownloadSIGRefDataProvider):
latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
altitude = float(row["AltM"]) if "AltM" in row and row["AltM"] != "" else None
ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
ref_type="Summit",
url=f"https://www.sotadata.org.uk/en/summit/{ref_id}",
latitude=latitude,
longitude=longitude,
altitude=altitude,
activation_score=int(row["Points"]) if "Points" in row else None)
ref = SIGRef(
sig=self.SIG,
id=ref_id,
name=row["SummitName"] if "SummitName" in row else None,
ref_type="Summit",
url=f"https://www.sotadata.org.uk/en/summit/{ref_id}",
latitude=latitude,
longitude=longitude,
altitude=altitude,
activation_score=int(row["Points"]) if "Points" in row else None,
)
if latitude and longitude:
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
providers/sigrefdata/toilets.py
+13 -3
View File
@@ -1,7 +1,9 @@
import csv
from data.sig_ref import SIGRef
from providers.sigrefdata.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
from providers.sigrefdata.local_file_sig_ref_data_provider import (
LocalFileSIGRefDataProvider,
)
class Toilets(LocalFileSIGRefDataProvider):
@@ -19,8 +21,16 @@ class Toilets(LocalFileSIGRefDataProvider):
csv_data = _f.read()
dr = csv.DictReader(csv_data.splitlines())
for row in dr:
new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], ref_type="Toilet",
latitude=float(row["lat"]), longitude=float(row["lon"])))
new_data.append(
SIGRef(
sig=self.SIG,
id=row["ref"],
name=row["ref"],
ref_type="Toilet",
latitude=float(row["lat"]),
longitude=float(row["lon"]),
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
# of the data in this case
providers/sigrefdata/towers.py
+15 -7
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class Towers(FileDownloadSIGRefDataProvider):
@@ -19,12 +21,18 @@ class Towers(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
ref_id = row["Ref"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
ref_type="Tower",
url=f"https://wwtota.com/seznam/karta_rozhledny.php?ref={ref_id}",
grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Nazev"] if "Nazev" in row else None,
ref_type="Tower",
url=f"https://wwtota.com/seznam/karta_rozhledny.php?ref={ref_id}",
grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/wca.py
+16 -8
View File
@@ -5,7 +5,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WCA(FileDownloadSIGRefDataProvider):
@@ -34,14 +36,20 @@ class WCA(FileDownloadSIGRefDataProvider):
longitude = float(split[1])
grid = latlong_to_locator(latitude, longitude)
except ValueError:
logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for {ref_id}")
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
ref_type="Castle",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=latitude,
longitude=longitude,
grid=grid))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
ref_type="Castle",
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=latitude,
longitude=longitude,
grid=grid,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/wota.py
+16 -7
View File
@@ -1,7 +1,9 @@
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WOTA(FileDownloadSIGRefDataProvider):
@@ -25,12 +27,19 @@ class WOTA(FileDownloadSIGRefDataProvider):
number = int(ref_id.upper().replace("LDO-", ""))
url = f"https://www.wota.org.uk/MM_LDO-{number + 214!s}"
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
ref_type="Summit",
grid=feature["properties"]["qthLocator"],
latitude=feature["geometry"]["coordinates"][1],
longitude=feature["geometry"]["coordinates"][0],
altitude=feature["properties"]["height"]))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=feature["properties"]["title"],
url=url,
ref_type="Summit",
grid=feature["properties"]["qthLocator"],
latitude=feature["geometry"]["coordinates"][1],
longitude=feature["geometry"]["coordinates"][0],
altitude=feature["properties"]["height"],
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/wwbota.py
+15 -7
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WWBOTA(FileDownloadSIGRefDataProvider):
@@ -19,12 +21,18 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["Reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
ref_type="Bunker",
url=f"https://bunkerwiki.org/?s={ref_id}" if ref_id.startswith("B/G") else None,
grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["Name"] if "Name" in row else None,
ref_type="Bunker",
url=f"https://bunkerwiki.org/?s={ref_id}" if ref_id.startswith("B/G") else None,
grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/wwff.py
+19 -10
View File
@@ -2,7 +2,9 @@ import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class WWFF(FileDownloadSIGRefDataProvider):
@@ -19,15 +21,22 @@ class WWFF(FileDownloadSIGRefDataProvider):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
ref_type="Park",
url=f"https://wwff.co/directory/?showRef={ref_id}",
grid=row["iaruLocator"] if "iaruLocator" in row and row[
"iaruLocator"] != "-" else None,
latitude=float(row["latitude"]) if "latitude" in row and row[
"latitude"] != "" and row["latitude"] != "-" else None,
longitude=float(row["longitude"]) if "longitude" in row and row[
"longitude"] != "" and row["longitude"] != "-" else None))
new_data.append(
SIGRef(
sig=self.SIG,
id=ref_id,
name=row["name"] if "name" in row else None,
ref_type="Park",
url=f"https://wwff.co/directory/?showRef={ref_id}",
grid=row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None,
latitude=float(row["latitude"])
if "latitude" in row and row["latitude"] != "" and row["latitude"] != "-"
else None,
longitude=float(row["longitude"])
if "longitude" in row and row["longitude"] != "" and row["longitude"] != "-"
else None,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/sigrefdata/zlota.py
+12 -6
View File
@@ -3,7 +3,9 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
)
class ZLOTA(FileDownloadSIGRefDataProvider):
@@ -25,11 +27,15 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
latitude = ref["latitude"]
longitude = ref["longitude"]
new_ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
ref_type=ref["asset_type"].title(),
url=f"https://ontheair.nz/assets/{ref_id.replace('/', '_')}",
latitude=latitude,
longitude=longitude)
new_ref = SIGRef(
sig=self.SIG,
id=ref_id,
name=ref["name"],
ref_type=ref["asset_type"].title(),
url=f"https://ontheair.nz/assets/{ref_id.replace('/', '_')}",
latitude=latitude,
longitude=longitude,
)
# Check lat/lon validity and update grid accordingly
if latitude and longitude:
providers/solarconditions/giroionosonde.py
+27 -13
View File
@@ -1,11 +1,11 @@
import csv
import logging
from datetime import datetime, timezone, timedelta
from threading import Thread, Event
from datetime import datetime, timedelta, timezone
from threading import Event, Thread
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from providers.solarconditions.ionosonde_utils import compute_band_states
@@ -40,9 +40,16 @@ class GIROIonosonde(SolarConditionsProvider):
# entries so KC2G cache data is preserved.
existing = self._solar_conditions.ionosonde_data or {}
new_entries = {
s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
"luf": None, "band_states": None}
for s in self._stations if s["ursi"] not in existing
s["ursi"]: {
"ursi": s["ursi"],
"name": s["name"],
"fof2": None,
"muf": None,
"luf": None,
"band_states": None,
}
for s in self._stations
if s["ursi"] not in existing
}
if new_entries:
self.update_data({"ionosonde_data": {**existing, **new_entries}})
@@ -50,7 +57,7 @@ class GIROIonosonde(SolarConditionsProvider):
@staticmethod
def _load_stations():
stations = []
with open(STATIONS_INDEX, newline='') as f:
with open(STATIONS_INDEX, newline="") as f:
for row in csv.reader(f):
if len(row) >= 2:
stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
@@ -94,8 +101,14 @@ class GIROIonosonde(SolarConditionsProvider):
# Merge GIRO's readings into any existing data for this station.
existing = ionosonde_data.get(ursi, {})
merged_fof2 = {**{float(t): v for t, v in (existing.get("fof2") or {}).items()}, **fof2}
merged_muf = {**{float(t): v for t, v in (existing.get("muf") or {}).items()}, **muf}
merged_fof2 = {
**{float(t): v for t, v in (existing.get("fof2") or {}).items()},
**fof2,
}
merged_muf = {
**{float(t): v for t, v in (existing.get("muf") or {}).items()},
**muf,
}
merged_luf = dict(luf) if luf else {}
merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts}
@@ -104,7 +117,8 @@ class GIROIonosonde(SolarConditionsProvider):
band_states = compute_band_states(merged_fof2, merged_muf, merged_luf)
ionosonde_data[ursi] = {
"ursi": ursi, "name": name,
"ursi": ursi,
"name": name,
"fof2": merged_fof2 or None,
"muf": merged_muf or None,
"luf": merged_luf or None,
@@ -132,7 +146,7 @@ class GIROIonosonde(SolarConditionsProvider):
return None, None, None
return self._parse_all(http_response.text)
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing Giro ionosonde API.")
logging.warning("Timeout when accessing Giro ionosonde API.")
return None, None, None
except ConnectionError:
logging.warning("Connection error when accessing Giro ionosonde API.")
@@ -147,14 +161,14 @@ class GIROIonosonde(SolarConditionsProvider):
luf_data = {}
for line in text.splitlines():
line = line.strip()
if not line or line.startswith('#'):
if not line or line.startswith("#"):
continue
# Data rows have the following format: timestamp CS foF2 QD MUFD QD fmin QD
parts = line.split()
if len(parts) >= 5:
try:
# Python 3.8 TZ parsing fudge
ts = datetime.fromisoformat(parts[0].replace('Z', '+00:00')).timestamp()
ts = datetime.fromisoformat(parts[0].replace("Z", "+00:00")).timestamp()
except ValueError:
continue
try:
providers/solarconditions/hamqsl.py
+11 -5
View File
@@ -2,9 +2,12 @@ import logging
from xml.etree import ElementTree
import pytz
from dateutil import parser as dateutil_parser, tz as dateutil_tz
from dateutil import parser as dateutil_parser
from dateutil import tz as dateutil_tz
from providers.solarconditions.http_solar_conditions_provider import HTTPSolarConditionsProvider
from providers.solarconditions.http_solar_conditions_provider import (
HTTPSolarConditionsProvider,
)
POLL_INTERVAL = 3600 # 1 hour
URL = "https://www.hamqsl.com/solarxml.php"
@@ -92,7 +95,8 @@ class HamQSL(HTTPSolarConditionsProvider):
"aurora_latitude": float_val("latdegree"),
"solar_wind": float_val("solarwind"),
"magnetic_field": float_val("magneticfield"),
"geomag_field": text("geomagfield").title()
"geomag_field": text("geomagfield")
.title()
.replace("Vr Quiet", "Very Quiet")
.replace("Unsettld", "Unsettled")
.replace("Min Strm", "Minor Storm")
@@ -102,8 +106,10 @@ class HamQSL(HTTPSolarConditionsProvider):
"geomag_noise": text("signalnoise"),
"hf_conditions": hf_conditions,
"vhf_conditions": {
"vhf_aurora_northern_hemi": (vhf_map.get(("vhf-aurora", "northern_hemi")) or "").title().replace(
"Lat Aur", "Latitude") or None,
"vhf_aurora_northern_hemi": (vhf_map.get(("vhf-aurora", "northern_hemi")) or "")
.title()
.replace("Lat Aur", "Latitude")
or None,
"es_2m_europe": vhf_map.get(("E-Skip", "europe")),
"es_4m_europe": vhf_map.get(("E-Skip", "europe_4m")),
"es_6m_europe": vhf_map.get(("E-Skip", "europe_6m")),
providers/solarconditions/http_solar_conditions_provider.py
+3 -4
View File
@@ -1,10 +1,10 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider
@@ -22,8 +22,7 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
self._stop_event = Event()
def start(self):
logging.info(
f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
logging.info(f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}")
self._thread.start()
providers/solarconditions/kc2gprop.py
+4 -4
View File
@@ -1,10 +1,10 @@
import logging
from datetime import datetime, timezone, timedelta
from threading import Thread, Event
from datetime import datetime, timedelta, timezone
from threading import Event, Thread
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from providers.solarconditions.ionosonde_utils import compute_band_states
@@ -119,7 +119,7 @@ class KC2GProp(SolarConditionsProvider):
except ConnectionError:
logging.warning("Connection error when accessing KC2G ionosonde API.")
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing KC2G ionosonde API.")
logging.warning("Timeout when accessing KC2G ionosonde API.")
except Exception:
self.status = "Error"
logging.exception("Exception in KC2G ionosonde data provider")
providers/solarconditions/noaa3dayforecast.py
+22 -12
View File
@@ -2,7 +2,9 @@ import logging
import re
from datetime import datetime, timezone
from providers.solarconditions.http_solar_conditions_provider import HTTPSolarConditionsProvider
from providers.solarconditions.http_solar_conditions_provider import (
HTTPSolarConditionsProvider,
)
POLL_INTERVAL = 10800 # Every 3 hours
URL = "https://services.swpc.noaa.gov/text/3-day-forecast.txt"
@@ -32,13 +34,13 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Find the date header line by scanning the next few lines for month & day patterns
date_header_idx = None
for j in range(start_idx + 1, min(start_idx + 6, len(lines))):
if re.search(r'[A-Za-z]{3}\s+\d{2}', lines[j]):
if re.search(r"[A-Za-z]{3}\s+\d{2}", lines[j]):
date_header_idx = j
break
if date_header_idx is None:
logging.warning(f"NOAA 3-day forecast: could not find date header after '{section_header}'")
return None
date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', lines[date_header_idx])
date_matches = re.findall(r"([A-Za-z]{3})\s+(\d{2})", lines[date_header_idx])
if not date_matches:
logging.warning(f"NOAA 3-day forecast: no dates in header: {lines[date_header_idx]}")
return None
@@ -55,20 +57,20 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Parse data rows. Each non-empty line should have a text label followed by percentage values
result = {}
for line in lines[date_header_idx + 1:]:
for line in lines[date_header_idx + 1 :]:
line_stripped = line.strip()
if not line_stripped:
if result:
break
continue
pct_matches = list(re.finditer(r'\b(\d+)%', line_stripped))
pct_matches = list(re.finditer(r"\b(\d+)%", line_stripped))
if not pct_matches:
if result:
break
continue
# Row label is everything before the first percentage value
row_label = line_stripped[:line_stripped.index(pct_matches[0].group())].strip()
row_label = line_stripped[: line_stripped.index(pct_matches[0].group())].strip()
row_data = {}
for j, match in enumerate(pct_matches):
if j >= len(column_timestamps):
@@ -94,7 +96,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Extract the year from the header line, e.g. "NOAA Kp index breakdown Apr 2-Apr 4, 2026"
header_line = lines[start_idx]
year_match = re.search(r'\b(\d{4})\b', header_line)
year_match = re.search(r"\b(\d{4})\b", header_line)
if not year_match:
logging.warning(f"NOAA K-index forecast: could not extract year from: {header_line}")
return None
@@ -106,7 +108,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
return None
date_header_line = lines[start_idx + 2]
date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', date_header_line)
date_matches = re.findall(r"([A-Za-z]{3})\s+(\d{2})", date_header_line)
if not date_matches:
logging.warning(f"NOAA K-index forecast: could not parse date headers from: {date_header_line}")
return None
@@ -121,8 +123,8 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
# Parse each data row, e.g. "00-03UT 2.00 3.00 2.00"
k_index_forecast = {}
for line in lines[start_idx + 3:]:
time_match = re.match(r'^(\d{2})-(\d{2})UT\s+(.*)', line.strip())
for line in lines[start_idx + 3 :]:
time_match = re.match(r"^(\d{2})-(\d{2})UT\s+(.*)", line.strip())
if not time_match:
if k_index_forecast:
break
@@ -130,7 +132,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
start_hour = int(time_match.group(1))
# Split on 2 or more spaces so that e.g. "5.67 (G2)" stays as one token per column
raw_values = re.split(r' {2,}', time_match.group(3).strip())
raw_values = re.split(r" {2,}", time_match.group(3).strip())
for i, val in enumerate(raw_values):
if i >= len(column_dates):
@@ -142,7 +144,15 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
continue
date = column_dates[i]
start_dt = datetime(date.year, date.month, date.day, start_hour, 0, 0, tzinfo=timezone.utc)
start_dt = datetime(
date.year,
date.month,
date.day,
start_hour,
0,
0,
tzinfo=timezone.utc,
)
# Key the data dict by start time
key = start_dt.timestamp()
providers/solarconditions/solar_conditions_provider.py
+1 -1
View File
@@ -7,7 +7,7 @@ from core.data_store import DATA_STORE
class SolarConditionsProvider:
"""Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
propagation data."""
propagation data."""
def __init__(self, name, provider_config):
"""Constructor"""
providers/spot/aprsis.py
+11 -10
View File
@@ -43,16 +43,17 @@ class APRSIS(SpotProvider):
via_parts = str(data["via"]).split("-")
de_call = via_parts[0].upper()
de_ssid = via_parts[1].upper() if len(via_parts) > 1 else None
spot = Spot(source="APRS-IS",
dx_call=dx_call,
dx_ssid=dx_ssid,
de_call=de_call,
de_ssid=de_ssid,
comment=str(data["comment"]) if "comment" in data else None,
dx_latitude=float(data["latitude"]) if "latitude" in data else None,
dx_longitude=float(data["longitude"]) if "longitude" in data else None,
time=datetime.now(
pytz.UTC).timestamp()) # APRS-IS spots are live so we can assume spot time is "now"
spot = Spot(
source="APRS-IS",
dx_call=dx_call,
dx_ssid=dx_ssid,
de_call=de_call,
de_ssid=de_ssid,
comment=str(data["comment"]) if "comment" in data else None,
dx_latitude=float(data["latitude"]) if "latitude" in data else None,
dx_longitude=float(data["longitude"]) if "longitude" in data else None,
time=datetime.now(pytz.UTC).timestamp(),
) # APRS-IS spots are live so we can assume spot time is "now"
# Add to our list
self._submit(spot)
providers/spot/dxcluster.py
+23 -13
View File
@@ -18,10 +18,12 @@ class DXCluster(SpotProvider):
_LINE_PATTERN_EXCLUDE_RBN = re.compile(
r"^DX de ([a-z0-9/]+):\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE)
re.IGNORECASE,
)
_LINE_PATTERN_ALLOW_RBN = re.compile(
r"^DX de ([a-z0-9/]+)-?#?:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE)
re.IGNORECASE,
)
def __init__(self, provider_config):
"""Constructor requires hostname and port"""
@@ -31,10 +33,13 @@ class DXCluster(SpotProvider):
self._hostname = provider_config["host"]
self._port = provider_config["port"]
self._login_prompt = provider_config["login_prompt"] if "login_prompt" in provider_config else "login:"
self._login_callsign = provider_config[
"login_callsign"] if "login_callsign" in provider_config else SERVER_OWNER_CALLSIGN
self._login_callsign = (
provider_config["login_callsign"] if "login_callsign" in provider_config else SERVER_OWNER_CALLSIGN
)
self._allow_rbn_spots = provider_config["allow_rbn_spots"] if "allow_rbn_spots" in provider_config else False
self._spot_line_pattern = self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
self._spot_line_pattern = (
self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
)
self._telnet = None
self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}")
self._thread.daemon = True
@@ -77,14 +82,19 @@ class DXCluster(SpotProvider):
match = self._spot_line_pattern.match(telnet_output.decode("latin-1"))
if match:
spot_time = datetime.strptime(match.group(5), "%H%MZ")
spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(),
tzinfo=pytz.UTC)
spot = Spot(source=self.name,
dx_call=match.group(3),
de_call=match.group(1),
freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(),
time=spot_datetime.timestamp())
spot_datetime = datetime.combine(
datetime.now(pytz.UTC).date(),
spot_time.time(),
tzinfo=pytz.UTC,
)
spot = Spot(
source=self.name,
dx_call=match.group(3),
de_call=match.group(1),
freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(),
time=spot_datetime.timestamp(),
)
# Add to our list
self._submit(spot)
providers/spot/gma.py
+64 -29
View File
@@ -27,7 +27,12 @@ class GMA(HTTPSpotProvider):
logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
self._url_data_cache = URLDataCache("GMA")
super().__init__("GMA", provider_config, f"{self.SPOTS_URL}?key={self._api_key}", self.POLL_INTERVAL_SEC)
super().__init__(
"GMA",
provider_config,
f"{self.SPOTS_URL}?key={self._api_key}",
self.POLL_INTERVAL_SEC,
)
def _http_response_to_spots(self, http_response):
new_spots = []
@@ -41,43 +46,67 @@ class GMA(HTTPSpotProvider):
# Seen some real janky times from GMA, if we don't understand it just ignore this spot
try:
time = datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
tzinfo=pytz.UTC).timestamp()
time = (
datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M")
.replace(tzinfo=pytz.UTC)
.timestamp()
)
except ValueError:
continue
spot = Spot(source=self.name,
dx_call=source_spot["ACTIVATOR"].upper(),
de_call=source_spot["SPOTTER"].upper(),
# Seen GMA spots with no frequency or with "QRT" in this field
freq=float(source_spot["QRG"]) * 1000 if (
source_spot["QRG"] != "" and source_spot["QRG"] != "QRT") else None,
# Filter out some weird mode strings
mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
comment=source_spot["TEXT"],
sig_refs=[SIGRef(id=source_spot["REF"], sig="", name=source_spot["NAME"], latitude=lat,
longitude=lon)],
time=time,
dx_latitude=lat,
dx_longitude=lon,
qrt=source_spot["QRG"] == "QRT")
spot = Spot(
source=self.name,
dx_call=source_spot["ACTIVATOR"].upper(),
de_call=source_spot["SPOTTER"].upper(),
# Seen GMA spots with no frequency or with "QRT" in this field
freq=float(source_spot["QRG"]) * 1000
if (source_spot["QRG"] != "" and source_spot["QRG"] != "QRT")
else None,
# Filter out some weird mode strings
mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
comment=source_spot["TEXT"],
sig_refs=[
SIGRef(
id=source_spot["REF"],
sig="",
name=source_spot["NAME"],
latitude=lat,
longitude=lon,
)
],
time=time,
dx_latitude=lat,
dx_longitude=lon,
qrt=source_spot["QRG"] == "QRT",
)
# GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
if "REF" in source_spot:
try:
ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
headers=HTTP_HEADERS)
ref_response = self._url_data_cache.get(
self.REF_INFO_URL_ROOT + source_spot["REF"],
headers=HTTP_HEADERS,
)
# Sometimes this is blank even if it's a 200 response, so handle that
if ref_response.ok and ref_response.text is not None and ref_response.text != "" and ref_response.text != "\n":
if (
ref_response.ok
and ref_response.text is not None
and ref_response.text != ""
and ref_response.text != "\n"
):
ref_info = ref_response.json()
# If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
# spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
# and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry
# to determine if it's a SOTA summit.
if spot.sig_refs and "reftype" in ref_info and ref_info["reftype"] not in ["POTA",
"WWFF"] and (
ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info[
"sota"] == ""):
if (
spot.sig_refs
and "reftype" in ref_info
and ref_info["reftype"] not in ["POTA", "WWFF"]
and (
ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info["sota"] == ""
)
):
match ref_info["reftype"]:
case "Summit":
spot.sig_refs[0].sig = "GMA"
@@ -98,7 +127,9 @@ class GMA(HTTPSpotProvider):
spot.sig_refs[0].sig = "MOTA"
spot.sig = "MOTA"
case _:
logging.warning(f"GMA spot found with ref type {ref_info['reftype']}, developer needs to add support for this!")
logging.warning(
f"GMA spot found with ref type {ref_info['reftype']}, developer needs to add support for this!"
)
spot.sig_refs[0].sig = ref_info["reftype"]
spot.sig = ref_info["reftype"]
@@ -109,12 +140,16 @@ class GMA(HTTPSpotProvider):
elif not ref_response.from_cache:
if not ref_response.ok:
logging.warning(
f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot['REF']}")
f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot['REF']}"
)
else:
logging.warning(
f"GMA API returned a malformed response when looking up ref {source_spot['REF']}")
f"GMA API returned a malformed response when looking up ref {source_spot['REF']}"
)
except:
logging.exception(f"Exception when looking up {self.REF_INFO_URL_ROOT}{source_spot['REF']}, ignoring this spot for now")
logging.exception(
f"Exception when looking up {self.REF_INFO_URL_ROOT}{source_spot['REF']}, ignoring this spot for now"
)
else:
logging.warning(f"The GMA API returned an unexpected response (HTTP {http_response.status_code}).")
providers/spot/hema.py
+24 -14
View File
@@ -4,7 +4,7 @@ from datetime import datetime
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from data.sig_ref import SIGRef
@@ -52,25 +52,35 @@ class HEMA(HTTPSpotProvider):
continue
# Convert to our spot format
spot = Spot(source=self.name,
dx_call=spot_items[2].upper(),
de_call=spotter_comment_match.group(1).upper(),
freq=float(freq_mode_match.group(1)) * 1000000,
mode=freq_mode_match.group(2).upper(),
comment=spotter_comment_match.group(2),
spot = Spot(
source=self.name,
dx_call=spot_items[2].upper(),
de_call=spotter_comment_match.group(1).upper(),
freq=float(freq_mode_match.group(1)) * 1000000,
mode=freq_mode_match.group(2).upper(),
comment=spotter_comment_match.group(2),
sig="HEMA",
sig_refs=[
SIGRef(
id=spot_items[3].upper(),
sig="HEMA",
sig_refs=[SIGRef(id=spot_items[3].upper(), sig="HEMA", name=spot_items[4],
latitude=float(spot_items[7]), longitude=float(spot_items[8]))],
time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(
tzinfo=pytz.UTC).timestamp(),
dx_latitude=float(spot_items[7]),
dx_longitude=float(spot_items[8]))
name=spot_items[4],
latitude=float(spot_items[7]),
longitude=float(spot_items[8]),
)
],
time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M")
.replace(tzinfo=pytz.UTC)
.timestamp(),
dx_latitude=float(spot_items[7]),
dx_longitude=float(spot_items[8]),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other
# code will do that for us.
new_spots.append(spot)
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing HEMA spots API.")
logging.warning("Timeout when accessing HEMA spots API.")
except ConnectionError:
logging.warning("Connection error when accessing HEMA spots API.")
return new_spots
providers/spot/http_spot_provider.py
+1 -1
View File
@@ -1,6 +1,6 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
import requests
providers/spot/llota.py
+17 -9
View File
@@ -25,16 +25,24 @@ class LLOTA(HTTPSpotProvider):
comment = str(source_spot["history"][-1]["comment"])
spotter = str(source_spot["history"][-1]["spotter_callsign"])
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["callsign"].upper(),
de_call=spotter.upper() if spotter else None,
freq=float(source_spot["frequency"]) * 1000000,
mode=source_spot["mode"].upper(),
comment=comment,
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["callsign"].upper(),
de_call=spotter.upper() if spotter else None,
freq=float(source_spot["frequency"]) * 1000000,
mode=source_spot["mode"].upper(),
comment=comment,
sig="LLOTA",
sig_refs=[
SIGRef(
id=source_spot["reference"],
sig="LLOTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="LLOTA", name=source_spot["reference_name"])],
time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp())
name=source_spot["reference_name"],
)
],
time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp(),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
providers/spot/parksnpeaks.py
+46 -16
View File
@@ -18,7 +18,17 @@ class ParksNPeaks(HTTPSpotProvider):
SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
SUBMIT_URL = "https://www.parksnpeaks.org/api/SPOT/"
SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
SUBMITTABLE_SIGS = ["POTA", "SOTA", "WWFF", "HEMA", "WOTA", "ZLOTA", "SIOTA", "KRMNPA", "SANPCPA"]
SUBMITTABLE_SIGS = [
"POTA",
"SOTA",
"WWFF",
"HEMA",
"WOTA",
"ZLOTA",
"SIOTA",
"KRMNPA",
"SANPCPA",
]
def __init__(self, provider_config):
super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -29,18 +39,23 @@ class ParksNPeaks(HTTPSpotProvider):
if http_response and http_response != "":
for source_spot in http_response.json():
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["actID"],
dx_call=source_spot["actCallsign"].upper(),
de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
# typo exists in API
freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "")
.replace("+/-", "").strip()) * 1000000 if (source_spot["actFreq"] != "") else None,
# Seen PNP spots with empty frequency, and with comma-separated thousands digits
mode=source_spot["actMode"].upper(),
comment=source_spot["actComments"],
time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp())
spot = Spot(
source=self.name,
source_id=source_spot["actID"],
dx_call=source_spot["actCallsign"].upper(),
de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
# typo exists in API
freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "").replace("+/-", "").strip())
* 1000000
if (source_spot["actFreq"] != "")
else None,
# Seen PNP spots with empty frequency, and with comma-separated thousands digits
mode=source_spot["actMode"].upper(),
comment=source_spot["actComments"],
time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
)
# Extract a de_call if it's in the comment but not in the "actSpoter" field
m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment or "")
@@ -53,7 +68,12 @@ class ParksNPeaks(HTTPSpotProvider):
sig_ref = source_spot["actSiteID"]
if sig and sig != "" and sig != "QRP" and sig_ref and sig_ref != "":
spot.sig = sig
sig_refs = [SIGRef(id=source_spot["actSiteID"], sig=source_spot["actClass"].upper())]
sig_refs = [
SIGRef(
id=source_spot["actSiteID"],
sig=source_spot["actClass"].upper(),
)
]
spot.sig_refs = sig_refs
# Free text location is not present in all spots, so only add it if it's set
@@ -61,7 +81,16 @@ class ParksNPeaks(HTTPSpotProvider):
sig_refs[0].name = source_spot["actLocation"]
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "SANPCPA", "LLOTA"]:
if sig not in [
"POTA",
"SOTA",
"WWFF",
"SIOTA",
"ZLOTA",
"KRMNPA",
"SANPCPA",
"LLOTA",
]:
logging.warning(f"PNP spot found with sig {sig}, developer needs to add support for this!")
# Add new spot to the list
@@ -77,7 +106,8 @@ class ParksNPeaks(HTTPSpotProvider):
api_key = credentials.get("api_key", "")
if not user_id or not api_key:
raise ValueError(
"Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account.")
"Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account."
)
sig_ref = spot.sig_refs[0].id if spot.sig_refs else ""
body = {
"actClass": spot.sig or "",
providers/spot/pota.py
+24 -14
View File
@@ -24,21 +24,31 @@ class POTA(HTTPSpotProvider):
# Iterate through source data
for source_spot in http_response.json():
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["spotId"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["frequency"]) * 1000,
mode=source_spot["mode"].upper(),
comment=source_spot["comments"],
spot = Spot(
source=self.name,
source_id=source_spot["spotId"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["frequency"]) * 1000,
mode=source_spot["mode"].upper(),
comment=source_spot["comments"],
sig="POTA",
sig_refs=[
SIGRef(
id=source_spot["reference"],
sig="POTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="POTA", name=source_spot["name"],
latitude=source_spot["latitude"], longitude=source_spot["longitude"])],
time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp(),
dx_grid=source_spot["grid6"],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"])
name=source_spot["name"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
)
],
time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
dx_grid=source_spot["grid6"],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
providers/spot/rbn.py
+15 -9
View File
@@ -18,7 +18,8 @@ class RBN(SpotProvider):
_LINE_PATTERN = re.compile(
r"^DX de ([a-z0-9/]+)-.*:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE)
re.IGNORECASE,
)
def __init__(self, provider_config):
"""Constructor requires port number."""
@@ -64,14 +65,19 @@ class RBN(SpotProvider):
match = self._LINE_PATTERN.match(telnet_output.decode("latin-1"))
if match:
spot_time = datetime.strptime(match.group(5), "%H%MZ")
spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(),
tzinfo=pytz.UTC)
spot = Spot(source=self.name,
dx_call=match.group(3),
de_call=match.group(1),
freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(),
time=spot_datetime.timestamp())
spot_datetime = datetime.combine(
datetime.now(pytz.UTC).date(),
spot_time.time(),
tzinfo=pytz.UTC,
)
spot = Spot(
source=self.name,
dx_call=match.group(3),
de_call=match.group(1),
freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(),
time=spot_datetime.timestamp(),
)
# Add to our list
self._submit(spot)
providers/spot/sota.py
+36 -23
View File
@@ -2,9 +2,9 @@ import logging
from datetime import datetime
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS, SSB_SUB_MODES, DV_SUB_MODES
from core.constants import DV_SUB_MODES, HTTP_HEADERS, SSB_SUB_MODES
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -41,24 +41,33 @@ class SOTA(HTTPSpotProvider):
# Iterate through source data
for source_spot in source_data:
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activatorCallsign"].upper(),
dx_name=source_spot["activatorName"],
de_call=source_spot["callsign"].upper(),
freq=(float(source_spot["frequency"]) * 1000000) if (
source_spot["frequency"] is not None) else None,
# Seen SOTA spots with no frequency!
mode=source_spot["mode"].upper(),
comment=source_spot["comments"],
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activatorCallsign"].upper(),
dx_name=source_spot["activatorName"],
de_call=source_spot["callsign"].upper(),
freq=(float(source_spot["frequency"]) * 1000000)
if (source_spot["frequency"] is not None)
else None,
# Seen SOTA spots with no frequency!
mode=source_spot["mode"].upper(),
comment=source_spot["comments"],
sig="SOTA",
sig_refs=[
SIGRef(
id=source_spot["summitCode"],
sig="SOTA",
sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA",
name=source_spot["summitName"], latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
activation_score=source_spot["points"])],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp())
name=source_spot["summitName"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
activation_score=source_spot["points"],
)
],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp(),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
@@ -66,7 +75,7 @@ class SOTA(HTTPSpotProvider):
except ConnectionError:
logging.warning("Connection error when accessing SOTA spots API")
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing SOTA spots API.")
logging.warning("Timeout when accessing SOTA spots API.")
return new_spots
def can_submit_spot(self, sig):
@@ -102,10 +111,14 @@ class SOTA(HTTPSpotProvider):
"mode": mode or "",
"callsign": spot.de_call,
"comments": spot.comment or "",
"type": "TEST" # todo replatce with NORMAL/QRT once testing complete
"type": "TEST", # todo replatce with NORMAL/QRT once testing complete
}
headers = {
**HTTP_HEADERS,
"Authorization": f"bearer {access_token}",
"id_token": id_token,
"Content-Type": "application/json",
}
headers = {**HTTP_HEADERS, "Authorization": f"bearer {access_token}", "id_token": id_token,
"Content-Type": "application/json"}
response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
if not response.ok:
raise RuntimeError(f"SOTA API returned {response.status_code!s}: {response.text}")
providers/spot/spot_provider.py
+1 -1
View File
@@ -32,7 +32,7 @@ class SpotProvider:
# Sort the batch so that earliest ones go in first. This helps keep the ordering correct when spots are fired
# off to SSE listeners.
spots = sorted(spots, key=lambda s: (s.time if s and s.time else 0))
spots = sorted(spots, key=lambda s: s.time if s and s.time else 0)
for spot in spots:
if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time:
# Fill in any blanks and add to the list
providers/spot/sse_spot_provider.py
+13 -7
View File
@@ -37,8 +37,7 @@ class SSESpotProvider(SpotProvider):
try:
event_source.close()
except Exception:
logging.exception(
f"Exception closing SSE connection for {self.name} during stop()")
logging.exception(f"Exception closing SSE connection for {self.name} during stop()")
if self._thread:
self._thread.join(timeout=15)
@@ -60,14 +59,20 @@ class SSESpotProvider(SpotProvider):
try:
logging.debug(f"Connecting to {self.name} spot API...")
self.status = "Connecting"
with EventSource(self._url, headers=HTTP_HEADERS, latest_event_id=self._last_event_id, timeout=10,
on_open=self._on_open, on_error=self._on_error) as event_source:
with EventSource(
self._url,
headers=HTTP_HEADERS,
latest_event_id=self._last_event_id,
timeout=10,
on_open=self._on_open,
on_error=self._on_error,
) as event_source:
self._set_event_source(event_source)
try:
for event in event_source:
if self._stop_event.is_set():
break
if event.type == 'message':
if event.type == "message":
try:
self._last_event_id = event.last_event_id
new_spot = self._sse_message_to_spot(event.data)
@@ -80,7 +85,8 @@ class SSESpotProvider(SpotProvider):
except Exception:
logging.exception(
f"Exception processing message from SSE Spot Provider ({self.name})")
f"Exception processing message from SSE Spot Provider ({self.name})"
)
finally:
self._set_event_source(None)
@@ -89,7 +95,7 @@ class SSESpotProvider(SpotProvider):
logging.exception(f"Exception in SSE Spot Provider ({self.name})")
else:
self.status = "Disconnected"
self._stop_event.wait(timeout=5) # Wait before trying to reconnect
self._stop_event.wait(timeout=5) # Wait before trying to reconnect
def _sse_message_to_spot(self, message_data):
"""Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass
providers/spot/tiles.py
+47 -26
View File
@@ -14,8 +14,22 @@ class Tiles(HTTPSpotProvider):
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/spots?active_hours=24"
SUBMIT_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/self-spot"
VALID_MODES = ["SSB", "CW", "FT8", "FT4", "FM", "DMR", "D-STAR", "M17", "AX.25", "JS8Call", "PSK31", "Olivia",
"VarAC", "Other"]
VALID_MODES = [
"SSB",
"CW",
"FT8",
"FT4",
"FM",
"DMR",
"D-STAR",
"M17",
"AX.25",
"JS8Call",
"PSK31",
"Olivia",
"VarAC",
"Other",
]
def __init__(self, provider_config):
super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -25,25 +39,33 @@ class Tiles(HTTPSpotProvider):
# Iterate through source data
for source_spot in http_response.json()["spots"]:
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["call_sign"].upper(),
# No separate spotter callsign, assume all spots are self-spots
de_call=source_spot["call_sign"].upper(),
freq=float(strip_extra_decimal_points(source_spot["frequency"])) * 1000000,
mode=source_spot["mode"].upper(),
comment=source_spot["notes"],
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["call_sign"].upper(),
# No separate spotter callsign, assume all spots are self-spots
de_call=source_spot["call_sign"].upper(),
freq=float(strip_extra_decimal_points(source_spot["frequency"])) * 1000000,
mode=source_spot["mode"].upper(),
comment=source_spot["notes"],
sig="Tiles",
# Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
# Just take the grid reference itself as the single Tiles SIG reference.
sig_refs=[
SIGRef(
id=source_spot["maidenhead_grid"],
sig="Tiles",
# Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
# Just take the grid reference itself as the single Tiles SIG reference.
sig_refs=[SIGRef(id=source_spot["maidenhead_grid"], sig="Tiles",
name=source_spot["maidenhead_grid"], latitude=source_spot["latitude"],
longitude=source_spot["longitude"])],
time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
dx_grid=source_spot["maidenhead_grid"],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
dx_location_source="GRID")
name=source_spot["maidenhead_grid"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
)
],
time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
dx_grid=source_spot["maidenhead_grid"],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
dx_location_source="GRID",
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
@@ -56,7 +78,6 @@ class Tiles(HTTPSpotProvider):
def submit_spot(self, spot, credentials):
# Tiles on the air currently only supports *self* spots
if spot.dx_call == spot.de_call:
# Figure out a valid mode. Borrowed this from PoLo :)
# https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js
if spot.mode:
@@ -80,24 +101,24 @@ class Tiles(HTTPSpotProvider):
"lat": spot.dx_latitude or None,
"lon": spot.dx_longitude or None,
"qrt": spot.qrt or False,
"pin": credentials.get("offline_spot_gateway_pin", "")
"pin": credentials.get("offline_spot_gateway_pin", ""),
}
headers = {**HTTP_HEADERS, "Content-Type": "application/json"}
response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
if not response.ok:
raise RuntimeError(
f"Tiles on the Air API returned {response.status_code!s}: {response.text}")
raise RuntimeError(f"Tiles on the Air API returned {response.status_code!s}: {response.text}")
else:
raise RuntimeError("The Tiles on the Air API requires a mode to be set.")
else:
raise RuntimeError(
"The Tiles on the Air API only supports self-spots, the DX call and spotter call must match.")
"The Tiles on the Air API only supports self-spots, the DX call and spotter call must match."
)
# Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles'
# strange frequency format where we can sometimes have e.g. "14.123.5".
def strip_extra_decimal_points(s):
parts = s.split('.', 1)
parts = s.split(".", 1)
if len(parts) == 1:
return s
return f"{parts[0]}.{parts[1].replace('.', '')}"
providers/spot/towers.py
+11 -8
View File
@@ -26,14 +26,17 @@ class Towers(HTTPSpotProvider):
likely_freq = float(source_spot["freq"]) * 1000
if likely_freq < 1000000:
likely_freq = likely_freq * 1000
spot = Spot(source=self.name,
dx_call=source_spot["call"].upper(),
freq=likely_freq,
comment=source_spot["comment"],
sig="Towers",
sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
time=datetime.strptime(response_json["updated"][:10] + source_spot["time"],
"%Y-%m-%d%H:%M").timestamp())
spot = Spot(
source=self.name,
dx_call=source_spot["call"].upper(),
freq=likely_freq,
comment=source_spot["comment"],
sig="Towers",
sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
time=datetime.strptime(
response_json["updated"][:10] + source_spot["time"], "%Y-%m-%d%H:%M"
).timestamp(),
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
providers/spot/ukpacketnet.py
+38 -23
View File
@@ -36,42 +36,56 @@ class UKPacketNet(HTTPSpotProvider):
# First build a "full" comment combining some of the extra info
comment = listed_port["comment"] if "comment" in listed_port else ""
comment = f"{comment} {listed_port['mode']}" if "mode" in listed_port else comment
comment = f"{comment} {listed_port['modulation']}" if "modulation" in listed_port else comment
comment = f"{comment} {listed_port['baud']!s} baud" if "baud" in listed_port and listed_port[
"baud"] > 0 else comment
comment = (
f"{comment} {listed_port['modulation']}" if "modulation" in listed_port else comment
)
comment = (
f"{comment} {listed_port['baud']!s} baud"
if "baud" in listed_port and listed_port["baud"] > 0
else comment
)
# Get frequency from the comment if it's not set properly in the data structure. This is
# very hacky but a lot of node comments contain their frequency as the first or second
# word of their comment, but not in the proper data structure field.
freq = listed_port["freq"] if "freq" in listed_port and listed_port[
"freq"] > 0 else None
freq = (
listed_port["freq"] if "freq" in listed_port and listed_port["freq"] > 0 else None
)
if not freq and comment:
possible_freq = comment.split(" ")[0].upper().replace("MHZ", "")
if re.match(r"^[0-9.]+$",
possible_freq) and possible_freq != "1200" and possible_freq != "9600":
if (
re.match(r"^[0-9.]+$", possible_freq)
and possible_freq != "1200"
and possible_freq != "9600"
):
freq = float(possible_freq) * 1000000
if not freq and len(comment.split(" ")) > 1:
possible_freq = comment.split(" ")[1].upper().replace("MHZ", "")
if re.match(r"^[0-9.]+$",
possible_freq) and possible_freq != "1200" and possible_freq != "9600":
if (
re.match(r"^[0-9.]+$", possible_freq)
and possible_freq != "1200"
and possible_freq != "9600"
):
freq = float(possible_freq) * 1000000
# Check for a found frequency likely having been in kHz, sorry to all GHz packet folks
if freq and freq > 1000000000:
freq = freq / 1000
# Now build the spot object
spot = Spot(source=self.name,
dx_call=heard["callsign"].upper(),
de_call=node["callsign"].upper(),
freq=freq,
mode="PKT",
comment=comment,
time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp(),
de_grid=node["location"]["locator"] if "locator" in node[
"location"] else None,
de_latitude=node["location"]["coords"]["lat"],
de_longitude=node["location"]["coords"]["lon"])
spot = Spot(
source=self.name,
dx_call=heard["callsign"].upper(),
de_call=node["callsign"].upper(),
freq=freq,
mode="PKT",
comment=comment,
time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
de_grid=node["location"]["locator"] if "locator" in node["location"] else None,
de_latitude=node["location"]["coords"]["lat"],
de_longitude=node["location"]["coords"]["lon"],
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
@@ -84,8 +98,9 @@ class UKPacketNet(HTTPSpotProvider):
# data, and we can use that to look these up.
for spot in new_spots:
if spot.dx_call in nodes:
spot.dx_grid = nodes[spot.dx_call]["location"]["locator"] if "locator" in nodes[spot.dx_call][
"location"] else None
spot.dx_grid = (
nodes[spot.dx_call]["location"]["locator"] if "locator" in nodes[spot.dx_call]["location"] else None
)
spot.dx_latitude = nodes[spot.dx_call]["location"]["coords"]["lat"]
spot.dx_longitude = nodes[spot.dx_call]["location"]["coords"]["lon"]
providers/spot/websocket_spot_provider.py
+1 -2
View File
@@ -60,8 +60,7 @@ class WebsocketSpotProvider(SpotProvider):
logging.debug(f"Received data from {self.name} spot API.")
except Exception:
logging.exception(
f"Exception processing message from Websocket Spot Provider ({self.name})")
logging.exception(f"Exception processing message from Websocket Spot Provider ({self.name})")
except Exception as e:
self.status = "Error"
providers/spot/wota.py
+18 -13
View File
@@ -28,10 +28,13 @@ class WOTA(HTTPSpotProvider):
rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
# Iterate through source data
for source_spot in rss.channel.items:
try:
# Reject GUID missing or zero
if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0":
if (
not source_spot.guid
or not source_spot.guid.content
or source_spot.guid.content == "http://www.wota.org.uk/spots/0"
):
continue
# Pick apart the title
@@ -48,7 +51,7 @@ class WOTA(HTTPSpotProvider):
# Pick apart the description
desc_split = source_spot.description.split(". ")
freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
freq_mode_split = re.split(r'[\-\s]+', freq_mode)
freq_mode_split = re.split(r"[\-\s]+", freq_mode)
freq_hz = float(freq_mode_split[0].replace("'", ".")) * 1000000
mode = None
if len(freq_mode_split) > 1:
@@ -64,16 +67,18 @@ class WOTA(HTTPSpotProvider):
time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot.guid.content,
dx_call=dx_call,
de_call=spotter,
freq=freq_hz,
mode=mode,
comment=comment,
sig="WOTA",
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
time=time.timestamp())
spot = Spot(
source=self.name,
source_id=source_spot.guid.content,
dx_call=dx_call,
de_call=spotter,
freq=freq_hz,
mode=mode,
comment=comment,
sig="WOTA",
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
time=time.timestamp(),
)
new_spots.append(spot)
except Exception as e:
providers/spot/wwbota.py
+24 -17
View File
@@ -20,25 +20,32 @@ class WWBOTA(SSESpotProvider):
# n-fer activations.
refs = []
for ref in source_spot["references"]:
sigref = SIGRef(id=ref["reference"], sig="WWBOTA", name=ref["name"], latitude=ref["lat"],
longitude=ref["long"])
sigref = SIGRef(
id=ref["reference"],
sig="WWBOTA",
name=ref["name"],
latitude=ref["lat"],
longitude=ref["long"],
)
refs.append(sigref)
spot = Spot(source=self.name,
dx_call=source_spot["call"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["freq"]) * 1000000,
mode=source_spot["mode"].upper(),
comment=source_spot["comment"],
sig="WWBOTA",
sig_refs=refs,
time=datetime.fromisoformat(source_spot["time"].replace("Z", "+00:00")).timestamp(),
# WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
# now, we will just pick the first one to use as our grid, latitude and longitude.
dx_grid=source_spot["references"][0]["locator"],
dx_latitude=source_spot["references"][0]["lat"],
dx_longitude=source_spot["references"][0]["long"],
qrt=source_spot["type"] == "QRT")
spot = Spot(
source=self.name,
dx_call=source_spot["call"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["freq"]) * 1000000,
mode=source_spot["mode"].upper(),
comment=source_spot["comment"],
sig="WWBOTA",
sig_refs=refs,
time=datetime.fromisoformat(source_spot["time"].replace("Z", "+00:00")).timestamp(),
# WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
# now, we will just pick the first one to use as our grid, latitude and longitude.
dx_grid=source_spot["references"][0]["locator"],
dx_latitude=source_spot["references"][0]["lat"],
dx_longitude=source_spot["references"][0]["long"],
qrt=source_spot["type"] == "QRT",
)
# WWBOTA does support a special "Test" spot type, we need to avoid adding that.
return spot if source_spot["type"] != "Test" else None
providers/spot/wwff.py
+21 -12
View File
@@ -21,19 +21,28 @@ class WWFF(HTTPSpotProvider):
# Iterate through source data
for source_spot in http_response.json():
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["frequency_khz"]) * 1000,
mode=source_spot["mode"].upper(),
comment=source_spot["remarks"],
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["frequency_khz"]) * 1000,
mode=source_spot["mode"].upper(),
comment=source_spot["remarks"],
sig="WWFF",
sig_refs=[
SIGRef(
id=source_spot["reference"],
sig="WWFF",
sig_refs=[SIGRef(id=source_spot["reference"], sig="WWFF", name=source_spot["reference_name"],
latitude=source_spot["latitude"], longitude=source_spot["longitude"])],
time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"])
name=source_spot["reference_name"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
)
],
time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"],
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
providers/spot/xota.py
+17 -10
View File
@@ -29,14 +29,21 @@ class XOTA(WebsocketSpotProvider):
string = b.decode("utf-8")
source_spot = json.loads(string)
ref_id = f"{self._sig_ref_prefix} {source_spot['reference']['title']}"
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["stationCallSign"].upper(),
freq=float(source_spot["freq"]) * 1000,
mode=source_spot["mode"].upper(),
sig=self.SIG,
sig_refs=[
SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"])],
time=datetime.now(pytz.UTC).timestamp(),
qrt=source_spot["state"] != "active")
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["stationCallSign"].upper(),
freq=float(source_spot["freq"]) * 1000,
mode=source_spot["mode"].upper(),
sig=self.SIG,
sig_refs=[
SIGRef(
id=ref_id,
sig=self.SIG or "",
url=source_spot["reference"]["website"],
)
],
time=datetime.now(pytz.UTC).timestamp(),
qrt=source_spot["state"] != "active",
)
return spot
providers/spot/zlota.py
+19 -10
View File
@@ -26,17 +26,26 @@ class ZLOTA(HTTPSpotProvider):
freq_hz = freq_hz * 1000
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=freq_hz,
mode=source_spot["mode"].upper().strip(),
comment=source_spot["comments"],
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=freq_hz,
mode=source_spot["mode"].upper().strip(),
comment=source_spot["comments"],
sig="ZLOTA",
sig_refs=[
SIGRef(
id=source_spot["reference"],
sig="ZLOTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="ZLOTA", name=source_spot["name"])],
time=datetime.fromisoformat(source_spot["referenced_time"].replace("Z", "+00:00")).astimezone(
pytz.UTC).timestamp())
name=source_spot["name"],
)
],
time=datetime.fromisoformat(source_spot["referenced_time"].replace("Z", "+00:00"))
.astimezone(pytz.UTC)
.timestamp(),
)
new_spots.append(spot)
return new_spots
providers/staticdata/cqzonedata.py
+4 -2
View File
@@ -5,7 +5,9 @@ import geopandas
from shapely import prepare
from core.data_store import DATA_STORE
from providers.staticdata.local_file_static_data_provider import LocalFileStaticDataProvider
from providers.staticdata.local_file_static_data_provider import (
LocalFileStaticDataProvider,
)
class CQZoneData(LocalFileStaticDataProvider):
@@ -21,7 +23,7 @@ class CQZoneData(LocalFileStaticDataProvider):
with open(path) as f:
cq_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
for idx in cq_zone_data.index:
prepare(cq_zone_data.at[idx, 'geometry'])
prepare(cq_zone_data.at[idx, "geometry"])
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to prepare the rest
# of the data in this case
providers/staticdata/file_download_static_data_provider.py
+8 -7
View File
@@ -1,6 +1,6 @@
import logging
from datetime import datetime
from threading import Thread, Event
from threading import Event, Thread
import pytz
from requests import ReadTimeout
@@ -13,10 +13,10 @@ from providers.staticdata.static_data_provider import StaticDataProvider
class FileDownloadStaticDataProvider(StaticDataProvider):
"""Generic static reference data provider class for providers that fetch their data from the web by downloading a
file."""
file."""
def __init__(self, name, provider_config, url, poll_interval):
""" Set up the provider, note poll_interval is in *days*."""
"""Set up the provider, note poll_interval is in *days*."""
super().__init__(name, provider_config)
self._url = url
self._poll_interval = poll_interval
@@ -27,8 +27,7 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting.
logging.info(
f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.")
logging.info(f"Set up query of {self.name} static reference data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}")
self._thread.start()
@@ -57,7 +56,9 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
logging.info(f"Updated static reference data for {self.name}")
else:
self.status = "Error"
logging.warning(f"HTTP {http_response.status_code} when downloading static reference data for {self.name}.")
logging.warning(
f"HTTP {http_response.status_code} when downloading static reference data for {self.name}."
)
except ConnectionError:
self.status = "Error"
@@ -72,6 +73,6 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
def _handle_http_response(self, http_response):
"""Handle an HTTP response returned by the server and load the data from it. Return true if successful,
false otherwise."""
false otherwise."""
raise NotImplementedError("Subclasses must implement this method")
providers/staticdata/ituzonedata.py
+4 -2
View File
@@ -5,7 +5,9 @@ import geopandas
from shapely import prepare
from core.data_store import DATA_STORE
from providers.staticdata.local_file_static_data_provider import LocalFileStaticDataProvider
from providers.staticdata.local_file_static_data_provider import (
LocalFileStaticDataProvider,
)
class ITUZoneData(LocalFileStaticDataProvider):
@@ -21,7 +23,7 @@ class ITUZoneData(LocalFileStaticDataProvider):
with open(path) as f:
itu_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
for idx in itu_zone_data.index:
prepare(itu_zone_data.at[idx, 'geometry'])
prepare(itu_zone_data.at[idx, "geometry"])
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to prepare the rest
# of the data in this case
providers/staticdata/k0swe.py
+3 -3
View File
@@ -1,7 +1,9 @@
import logging
from core.data_store import DATA_STORE
from providers.staticdata.file_download_static_data_provider import FileDownloadStaticDataProvider
from providers.staticdata.file_download_static_data_provider import (
FileDownloadStaticDataProvider,
)
class K0SWE(FileDownloadStaticDataProvider):
@@ -44,5 +46,3 @@ class K0SWE(FileDownloadStaticDataProvider):
except Exception:
logging.exception("Exception when loading K0SWE dxcc.json.")
return False
providers/staticdata/static_data_provider.py
-2
View File
@@ -15,13 +15,11 @@ class StaticDataProvider:
self.status = "Not Started" if self.enabled else "Disabled"
self.reference_count = 0
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
raise NotImplementedError("Subclasses must implement this method")
def stop(self):
"""Stop any threads and prepare for application shutdown"""