w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 02:21:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 9 #118

Browse Source
This commit is contained in:
Ian Renton
2026-08-02 09:06:10 +01:00
parent 0b0c8aa4f3
commit 2157bf114e
72 changed files with 159 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
providers/spot/aprsis.py
+62
View File
@@ -0,0 +1,62 @@
import logging
from datetime import datetime
from threading import Thread
import aprslib
import pytz
from core.config import SERVER_OWNER_CALLSIGN
from data.spot import Spot
from providers.spot.spot_provider import SpotProvider
class APRSIS(SpotProvider):
"""Spot provider for the APRS-IS."""
def __init__(self, provider_config):
super().__init__("APRS-IS", provider_config)
self._thread = Thread(target=self._connect)
self._thread.daemon = True
self._aprsis = None
def start(self):
self._thread.start()
def _connect(self):
self._aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN)
self.status = "Connecting"
logging.info("APRS-IS connecting...")
self._aprsis.connect()
self._aprsis.consumer(self._handle)
logging.info("APRS-IS connected.")
def stop(self):
self.status = "Shutting down"
self._aprsis.close()
self._thread.join()
def _handle(self, data):
# Split SSID in "from" call and store separately
from_parts = str(data["from"]).split("-")
dx_call = from_parts[0].upper()
dx_ssid = from_parts[1].upper() if len(from_parts) > 1 else None
via_parts = str(data["via"]).split("-")
de_call = via_parts[0].upper()
de_ssid = via_parts[1].upper() if len(via_parts) > 1 else None
spot = Spot(source="APRS-IS",
dx_call=dx_call,
dx_ssid=dx_ssid,
de_call=de_call,
de_ssid=de_ssid,
comment=str(data["comment"]) if "comment" in data else None,
dx_latitude=float(data["latitude"]) if "latitude" in data else None,
dx_longitude=float(data["longitude"]) if "longitude" in data else None,
time=datetime.now(
pytz.UTC).timestamp()) # APRS-IS spots are live so we can assume spot time is "now"
# Add to our list
self._submit(spot)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
logging.debug("Data received from APRS-IS.")
providers/spot/dxcluster.py
+106
View File
@@ -0,0 +1,106 @@
import logging
import re
from datetime import datetime
from threading import Thread
from time import sleep
import pytz
import telnetlib3
from core.config import SERVER_OWNER_CALLSIGN
from data.spot import Spot
from providers.spot.spot_provider import SpotProvider
class DXCluster(SpotProvider):
"""Spot provider for a DX Cluster. Hostname, port, login_prompt, login_callsign and allow_rbn_spots are provided in config.
See config-example.yml for examples."""
_LINE_PATTERN_EXCLUDE_RBN = re.compile(
r"^DX de ([a-z0-9/]+):\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE)
_LINE_PATTERN_ALLOW_RBN = re.compile(
r"^DX de ([a-z0-9/]+)-?#?:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE)
def __init__(self, provider_config):
"""Constructor requires hostname and port"""
name = provider_config["name"] if "name" in provider_config else "Cluster"
super().__init__(name, provider_config)
self._hostname = provider_config["host"]
self._port = provider_config["port"]
self._login_prompt = provider_config["login_prompt"] if "login_prompt" in provider_config else "login:"
self._login_callsign = provider_config[
"login_callsign"] if "login_callsign" in provider_config else SERVER_OWNER_CALLSIGN
self._allow_rbn_spots = provider_config["allow_rbn_spots"] if "allow_rbn_spots" in provider_config else False
self._spot_line_pattern = self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
self._telnet = None
self._thread = Thread(target=self._handle)
self._thread.daemon = True
self._running = True
def start(self):
self._thread.start()
def stop(self):
self._running = False
self._telnet.close()
self._thread.join()
def _handle(self):
while self._running:
connected = False
while not connected and self._running:
try:
self.status = "Connecting"
logging.info("DX Cluster " + self._hostname + " connecting...")
self._telnet = telnetlib3.Telnet(self._hostname, self._port)
self._telnet.read_until(self._login_prompt.encode("latin-1"))
self._telnet.write((self._login_callsign + "\n").encode("latin-1"))
connected = True
logging.info("DX Cluster " + self._hostname + " connected.")
except ConnectionRefusedError:
self.status = "Error"
logging.warning("Connection refused to DX cluster " + self._hostname)
sleep(300)
except Exception:
self.status = "Error"
logging.exception("Exception while connecting to DX Cluster Provider (" + self._hostname + ").")
sleep(5)
self.status = "Waiting for Data"
while connected and self._running:
try:
# Check new telnet info against regular expression
telnet_output = self._telnet.read_until("\n".encode("latin-1"))
match = self._spot_line_pattern.match(telnet_output.decode("latin-1"))
if match:
spot_time = datetime.strptime(match.group(5), "%H%MZ")
spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(),
tzinfo=pytz.UTC)
spot = Spot(source=self.name,
dx_call=match.group(3),
de_call=match.group(1),
freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(),
time=spot_datetime.timestamp())
# Add to our list
self._submit(spot)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
logging.debug("Data received from DX Cluster " + self._hostname + ".")
except Exception:
connected = False
if self._running:
self.status = "Error"
logging.exception("Exception in DX Cluster Provider (" + self._hostname + ")")
sleep(5)
else:
logging.info("DX Cluster " + self._hostname + " shutting down...")
self.status = "Shutting down"
self.status = "Disconnected"
providers/spot/gma.py
+115
View File
@@ -0,0 +1,115 @@
import logging
from datetime import datetime
import pytz
from core.constants import HTTP_HEADERS
from core.url_data_cache import URLDataCache
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class GMA(HTTPSpotProvider):
"""Spot provider for General Mountain Activity"""
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://www.gma.rocks/api/spots/25/"
# GMA spots don't contain the details of the programme they are for, we need a separate lookup for that
REF_INFO_URL_ROOT = "https://www.gma.rocks/api/ref/?"
def __init__(self, provider_config):
# Ensure there is an API key in our config, and set up the query URL using it. If no key is provided,
# disable this spot provider.
self._api_key = provider_config.get("api-key", "")
if self._api_key == "":
provider_config["enabled"] = False
logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
self._url_data_cache = URLDataCache("GMA")
super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self._api_key, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
# Iterate through source data
if "RCD" in http_response.json():
for source_spot in http_response.json()["RCD"]:
# Convert to our spot format
spot = Spot(source=self.name,
dx_call=source_spot["ACTIVATOR"].upper(),
de_call=source_spot["SPOTTER"].upper(),
# Seen GMA spots with no frequency or with "QRT" in this field
freq=float(source_spot["QRG"]) * 1000 if (
source_spot["QRG"] != "" and source_spot["QRG"] != "QRT") else None,
# Filter out some weird mode strings
mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
comment=source_spot["TEXT"],
sig_refs=[SIGRef(id=source_spot["REF"], sig="", name=source_spot["NAME"])],
time=datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
tzinfo=pytz.UTC).timestamp(),
# Seen GMA spots with no (or empty) lat/lon
dx_latitude=float(source_spot["LAT"]) if (
source_spot["LAT"] and source_spot["LAT"] != "") else None,
dx_longitude=float(source_spot["LON"]) if (
source_spot["LON"] and source_spot["LON"] != "") else None,
qrt=source_spot["QRG"] == "QRT")
# GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
if "REF" in source_spot:
try:
ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
headers=HTTP_HEADERS)
# Sometimes this is blank even if it's a 200 response, so handle that
if ref_response.ok and ref_response.text is not None and ref_response.text != "":
ref_info = ref_response.json()
# If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
# spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
# and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry
# to determine if it's a SOTA summit.
if spot.sig_refs and "reftype" in ref_info and ref_info["reftype"] not in ["POTA",
"WWFF"] and (
ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info[
"sota"] == ""):
match ref_info["reftype"]:
case "Summit":
spot.sig_refs[0].sig = "GMA"
spot.sig = "GMA"
case "IOTA Island":
spot.sig_refs[0].sig = "IOTA"
spot.sig = "IOTA"
case "Lighthouse (ILLW)":
spot.sig_refs[0].sig = "ILLW"
spot.sig = "ILLW"
case "Lighthouse (ARLHS)":
spot.sig_refs[0].sig = "ARLHS"
spot.sig = "ARLHS"
case "Castle":
spot.sig_refs[0].sig = "WCA"
spot.sig = "WCA"
case "Mill":
spot.sig_refs[0].sig = "MOTA"
spot.sig = "MOTA"
case _:
logging.warning("GMA spot found with ref type " + ref_info[
"reftype"] + ", developer needs to add support for this!")
spot.sig_refs[0].sig = ref_info["reftype"]
spot.sig = ref_info["reftype"]
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point;
# other code will do that for us.
new_spots.append(spot)
elif not ref_response.from_cache:
if not ref_response.ok:
logging.warning(
f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot["REF"]}")
else:
logging.warning(
f"GMA API returned a malformed response when looking up ref {source_spot["REF"]}")
except:
logging.warning("Exception when looking up " + self.REF_INFO_URL_ROOT + source_spot[
"REF"] + ", ignoring this spot for now")
else:
logging.warning(f"The GMA API returned an unexpected response (HTTP {http_response.status_code}).")
return new_spots
providers/spot/hema.py
+75
View File
@@ -0,0 +1,75 @@
import logging
import re
from datetime import datetime
import pytz
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from core.constants import HTTP_HEADERS
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class HEMA(HTTPSpotProvider):
"""Spot provider for HuMPs Excluding Marilyns Award"""
POLL_INTERVAL_SEC = 300
# HEMA wants us to check for a "spot seed" from the API and see if it's actually changed before querying the main
# data API. So it's actually the SPOT_SEED_URL that we pass into the constructor and get the superclass to call on a
# timer. The actual data lookup all happens after parsing and checking the seed.
SPOT_SEED_URL = "http://www.hema.org.uk/spotSeed.jsp"
SPOTS_URL = "http://www.hema.org.uk/spotsMobile.jsp"
FREQ_MODE_PATTERN = re.compile("^([\\d.]*) \\((.*)\\)$")
SPOTTER_COMMENT_PATTERN = re.compile("^\\((.*)\\) (.*)$")
def __init__(self, provider_config):
super().__init__("HEMA", provider_config, self.SPOT_SEED_URL, self.POLL_INTERVAL_SEC)
self._spot_seed = ""
def _http_response_to_spots(self, http_response):
# OK, source data is actually just the spot seed at this point. We'll then go on to fetch real data if we know
# this has changed.
spot_seed_changed = http_response.text != self._spot_seed
self._spot_seed = http_response.text
new_spots = []
# OK, if the spot seed actually changed, now we make the real request for data.
if spot_seed_changed:
try:
source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30))
source_data_items = source_data.text.split("=")
# Iterate through source data items.
for source_spot in source_data_items:
spot_items = source_spot.split(";")
# Any line with less than 9 items is not a proper spot line
if len(spot_items) >= 9:
# Fiddle with some data to extract bits we need. Freq/mode and spotter/comment come in combined fields.
freq_mode_match = re.search(self.FREQ_MODE_PATTERN, spot_items[5])
spotter_comment_match = re.search(self.SPOTTER_COMMENT_PATTERN, spot_items[6])
if not freq_mode_match or not spotter_comment_match:
continue
# Convert to our spot format
spot = Spot(source=self.name,
dx_call=spot_items[2].upper(),
de_call=spotter_comment_match.group(1).upper(),
freq=float(freq_mode_match.group(1)) * 1000000,
mode=freq_mode_match.group(2).upper(),
comment=spotter_comment_match.group(2),
sig="HEMA",
sig_refs=[SIGRef(id=spot_items[3].upper(), sig="HEMA", name=spot_items[4])],
time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(
tzinfo=pytz.UTC).timestamp(),
dx_latitude=float(spot_items[7]),
dx_longitude=float(spot_items[8]))
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other
# code will do that for us.
new_spots.append(spot)
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing HEMA spots API.")
except ConnectionError:
logging.warning("Connection error when accessing HEMA spots API.")
return new_spots
providers/spot/http_spot_provider.py
+75
View File
@@ -0,0 +1,75 @@
import logging
from datetime import datetime
from threading import Thread, Event
import pytz
import requests
from requests import ReadTimeout
from requests.exceptions import ConnectionError, ConnectTimeout
from core.constants import HTTP_HEADERS
from providers.spot.spot_provider import SpotProvider
class HTTPSpotProvider(SpotProvider):
"""Generic spot provider class for providers that request data via HTTP(S). Just for convenience to avoid code
duplication. Subclasses of this query the individual APIs for data."""
def __init__(self, name, provider_config, url, poll_interval):
super().__init__(name, provider_config)
self._url = url
self._poll_interval = poll_interval
self._thread = None
self._stop_event = Event()
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting.
logging.info("Set up query of " + self.name + " spot API every " + str(self._poll_interval) + " seconds.")
self._thread = Thread(target=self._run, daemon=True)
self._thread.start()
def stop(self):
self._stop_event.set()
def _run(self):
while True:
self._poll()
if self._stop_event.wait(timeout=self._poll_interval):
break
def _poll(self):
try:
# Request data from API
logging.debug("Polling " + self.name + " spot API...")
http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
# Check response code was good
if http_response.ok:
# Pass off to the subclass for processing
new_spots = self._http_response_to_spots(http_response)
# Submit the new spots for processing. There might not be any spots for the less popular programs.
if new_spots:
self._submit_batch(new_spots)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
logging.debug("Received data from " + self.name + " spot API.")
else:
self.status = "Error"
logging.warning(f"HTTP {http_response.status_code} when calling {self.name} spot API.")
except ConnectionError:
logging.warning(f"Connection error when accessing {self.name} spots API.")
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing {self.name} spots API.")
except Exception:
self.status = "Error"
logging.exception("Exception in HTTP Spot Provider (" + self.name + ")")
self._stop_event.wait(timeout=1)
def _http_response_to_spots(self, http_response):
"""Convert an HTTP response returned by the API into spot data. The whole response is provided here so the subclass
implementations can check for HTTP status codes if necessary, and handle the response as JSON, XML, text, whatever
the API actually provides."""
raise NotImplementedError("Subclasses must implement this method")
providers/spot/llota.py
+42
View File
@@ -0,0 +1,42 @@
from datetime import datetime
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class LLOTA(HTTPSpotProvider):
"""Spot provider for Lagos y Lagunas On the Air"""
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://llota.app/api/public/spots"
def __init__(self, provider_config):
super().__init__("LLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
# Iterate through source data
for source_spot in http_response.json():
# Find the most recent spotter and comment from the history array
comment = None
spotter = None
if "history" in source_spot and len(source_spot["history"]) > 0:
comment = str(source_spot["history"][-1]["comment"])
spotter = str(source_spot["history"][-1]["spotter_callsign"])
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["callsign"].upper(),
de_call=spotter.upper() if spotter else None,
freq=float(source_spot["frequency"]) * 1000000,
mode=source_spot["mode"].upper(),
comment=comment,
sig="LLOTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="LLOTA", name=source_spot["reference_name"])],
time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp())
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
new_spots.append(spot)
return new_spots
providers/spot/parksnpeaks.py
+65
View File
@@ -0,0 +1,65 @@
import logging
import re
from datetime import datetime
import pytz
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class ParksNPeaks(HTTPSpotProvider):
"""Spot provider for Parks n Peaks"""
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
def __init__(self, provider_config):
super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
# Iterate through source data
if http_response and http_response != "":
for source_spot in http_response.json():
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["actID"],
dx_call=source_spot["actCallsign"].upper(),
de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
# typo exists in API
freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "")
.replace("+/-", "").strip()) * 1000000 if (source_spot["actFreq"] != "") else None,
# Seen PNP spots with empty frequency, and with comma-separated thousands digits
mode=source_spot["actMode"].upper(),
comment=source_spot["actComments"],
time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp())
# Extract a de_call if it's in the comment but not in the "actSpoter" field
m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment or "")
if not spot.de_call and m:
spot.de_call = str(m.group(1))
# Record SIG information. Sometimes we get a "SIG" of "QRP", which we ignore as it's not a programme with a
# defined set of references
sig = source_spot["actClass"].upper()
sig_ref = source_spot["actSiteID"]
if sig and sig != "" and sig != "QRP" and sig_ref and sig_ref != "":
spot.sig = sig
sig_refs = [SIGRef(id=source_spot["actSiteID"], sig=source_spot["actClass"].upper())]
spot.sig_refs = sig_refs
# Free text location is not present in all spots, so only add it if it's set
if "actLocation" in source_spot and source_spot["actLocation"] != "":
sig_refs[0].name = source_spot["actLocation"]
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA"]:
logging.warning("PNP spot found with sig " + sig + ", developer needs to add support for this!")
# Add new spot to the list
new_spots.append(spot)
return new_spots
providers/spot/pota.py
+42
View File
@@ -0,0 +1,42 @@
from datetime import datetime
import pytz
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class POTA(HTTPSpotProvider):
"""Spot provider for Parks on the Air"""
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://api.pota.app/spot/activator"
def __init__(self, provider_config):
super().__init__("POTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
# Iterate through source data
for source_spot in http_response.json():
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["spotId"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["frequency"]) * 1000,
mode=source_spot["mode"].upper(),
comment=source_spot["comments"],
sig="POTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="POTA", name=source_spot["name"])],
time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp(),
dx_grid=source_spot["grid6"],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"])
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
new_spots.append(spot)
return new_spots
providers/spot/rbn.py
+93
View File
@@ -0,0 +1,93 @@
import logging
import re
from datetime import datetime
from threading import Thread
from time import sleep
import pytz
import telnetlib3
from core.config import SERVER_OWNER_CALLSIGN
from data.spot import Spot
from providers.spot.spot_provider import SpotProvider
class RBN(SpotProvider):
"""Spot provider for the Reverse Beacon Network. Connects to a single port, if you want both CW/RTTY (port 7000) and FT8
(port 7001) you need to instantiate two copies of this. The port is provided as an argument to the constructor."""
_LINE_PATTERN = re.compile(
r"^DX de ([a-z0-9/]+)-.*:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
re.IGNORECASE)
def __init__(self, provider_config):
"""Constructor requires port number."""
name = provider_config["name"] if "name" in provider_config else "RBN"
super().__init__(name, provider_config)
self._port = provider_config["port"]
self._telnet = None
self._thread = Thread(target=self._handle)
self._thread.daemon = True
self._running = True
def start(self):
self._thread.start()
def stop(self):
self._running = False
self._telnet.close()
self._thread.join()
def _handle(self):
while self._running:
connected = False
while not connected and self._running:
try:
self.status = "Connecting"
logging.info("RBN port " + str(self._port) + " connecting...")
self._telnet = telnetlib3.Telnet("telnet.reversebeacon.net", self._port)
self._telnet.read_until("Please enter your call: ".encode("latin-1"))
self._telnet.write((SERVER_OWNER_CALLSIGN + "\n").encode("latin-1"))
connected = True
logging.info("RBN port " + str(self._port) + " connected.")
except Exception:
self.status = "Error"
logging.exception("Exception while connecting to RBN (port " + str(self._port) + ").")
sleep(5)
self.status = "Waiting for Data"
while connected and self._running:
try:
# Check new telnet info against regular expression
telnet_output = self._telnet.read_until("\n".encode("latin-1"))
match = self._LINE_PATTERN.match(telnet_output.decode("latin-1"))
if match:
spot_time = datetime.strptime(match.group(5), "%H%MZ")
spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(),
tzinfo=pytz.UTC)
spot = Spot(source=self.name,
dx_call=match.group(3),
de_call=match.group(1),
freq=float(match.group(2)) * 1000,
comment=match.group(4).strip(),
time=spot_datetime.timestamp())
# Add to our list
self._submit(spot)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
logging.debug("Data received from RBN on port " + str(self._port) + ".")
except Exception:
connected = False
if self._running:
self.status = "Error"
logging.exception("Exception in RBN provider (port " + str(self._port) + ")")
sleep(5)
else:
logging.info("RBN provider (port " + str(self._port) + ") shutting down...")
self.status = "Shutting down"
self.status = "Disconnected"
providers/spot/sota.py
+63
View File
@@ -0,0 +1,63 @@
import logging
from datetime import datetime
import requests
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from core.constants import HTTP_HEADERS
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class SOTA(HTTPSpotProvider):
"""Spot provider for Summits on the Air"""
POLL_INTERVAL_SEC = 120
# SOTA wants us to check for an "epoch" from the API and see if it's actually changed before querying the main data
# APIs. So it's actually the EPOCH_URL that we pass into the constructor and get the superclass to call on a timer.
# The actual data lookup all happens after parsing and checking the epoch.
EPOCH_URL = "https://api-db2.sota.org.uk/api/spots/epoch"
SPOTS_URL = "https://api-db2.sota.org.uk/api/spots/60/all/all"
def __init__(self, provider_config):
super().__init__("SOTA", provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)
self._api_epoch = ""
def _http_response_to_spots(self, http_response):
# OK, source data is actually just the epoch at this point. We'll then go on to fetch real data if we know this
# has changed.
epoch_changed = http_response.text != self._api_epoch
self._api_epoch = http_response.text
new_spots = []
# OK, if the epoch actually changed, now we make the real request for data.
if epoch_changed:
try:
source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30)).json()
# Iterate through source data
for source_spot in source_data:
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activatorCallsign"].upper(),
dx_name=source_spot["activatorName"],
de_call=source_spot["callsign"].upper(),
freq=(float(source_spot["frequency"]) * 1000000) if (
source_spot["frequency"] is not None) else None,
# Seen SOTA spots with no frequency!
mode=source_spot["mode"].upper(),
comment=source_spot["comments"],
sig="SOTA",
sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA", name=source_spot["summitName"],
activation_score=source_spot["points"])],
time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp())
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
new_spots.append(spot)
except ConnectionError:
logging.warning("Connection error when accessing SOTA spots API")
except (ConnectTimeout, ReadTimeout):
logging.warning(f"Timeout when accessing SOTA spots API.")
return new_spots
providers/spot/spot_provider.py
+60
View File
@@ -0,0 +1,60 @@
from datetime import datetime
import pytz
from core.data_store import DATA_STORE
class SpotProvider:
"""Generic spot provider class. Subclasses of this query the individual APIs for data."""
def __init__(self, name, provider_config):
"""Constructor"""
self.name = name
self.enabled = provider_config["enabled"]
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
self.status = "Not Started" if self.enabled else "Disabled"
self._spots = DATA_STORE.spots
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
raise NotImplementedError("Subclasses must implement this method")
def _submit_batch(self, spots):
"""Submit a batch of spots retrieved from the provider. Only spots that are newer than the last spot retrieved
by this provider will be added to the spot list, to prevent duplications. Spots passing the check will also have
their infer_missing() method called to complete their data set. This is called by the API-querying
subclasses on receiving spots."""
# Sort the batch so that earliest ones go in first. This helps keep the ordering correct when spots are fired
# off to SSE listeners.
spots = sorted(spots, key=lambda s: (s.time if s and s.time else 0))
for spot in spots:
if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time:
# Fill in any blanks and add to the list
spot.infer_missing()
self._add_spot(spot)
if spots:
self.last_spot_time = datetime.fromtimestamp(max(map(lambda s: s.time, spots)), pytz.UTC)
def _submit(self, spot):
"""Submit a single spot retrieved from the provider. This will be added to the list regardless of its age. Spots
passing the check will also have their infer_missing() method called to complete their data set. This is called by
the data streaming subclasses, which can be relied upon not to re-provide old spots."""
# Fill in any blanks and add to the list
spot.infer_missing()
self._add_spot(spot)
self.last_spot_time = datetime.fromtimestamp(spot.time, pytz.UTC)
def _add_spot(self, spot):
if not spot.expired():
self._spots.set(spot.id, spot)
def stop(self):
"""Stop any threads and prepare for application shutdown"""
raise NotImplementedError("Subclasses must implement this method")
providers/spot/sse_spot_provider.py
+79
View File
@@ -0,0 +1,79 @@
import logging
from datetime import datetime
from threading import Thread
from time import sleep
import pytz
from requests_sse import EventSource
from core.constants import HTTP_HEADERS
from providers.spot.spot_provider import SpotProvider
class SSESpotProvider(SpotProvider):
"""Spot provider using Server-Sent Events."""
def __init__(self, name, provider_config, url):
super().__init__(name, provider_config)
self._url = url
self._event_source = None
self._thread = None
self._stopped = False
self._last_event_id = None
def start(self):
logging.info("Set up SSE connection to " + self.name + " spot API.")
self._stopped = False
self._thread = Thread(target=self._run)
self._thread.daemon = True
self._thread.start()
def stop(self):
self._stopped = True
if self._event_source:
self._event_source.close()
if self._thread:
self._thread.join()
def _on_open(self):
self.status = "Waiting for Data"
def _on_error(self):
self.status = "Connecting"
def _run(self):
while not self._stopped:
try:
logging.debug("Connecting to " + self.name + " spot API...")
self.status = "Connecting"
with EventSource(self._url, headers=HTTP_HEADERS, latest_event_id=self._last_event_id, timeout=30,
on_open=self._on_open, on_error=self._on_error) as event_source:
self._event_source = event_source
for event in self._event_source:
if event.type == 'message':
try:
self._last_event_id = event.last_event_id
new_spot = self._sse_message_to_spot(event.data)
if new_spot:
self._submit(new_spot)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
logging.debug("Received data from " + self.name + " spot API.")
except Exception:
logging.exception(
"Exception processing message from SSE Spot Provider (" + self.name + ")")
except Exception:
self.status = "Error"
logging.exception("Exception in SSE Spot Provider (" + self.name + ")")
else:
self.status = "Disconnected"
sleep(5) # Wait before trying to reconnect
def _sse_message_to_spot(self, message_data):
"""Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass
implementations can handle the message as JSON, XML, text, whatever the API actually provides."""
raise NotImplementedError("Subclasses must implement this method")
providers/spot/tiles.py
+52
View File
@@ -0,0 +1,52 @@
from datetime import datetime
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class Tiles(HTTPSpotProvider):
"""Spot provider for Tiles on the Air"""
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/spots?active_hours=24"
def __init__(self, provider_config):
super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
# Iterate through source data
for source_spot in http_response.json()["spots"]:
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["call_sign"].upper(),
# No separate spotter callsign, assume all spots are self-spots
de_call=source_spot["call_sign"].upper(),
freq=float(strip_extra_decimal_points(source_spot["frequency"])) * 1000000,
mode=source_spot["mode"].upper(),
comment=source_spot["notes"],
sig="Tiles",
# Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
# Just take the grid reference itself as the single Tiles SIG reference.
sig_refs=[SIGRef(id=source_spot["maidenhead_grid"], sig="Tiles",
name=source_spot["maidenhead_grid"])],
time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
dx_grid=source_spot["maidenhead_grid"],
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"])
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
new_spots.append(spot)
return new_spots
# Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles'
# strange frequency format where we can sometimes have e.g. "14.123.5".
def strip_extra_decimal_points(s):
parts = s.split('.', 1)
if len(parts) == 1:
return s
return parts[0] + '.' + parts[1].replace('.', '')
providers/spot/towers.py
+41
View File
@@ -0,0 +1,41 @@
import json
from datetime import datetime
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class Towers(HTTPSpotProvider):
"""Spot provider for Towers on the Air"""
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://wwtota.com/api/cluster_live.php"
def __init__(self, provider_config):
super().__init__("Towers", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
response_fixed = http_response.text.replace("\\/", "/")
response_json = json.loads(response_fixed)
# Iterate through source data
for source_spot in response_json["spots"]:
# Convert to our spot format
likely_freq = float(source_spot["freq"]) * 1000
if likely_freq < 1000000:
likely_freq = likely_freq * 1000
spot = Spot(source=self.name,
dx_call=source_spot["call"].upper(),
freq=likely_freq,
comment=source_spot["comment"],
sig="Towers",
sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
time=datetime.strptime(response_json["updated"][:10] + source_spot["time"],
"%Y-%m-%d%H:%M").timestamp())
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
new_spots.append(spot)
return new_spots
providers/spot/ukpacketnet.py
+94
View File
@@ -0,0 +1,94 @@
import re
from datetime import datetime
import pytz
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class UKPacketNet(HTTPSpotProvider):
"""Spot provider for UK Packet Radio network API"""
POLL_INTERVAL_SEC = 600
SPOTS_URL = "https://nodes.ukpacketradio.network/api/nodedata"
def __init__(self, provider_config):
super().__init__("UK Packet Net", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
# Iterate through source data
nodes = http_response.json()["nodes"]
for callsign, node in nodes.items():
# The node corresponse to the spotter here. It has an "mheard" section which indicates which nodes it has
# recently heard, which will be our "DX". But "mheard" stations are not necessarily over RF, they could be
# via the internet, so we also need to look up the "port" on which the station was heard, and check that it
# is RF.
if "mheard" in node:
for heard in node["mheard"]:
heard_port_id = heard["port"]
if "ports" in node:
for listed_port in node["ports"]:
if listed_port["id"] == heard_port_id and listed_port["linkType"] == "RF":
# This is another packet station heard over RF, so we are good to create a Spot object.
# First build a "full" comment combining some of the extra info
comment = listed_port["comment"] if "comment" in listed_port else ""
comment = (comment + " " + listed_port["mode"]) if "mode" in listed_port else comment
comment = (comment + " " + listed_port[
"modulation"]) if "modulation" in listed_port else comment
comment = (comment + " " + str(
listed_port["baud"]) + " baud") if "baud" in listed_port and listed_port[
"baud"] > 0 else comment
# Get frequency from the comment if it's not set properly in the data structure. This is
# very hacky but a lot of node comments contain their frequency as the first or second
# word of their comment, but not in the proper data structure field.
freq = listed_port["freq"] if "freq" in listed_port and listed_port[
"freq"] > 0 else None
if not freq and comment:
possible_freq = comment.split(" ")[0].upper().replace("MHZ", "")
if re.match(r"^[0-9.]+$",
possible_freq) and possible_freq != "1200" and possible_freq != "9600":
freq = float(possible_freq) * 1000000
if not freq and len(comment.split(" ")) > 1:
possible_freq = comment.split(" ")[1].upper().replace("MHZ", "")
if re.match(r"^[0-9.]+$",
possible_freq) and possible_freq != "1200" and possible_freq != "9600":
freq = float(possible_freq) * 1000000
# Check for a found frequency likely having been in kHz, sorry to all GHz packet folks
if freq and freq > 1000000000:
freq = freq / 1000
# Now build the spot object
spot = Spot(source=self.name,
dx_call=heard["callsign"].upper(),
de_call=node["callsign"].upper(),
freq=freq,
mode="PKT",
comment=comment,
time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S").replace(
tzinfo=pytz.UTC).timestamp(),
de_grid=node["location"]["locator"] if "locator" in node[
"location"] else None,
de_latitude=node["location"]["coords"]["lat"],
de_longitude=node["location"]["coords"]["lon"])
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
new_spots.append(spot)
break
# Now we have a list of every node that heard every other node via RF, as Spothole spots. What each spot doesn't
# yet have is a DX lat/lon/grid, because the data doesn't provide the location of the "mheard" stations within
# the structure. However, each "heard" station should also be represented in the list somewhere with its own
# data, and we can use that to look these up.
for spot in new_spots:
if spot.dx_call in nodes:
spot.dx_grid = nodes[spot.dx_call]["location"]["locator"] if "locator" in nodes[spot.dx_call][
"location"] else None
spot.dx_latitude = nodes[spot.dx_call]["location"]["coords"]["lat"]
spot.dx_longitude = nodes[spot.dx_call]["location"]["coords"]["lon"]
return new_spots
providers/spot/websocket_spot_provider.py
+77
View File
@@ -0,0 +1,77 @@
import logging
from datetime import datetime
from threading import Thread
from time import sleep
import pytz
from websocket import create_connection
from core.constants import HTTP_HEADERS
from providers.spot.spot_provider import SpotProvider
class WebsocketSpotProvider(SpotProvider):
"""Spot provider using websockets."""
def __init__(self, name, provider_config, url):
super().__init__(name, provider_config)
self._url = url
self._ws = None
self._thread = None
self._stopped = False
self._last_event_id = None
def start(self):
logging.info("Set up websocket connection to " + self.name + " spot API.")
self._stopped = False
self._thread = Thread(target=self._run)
self._thread.daemon = True
self._thread.start()
def stop(self):
self._stopped = True
if self._ws:
self._ws.close()
if self._thread:
self._thread.join()
def _on_open(self):
self.status = "Waiting for Data"
def _on_error(self):
self.status = "Connecting"
def _run(self):
while not self._stopped:
try:
logging.debug("Connecting to " + self.name + " spot API...")
self.status = "Connecting"
self._ws = create_connection(self._url, header=HTTP_HEADERS)
self.status = "Connected"
data = self._ws.recv()
if data:
try:
new_spot = self._ws_message_to_spot(data)
if new_spot:
self._submit(new_spot)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
logging.debug("Received data from " + self.name + " spot API.")
except Exception:
logging.exception(
"Exception processing message from Websocket Spot Provider (" + self.name + ")")
except Exception as e:
self.status = "Error"
logging.exception("Exception in Websocket Spot Provider (" + self.name + ")", e)
else:
self.status = "Disconnected"
sleep(5) # Wait before trying to reconnect
def _ws_message_to_spot(self, b):
"""Convert a WS message received from the API into a spot. The exact message data (in bytes) is provided here so the
subclass implementations can handle the message as string, JSON, XML, whatever the API actually provides."""
raise NotImplementedError("Subclasses must implement this method")
providers/spot/wota.py
+81
View File
@@ -0,0 +1,81 @@
import logging
import re
from datetime import datetime
from typing import cast
import pytz
from rss_parser import Parser
from rss_parser.models.rss import RSS
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class WOTA(HTTPSpotProvider):
"""Spot provider for Wainwrights on the Air"""
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://www.wota.org.uk/spots_rss.php"
LIST_URL = "https://www.wota.org.uk/mapping/data/summits.json"
RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
def __init__(self, provider_config):
super().__init__("WOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
# Iterate through source data
for source_spot in rss.channel.items:
try:
# Reject GUID missing or zero
if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0":
continue
# Pick apart the title
title_split = source_spot.title.split(" on ")
dx_call = title_split[0]
ref = None
ref_name = None
if len(title_split) > 1:
ref_split = title_split[1].split(" - ")
ref = str(ref_split[0])
if len(ref_split) > 1:
ref_name = str(ref_split[1])
# Pick apart the description
desc_split = source_spot.description.split(". ")
freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
freq_mode_split = re.split(r'[\-\s]+', freq_mode)
freq_hz = float(freq_mode_split[0].replace("'", ".")) * 1000000
mode = None
if len(freq_mode_split) > 1:
mode = freq_mode_split[1].upper()
comment = None
if len(desc_split) > 1:
comment = desc_split[1].strip()
spotter = None
if len(desc_split) > 2:
spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").upper().strip()
time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot.guid.content,
dx_call=dx_call,
de_call=spotter,
freq=freq_hz,
mode=mode,
comment=comment,
sig="WOTA",
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
time=time.timestamp())
new_spots.append(spot)
except Exception as e:
logging.error("Exception parsing WOTA spot", e)
return new_spots
providers/spot/wwbota.py
+43
View File
@@ -0,0 +1,43 @@
import json
from datetime import datetime
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.sse_spot_provider import SSESpotProvider
class WWBOTA(SSESpotProvider):
"""Spot provider for Worldwide Bunkers on the Air"""
SPOTS_URL = "https://api.wwbota.net/spots/"
def __init__(self, provider_config):
super().__init__("WWBOTA", provider_config, self.SPOTS_URL)
def _sse_message_to_spot(self, message_data):
source_spot = json.loads(message_data)
# Convert to our spot format. First we unpack references, because WWBOTA spots can have more than one for
# n-fer activations.
refs = []
for ref in source_spot["references"]:
sigref = SIGRef(id=ref["reference"], sig="WWBOTA", name=ref["name"])
refs.append(sigref)
spot = Spot(source=self.name,
dx_call=source_spot["call"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["freq"]) * 1000000,
mode=source_spot["mode"].upper(),
comment=source_spot["comment"],
sig="WWBOTA",
sig_refs=refs,
time=datetime.fromisoformat(source_spot["time"].replace("Z", "+00:00")).timestamp(),
# WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
# now, we will just pick the first one to use as our grid, latitude and longitude.
dx_grid=source_spot["references"][0]["locator"],
dx_latitude=source_spot["references"][0]["lat"],
dx_longitude=source_spot["references"][0]["long"],
qrt=source_spot["type"] == "QRT")
# WWBOTA does support a special "Test" spot type, we need to avoid adding that.
return spot if source_spot["type"] != "Test" else None
providers/spot/wwff.py
+40
View File
@@ -0,0 +1,40 @@
from datetime import datetime
import pytz
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class WWFF(HTTPSpotProvider):
"""Spot provider for Worldwide Flora & Fauna"""
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://spots.wwff.co/static/spots.json"
def __init__(self, provider_config):
super().__init__("WWFF", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
# Iterate through source data
for source_spot in http_response.json():
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=float(source_spot["frequency_khz"]) * 1000,
mode=source_spot["mode"].upper(),
comment=source_spot["remarks"],
sig="WWFF",
sig_refs=[SIGRef(id=source_spot["reference"], sig="WWFF", name=source_spot["reference_name"])],
time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
dx_latitude=source_spot["latitude"],
dx_longitude=source_spot["longitude"])
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
# that for us.
new_spots.append(spot)
return new_spots
providers/spot/xota.py
+42
View File
@@ -0,0 +1,42 @@
import json
from datetime import datetime
import pytz
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.websocket_spot_provider import WebsocketSpotProvider
class XOTA(WebsocketSpotProvider):
"""Spot provider for servers based on the "xOTA" software at https://github.com/nischu/xOTA/
The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG which we can
then use for lookups. This functionality is implemented for Toilets on the Air events, of which there are
several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA, which is why we also
provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3 might become
"C3 T-01". This allows us to provide location lookups for TOTA at several conferences."""
LOCATION_DATA = {}
SIG = None
def __init__(self, provider_config):
name = provider_config["name"] if "name" in provider_config else "xOTA"
super().__init__(name, provider_config, provider_config["url"])
self.SIG = str(provider_config["sig"]) if "sig" in provider_config else None
self._sig_ref_prefix = str(provider_config["sig-ref-prefix"]) if "sig-ref-prefix" in provider_config else ""
def _ws_message_to_spot(self, b):
string = b.decode("utf-8")
source_spot = json.loads(string)
ref_id = self._sig_ref_prefix + " " + source_spot["reference"]["title"]
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["stationCallSign"].upper(),
freq=float(source_spot["freq"]) * 1000,
mode=source_spot["mode"].upper(),
sig=self.SIG,
sig_refs=[
SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"])],
time=datetime.now(pytz.UTC).timestamp(),
qrt=source_spot["state"] != "active")
return spot
providers/spot/zlota.py
+43
View File
@@ -0,0 +1,43 @@
from datetime import datetime
import pytz
from data.sig_ref import SIGRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
class ZLOTA(HTTPSpotProvider):
"""Spot provider for ZLOTA"""
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://ontheair.nz/api/spots?zlota_only=true"
LIST_URL = "https://ontheair.nz/assets/assets.json"
def __init__(self, provider_config):
super().__init__("ZLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
def _http_response_to_spots(self, http_response):
new_spots = []
# Iterate through source data
for source_spot in http_response.json():
# Frequency is often inconsistent as to whether it's in Hz or kHz. Make a guess.
freq_hz = float(source_spot["frequency"])
if freq_hz < 1000000:
freq_hz = freq_hz * 1000
# Convert to our spot format
spot = Spot(source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["activator"].upper(),
de_call=source_spot["spotter"].upper(),
freq=freq_hz,
mode=source_spot["mode"].upper().strip(),
comment=source_spot["comments"],
sig="ZLOTA",
sig_refs=[SIGRef(id=source_spot["reference"], sig="ZLOTA", name=source_spot["name"])],
time=datetime.fromisoformat(source_spot["referenced_time"].replace("Z", "+00:00")).astimezone(
pytz.UTC).timestamp())
new_spots.append(spot)
return new_spots