diff --git a/config-example.yml b/config-example.yml index 54d338d..2842f3d 100644 --- a/config-example.yml +++ b/config-example.yml @@ -350,9 +350,10 @@ max_alert_age_sec: 604800 # Allow submitting spots to the Spothole API? allow_spotting: true -# Allow upstream submission of spots to external providers (POTA, SOTA, etc.) via the API? -# Requires allow_spotting to also be true. Set to false to only accept spots into the local -# Spothole database, without forwarding them to any external service. +# Allow upstream submission of spots to external providers via the API? Currently only DX cluster providers support +# this, and will log in using the spotter's callsign to send the spot. Upstream spotting to POTA, SOTA etc. is a work +# in progress. Requires allow_spotting to also be true. Set to false to only accept spots into the local Spothole +# database, without forwarding them to any external service. allow_upstream_spotting: false # Google reCAPTCHA v2 keys for CAPTCHA protection on upstream spot submission. Both keys must be set to enable CAPTCHA. diff --git a/providers/spot/dxcluster.py b/providers/spot/dxcluster.py index 911c183..a2510ac 100644 --- a/providers/spot/dxcluster.py +++ b/providers/spot/dxcluster.py @@ -3,9 +3,12 @@ import re from datetime import datetime import pytz +import telnetlib3 from core.config import SERVER_OWNER_CALLSIGN +from core.utils import decode_telnet_bytes from data.spot import Spot +from providers.spot.spot_provider import SpotSubmissionError from providers.spot.telnet_spot_provider import TelnetSpotProvider logger = logging.getLogger(__name__) @@ -24,6 +27,9 @@ class DXCluster(TelnetSpotProvider): re.IGNORECASE, ) + # Timeout for each step of the telnet conversation when submitting a spot + _SUBMIT_TIMEOUT_SEC = 10 + def __init__(self, provider_config): """Constructor requires hostname and port""" @@ -58,3 +64,73 @@ class DXCluster(TelnetSpotProvider): comment=match.group(4).strip(), time=spot_datetime.timestamp(), ) + + def can_submit_spot(self, activity): + # Clusters accept spots for any activity, or for no activity at all. + return True + + def submit_spot(self, spot, credentials): + """Submit a spot to the cluster. This makes a new telnet connection to the same node that we receive spots from, + logs in as the spotter, sends the spot using the DX command, then logs out. If anything goes wrong we raise an + error which can then be logged by the caller and returned to the user as an error message in the API JSON / web + UI.""" + + command = self._build_dx_command(spot) + telnet = telnetlib3.Telnet(self._host, self._port, timeout=self._SUBMIT_TIMEOUT_SEC) + try: + # Wait for login prompt and log in + output = telnet.read_until(self._login_prompt.encode("latin-1"), timeout=self._SUBMIT_TIMEOUT_SEC) + if self._login_prompt.encode("latin-1") not in output: + logger.debug(f"{self.name} response while waiting for login prompt: {decode_telnet_bytes(output)}") + raise SpotSubmissionError(f"{self.name} did not give the expected login prompt.") + telnet.write(f"{spot.de_call}\n".encode("latin-1")) + + # Wait for the command prompt. This is a line like "M0TRT de W3LPL 27-Sep-2026 0945Z dxspider >". We can't + # just wait for ">" as the welcome message might contain that character. + call = re.escape(spot.de_call).encode("latin-1") + prompt = re.compile(rb"(?im)^" + call + rb" de \S+.*>\s*$") + index, _, output = telnet.expect([prompt], timeout=self._SUBMIT_TIMEOUT_SEC) + if index < 0: + logger.debug(f"{self.name} response while waiting for command prompt: {decode_telnet_bytes(output)}") + raise SpotSubmissionError(f"{self.name} did not accept a login as {spot.de_call}.") + + # Send the spot. + logger.debug(f"Submitting spot to {self.name} as {spot.de_call}: {command}") + telnet.write(f"{command}\n".encode("latin-1")) + + # Log out + telnet.write(b"bye\n") + finally: + telnet.close() + + @staticmethod + def _build_dx_command(spot): + """Build the "DX " command used to submit a spot to the cluster. The mode, and + any activities and references e.g. "SSB POTA GB-1234 tnx 73".""" + + freq = spot.freq + if not freq or not spot.dx_call: + raise ValueError("A frequency and DX callsign are required to submit a spot to a cluster.") + + comment_parts = [] + # Mode + if spot.mode: + comment_parts.append(spot.mode) + # Build list of activities with refs like "POTA GB-0001 SOTA G/SC-001" + activities_with_refs = set() + for activity_ref in spot.activity_refs: + if activity_ref.id: + comment_parts.append(f"{activity_ref.activity} {activity_ref.id}") + activities_with_refs.add(activity_ref.activity) + # If we have activities that don't have refs, add them to the comment + comment_parts.extend(str(a) for a in spot.activities if a not in activities_with_refs) + # Add the actual spot comment + if spot.comment: + comment_parts.append(spot.comment) + + # Build command + command = f"DX {freq / 1000.0:.1f} {spot.dx_call} {' '.join(comment_parts)}" + + # Strip anything that isn't printable ASCII, so that e.g. a newline in the comment can't be used to send + # exploit the system and send additional commands to the cluster as the spotter. + return re.sub(r"\s+", " ", re.sub(r"[^\x20-\x7E]", " ", command)).strip() diff --git a/providers/spot/gma.py b/providers/spot/gma.py index 97b0279..34dd274 100644 --- a/providers/spot/gma.py +++ b/providers/spot/gma.py @@ -158,8 +158,9 @@ class GMA(HTTPSpotProvider): return new_spots + # Upstream spotting disabled until implemented def can_submit_spot(self, activity): - return activity == ActivityName.GMA + return False # return activity == ActivityName.GMA def submit_spot(self, spot, credentials): # TODO: Implement. diff --git a/providers/spot/hema.py b/providers/spot/hema.py index a79fd53..6ed6869 100644 --- a/providers/spot/hema.py +++ b/providers/spot/hema.py @@ -89,8 +89,9 @@ class HEMA(HTTPSpotProvider): logger.warning("Connection error when accessing HEMA spots API.") return new_spots + # Upstream spotting disabled until implemented def can_submit_spot(self, activity): - return activity == ActivityName.HEMA + return False # return activity == ActivityName.HEMA def submit_spot(self, spot, credentials): # TODO: Implement. Currently blocked awaiting their API team to make a change to allow us to spot with a diff --git a/providers/spot/parksnpeaks.py b/providers/spot/parksnpeaks.py index 4e90397..322e056 100644 --- a/providers/spot/parksnpeaks.py +++ b/providers/spot/parksnpeaks.py @@ -104,8 +104,9 @@ class ParksNPeaks(HTTPSpotProvider): new_spots.append(spot) return new_spots + # Upstream spotting disabled until properly tested def can_submit_spot(self, activity): - return activity in self.SUBMITTABLE_ACTIVITIES + return False # return activity in self.SUBMITTABLE_ACTIVITIES def submit_spot(self, spot, credentials): # TODO test this works diff --git a/providers/spot/pota.py b/providers/spot/pota.py index 31094e4..8c26021 100644 --- a/providers/spot/pota.py +++ b/providers/spot/pota.py @@ -41,7 +41,7 @@ class POTA(HTTPSpotProvider): name=source_spot["name"], latitude=source_spot["latitude"], longitude=source_spot["longitude"], - ref_type=ActivityRefType.PARK + ref_type=ActivityRefType.PARK, ) ], time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S") @@ -57,8 +57,9 @@ class POTA(HTTPSpotProvider): new_spots.append(spot) return new_spots + # Upstream spotting disabled until properly tested def can_submit_spot(self, activity): - return activity == ActivityName.POTA + return False # return activity == ActivityName.POTA def submit_spot(self, spot, credentials): ref_id = spot.activity_refs[0].id if spot.activity_refs else None diff --git a/providers/spot/sota.py b/providers/spot/sota.py index 2ec1e92..926caf7 100644 --- a/providers/spot/sota.py +++ b/providers/spot/sota.py @@ -83,8 +83,9 @@ class SOTA(HTTPSpotProvider): logger.warning("Timeout when accessing SOTA spots API.") return new_spots + # Upstream spotting disabled until properly tested def can_submit_spot(self, activity): - return activity == ActivityName.SOTA + return False # return activity == ActivityName.SOTA def submit_spot(self, spot, credentials): # TODO test this method works diff --git a/providers/spot/spot_provider.py b/providers/spot/spot_provider.py index 88df0a1..9f248b0 100644 --- a/providers/spot/spot_provider.py +++ b/providers/spot/spot_provider.py @@ -5,6 +5,12 @@ import pytz from core.data_store import DATA_STORE +class SpotSubmissionError(Exception): + """Raised by a provider's submit_spot() when the upstream service refused or didn't accept the spot. The message + is returned in the API / showed on the web UI. This is probably a user error, so we log a warning rather than a big + stack trace like we would if we got some unexpected exception.""" + + class SpotProvider: """Generic spot provider class. Subclasses of this query the individual APIs for data.""" @@ -61,7 +67,8 @@ class SpotProvider: raise NotImplementedError("Subclasses must implement this method") def can_submit_spot(self, activity): - """Return True if this provider supports submitting spots upstream for the given activity.""" + """Return True if this provider supports submitting spots upstream for the given activity. Activity can be None, + to check whether this provider supports submitting spots that have no activity.""" return False diff --git a/providers/spot/tiles.py b/providers/spot/tiles.py index e30f43d..fcfd441 100644 --- a/providers/spot/tiles.py +++ b/providers/spot/tiles.py @@ -69,7 +69,7 @@ class Tiles(HTTPSpotProvider): name=source_spot["maidenhead_grid"], latitude=source_spot["latitude"], longitude=source_spot["longitude"], - ref_type=ActivityRefType.GRID + ref_type=ActivityRefType.GRID, ) ], time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(), @@ -84,8 +84,9 @@ class Tiles(HTTPSpotProvider): new_spots.append(spot) return new_spots + # Upstream spotting disabled until properly tested def can_submit_spot(self, activity): - return activity == ActivityName.TILES + return False # return activity == ActivityName.TILES def submit_spot(self, spot, credentials): # Tiles on the air currently only supports *self* spots diff --git a/providers/spot/wota.py b/providers/spot/wota.py index 50c47fa..e34776a 100644 --- a/providers/spot/wota.py +++ b/providers/spot/wota.py @@ -114,8 +114,9 @@ class WOTA(HTTPSpotProvider): return new_spots + # Upstream spotting disabled until implemented def can_submit_spot(self, activity): - return activity == ActivityName.WOTA + return False # return activity == ActivityName.WOTA def submit_spot(self, spot, credentials): # TODO Ask M5TEA if he's happy to share how this is done from his app diff --git a/providers/spot/wwbota.py b/providers/spot/wwbota.py index 5026bb6..9a68d12 100644 --- a/providers/spot/wwbota.py +++ b/providers/spot/wwbota.py @@ -52,8 +52,9 @@ class WWBOTA(SSESpotProvider): # WWBOTA does support a special "Test" spot type, we need to avoid adding that. return spot if source_spot["type"] != "Test" else None + # Upstream spotting disabled until implemented def can_submit_spot(self, activity): - return activity == ActivityName.WWBOTA + return False # return activity == ActivityName.WWBOTA def submit_spot(self, spot, credentials): # TODO: Implement. WWBOTA API docs cover this: https://api.wwbota.org/#tag/Spots/operation/create_spot_spots__post diff --git a/providers/spot/wwff.py b/providers/spot/wwff.py index cc439f9..ee7b3a1 100644 --- a/providers/spot/wwff.py +++ b/providers/spot/wwff.py @@ -38,7 +38,7 @@ class WWFF(HTTPSpotProvider): name=source_spot["reference_name"], latitude=source_spot["latitude"], longitude=source_spot["longitude"], - ref_type=ActivityRefType.PARK + ref_type=ActivityRefType.PARK, ) ], time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(), @@ -51,8 +51,9 @@ class WWFF(HTTPSpotProvider): new_spots.append(spot) return new_spots + # Upstream spotting disabled until implemented def can_submit_spot(self, activity): - return activity == ActivityName.WWFF + return False # return activity == ActivityName.WWFF def submit_spot(self, spot, credentials): # TODO: Implement. Spotting to WWFF should be possible, need to look up the Spotline docs or copy approach from diff --git a/providers/spot/zlota.py b/providers/spot/zlota.py index eaf360b..5727024 100644 --- a/providers/spot/zlota.py +++ b/providers/spot/zlota.py @@ -51,8 +51,9 @@ class ZLOTA(HTTPSpotProvider): new_spots.append(spot) return new_spots + # Upstream spotting disabled until implemented def can_submit_spot(self, activity): - return activity == ActivityName.ZLOTA + return False # return activity == ActivityName.ZLOTA def submit_spot(self, spot, credentials): # TODO: Implement. Spotting to ZLOTA is supported via POST, see https://ontheair.nz/api diff --git a/static/js/add-spot.js b/static/js/add-spot.js index 7ef87b5..dec7188 100644 --- a/static/js/add-spot.js +++ b/static/js/add-spot.js @@ -196,7 +196,7 @@ function addSpot() { data: JSON.stringify({spot, handling}), contentType: 'application/json', type: 'POST', - timeout: 10000, + timeout: 60000, success: async function (result) { // Reset CAPTCHA for next use if (window._recaptchaWidgetId !== undefined) { diff --git a/telnetserver/telnetserver.py b/telnetserver/telnetserver.py index 06da362..2e08282 100644 --- a/telnetserver/telnetserver.py +++ b/telnetserver/telnetserver.py @@ -115,7 +115,7 @@ class TelnetServer: break input_buffer, command = self._consume_input(input_buffer, data) - if command == "exit": + if command == "exit" or command == "bye": writer.write(b"Goodbye!\r\n") await writer.drain() # Exit the while read loop, this will disconnect the client. diff --git a/templates/add-spot.html b/templates/add-spot.html index 4a974ec..156136c 100644 --- a/templates/add-spot.html +++ b/templates/add-spot.html @@ -3,13 +3,15 @@
@@ -136,7 +138,7 @@ const ALLOW_UPSTREAM_SPOTTING = {% raw safe_json_dumps(web_ui_options["allow_upstream_spotting"]) %}; - + diff --git a/templates/alerts.html b/templates/alerts.html index 9de9b11..5523f16 100644 --- a/templates/alerts.html +++ b/templates/alerts.html @@ -87,7 +87,7 @@
- + diff --git a/templates/bands.html b/templates/bands.html index 5f01654..25aa23a 100644 --- a/templates/bands.html +++ b/templates/bands.html @@ -82,8 +82,8 @@ const BANDS = {% raw safe_json_dumps(options["bands"]) %}; - - + + diff --git a/templates/base.html b/templates/base.html index 9ee2677..530b8ea 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,6 +1,6 @@ {% extends "skeleton.html" %} {% block head_extra %} - + @@ -16,10 +16,10 @@ window.fetchEventSource = fetchEventSource; - - - - + + + + {% end %} {% block body %}
diff --git a/templates/conditions.html b/templates/conditions.html index b9d0e7f..25dbbf2 100644 --- a/templates/conditions.html +++ b/templates/conditions.html @@ -284,7 +284,7 @@
- + diff --git a/templates/help/usage/telnet.html b/templates/help/usage/telnet.html index e11235e..07b7fc8 100644 --- a/templates/help/usage/telnet.html +++ b/templates/help/usage/telnet.html @@ -7,9 +7,11 @@ data into a traditional desktop logging program. The data Spothole produces is compatible with DXSpider, and therefore many loggers should accept it with no problems.

This is a relatively new feature however, so if you run into problems, please do let me know.

-

The one caveat with the Spothole telnet server is that it does not support any commands, other than exit. +

The one caveat with the Spothole telnet server is that it does not support any commands, other than + exit or bye. You cannot therefore issue commands to retrieve past entries, set up filters, etc. If you need to filter the data, - many desktop logging programs support this client-side, so hopefully this is not too big of an obstacle. + many desktop logging programs support this client-side, so hopefully this is not too big of an obstacle. You also + can't currently send spots to Spothole using the Telnet server.

To access Spothole via telnet, set up your logger with the server address {{ telnet_server_address }} and port {{ telnet_server_port }}. You can also access it from a terminal with telnet {{ diff --git a/templates/help/usage/web.html b/templates/help/usage/web.html index 870f02e..0bd89a0 100644 --- a/templates/help/usage/web.html +++ b/templates/help/usage/web.html @@ -127,7 +127,10 @@

Add Spot

-

The Add Spot page allows you to add a new spot into the Spothole system directly. Currently, this stays within - Spothole and is not sent "upstream" to DX clusters, POTA, SOTA etc. regardless of any activity you select.

+

The Add Spot page allows you to add a new spot into the Spothole system directly. If the server allows it, a "Send + spot to..." checkbox will also appear, allowing you to send your spot "upstream" to the DX cluster that Spothole + receives spots from. Spothole will log in to the cluster using the callsign you entered in "Your Call", so if you + are already connected to the same cluster node with that callsign, you may need to use a different SSID. Sending + spots upstream to POTA, SOTA etc. is not yet supported.

{% end %} diff --git a/templates/map.html b/templates/map.html index efdae6e..96d3745 100644 --- a/templates/map.html +++ b/templates/map.html @@ -115,8 +115,8 @@ const CARTODB_API_KEY = "{{ web_ui_options.get('cartodb_api_key', '') }}"; - - + + diff --git a/templates/spots.html b/templates/spots.html index dedd8a3..a0cf0e4 100644 --- a/templates/spots.html +++ b/templates/spots.html @@ -127,8 +127,8 @@ - - + + diff --git a/templates/status.html b/templates/status.html index 6883a97..920d60b 100644 --- a/templates/status.html +++ b/templates/status.html @@ -96,7 +96,7 @@ - +