Telnet clients disconnecting shouldn't log an exception

core/utils.py

@@ -156,3 +156,13 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
     except (KeyError, ValueError):
         # Unknown callsign, can't look anything up, return a Callsign object with basic data so that gets cached
         return Callsign(call=callsign)
+
+
+def decode_telnet_bytes(data: bytes) -> str:
+    """Decode a line of text received from a telnet connection, which might be UTF-8 or Latin-1 depending on the
+    whims of the server owner."""
+
+    try:
+        return data.decode("utf-8")
+    except UnicodeDecodeError:
+        return data.decode("latin-1")

data/activities.py

@@ -10,7 +10,7 @@ ACTIVITIES: dict[ActivityName, Activity] = {
         refs_globally_unique=False,
         # No sensible way to determine *which* contest, but if we set comment_names=["CONTEST"] then at least
         # any spots with "contest" in the comment will get allocated to this activity.
-        comment_names=["CONTEST"],
+        comment_names=["CONTEST", "TEST"],
         icon="fa-trophy",
         alerts_possible=True,
     ),
@@ -92,7 +92,7 @@ ACTIVITIES: dict[ActivityName, Activity] = {
         description="Summits on the Air",
         sig_type=ActivityType.ADVENTURE,
         has_refs=True,
-        refs_globally_unique=False,
+        refs_globally_unique=True,
         ref_type=ActivityRefType.SUMMIT,
         ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
         comment_names=["SOTA"],
@@ -150,9 +150,9 @@ ACTIVITIES: dict[ActivityName, Activity] = {
         description="Islands on the Air",
         sig_type=ActivityType.ADVENTURE,
         has_refs=True,
-        refs_globally_unique=False,
+        refs_globally_unique=True,
         ref_type=ActivityRefType.ISLAND,
-        ref_regex=r"[A-Z]{2}\-\d{3}",
+        ref_regex=r"(?:AF|AN|AS|EU|NA|OC|SA)\-\d{3}",
         comment_names=["IOTA"],
         icon="fa-book-atlas",
     ),
@@ -163,7 +163,7 @@ ACTIVITIES: dict[ActivityName, Activity] = {
         has_refs=True,
         refs_globally_unique=False,
         ref_type=ActivityRefType.ISLAND,
-        ref_regex=r"(([A-Z]{2}\-\d{3})|([A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}))",
+        ref_regex=r"(?:(?:[A-Z]{2}\-\d{3})|(?:[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}))",
         comment_names=[],
         icon="fa-person-hiking",
     ),
@@ -386,7 +386,7 @@ ACTIVITIES: dict[ActivityName, Activity] = {
         # FEA references are technically [DE]\-\d{4}(\.\d)? but spotters always seem to miss out the D- or E-
         # prefix and just use FEA-1234 or FEA 1234, so allow for that. The FEA activity ref data provider adds both
         # forms to the database.
-        ref_regex=r"([DE]|FEA)[\- ]\d{4}(\.\d)?",
+        ref_regex=r"(?:[DE]|FEA)[\- ]\d{4}(?:\.\d)?",
         comment_names=["FEA"],
         icon="fa-house-flood-water",
         region_flag="🇪🇸",

data/spot.py

@@ -359,20 +359,20 @@ class Spot:
             # These are common on cluster spots and can provide grid references in preference to e.g. QRZ lookup, as well as
             # being the only source we have for propagation mode. Brace for nightmare regex from hell.
             if self.comment:
-                grid_mode_grid_match = re.search(
+                grid_match = re.search(
                     r"\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b",
                     self.comment,
                 )
-                if grid_mode_grid_match:
+                if grid_match:
                     # regex matches, so extract grids:
                     if not self.de_grid:
-                        self.de_grid = grid_mode_grid_match.group(1).upper()
+                        self.de_grid = grid_match.group(1).upper()
                     if not self.dx_grid:
-                        self.dx_grid = grid_mode_grid_match.group(4).upper()
+                        self.dx_grid = grid_match.group(4).upper()
                         self.dx_location_source = LocationSourceForSpot.GRID
 
                     # And extract propagation mode (group 2 for <...>, group 3 for (...)):
-                    mode_tag = (grid_mode_grid_match.group(2) or grid_mode_grid_match.group(3) or "").upper()
+                    mode_tag = (grid_match.group(2) or grid_match.group(3) or "").upper()
                     if mode_tag and not self.propagation_mode:
                         if mode_tag in PROPAGATION_MODES:
                             self.propagation_mode = PROPAGATION_MODES[mode_tag]
@@ -380,6 +380,31 @@ class Spot:
                             self.propagation_mode = mode_tag
                             logger.info(f"Seen a new propagation mode tag not yet in the system: {mode_tag}")
 
+            # Parse "de_grid -> dx_grid" structures from the comment
+            if self.comment:
+                grid_match = re.search(
+                    r"\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b",
+                    self.comment,
+                )
+                if grid_match:
+                    # regex matches, so extract grids:
+                    if not self.de_grid:
+                        self.de_grid = grid_match.group(1).upper()
+                    if not self.dx_grid:
+                        self.dx_grid = grid_match.group(2).upper()
+                        self.dx_location_source = LocationSourceForSpot.GRID
+
+            # Parse 6-digit grid structures from the comment if the frequency is VHF or above, as it's common to see
+            # VHF+ operators spotted with their grid
+            if self.comment and self.freq and self.freq > 30000000:
+                grid_match = re.search(
+                    r"\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b",
+                    self.comment,
+                )
+                if grid_match and not self.dx_grid:
+                    self.dx_grid = grid_match.group(1).upper()
+                    self.dx_location_source = LocationSourceForSpot.GRID
+
             # Set activities based on propagation mode
             if self.propagation_mode == "Satellite" and not self.sig:
                 self.sig = ActivityName.SATELLITE
@@ -409,20 +434,6 @@ class Spot:
                         self.sig = ActivityName.DXPEDITION
                         break
 
-            # Parse "de_grid -> dx_grid" structures from the comment
-            if self.comment:
-                grid_mode_grid_match = re.search(
-                    r"\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b",
-                    self.comment,
-                )
-                if grid_mode_grid_match:
-                    # regex matches, so extract grids:
-                    if not self.dx_grid:
-                        self.dx_grid = grid_mode_grid_match.group(1).upper()
-                        self.dx_location_source = LocationSourceForSpot.GRID
-                    if not self.de_grid:
-                        self.de_grid = grid_mode_grid_match.group(2).upper()
-
             # DX Grid to lat/lon and vice versa in case one is missing
             if self.dx_grid and (not self.dx_latitude or not self.dx_longitude):
                 try:

providers/spot/dxcluster.py

@@ -10,6 +10,7 @@ import telnetlib3
 from core.config import SERVER_OWNER_CALLSIGN
 from data.spot import Spot
 from providers.spot.spot_provider import SpotProvider
+from core.utils import decode_telnet_bytes
 
 logger = logging.getLogger(__name__)
 
@@ -96,7 +97,7 @@ class DXCluster(SpotProvider):
                 try:
                     # Check new telnet info against regular expression
                     telnet_output = self._telnet.read_until("\n".encode("latin-1"))
-                    match = self._spot_line_pattern.match(telnet_output.decode("latin-1"))
+                    match = self._spot_line_pattern.match(decode_telnet_bytes(telnet_output))
                     if match:
                         spot_time = datetime.strptime(match.group(5), "%H%MZ").replace(tzinfo=pytz.UTC)
                         spot_datetime = datetime.combine(

providers/spot/gma.py

@@ -98,24 +98,27 @@ class GMA(HTTPSpotProvider):
                             and ref_response.text != "\n"
                         ):
                             ref_info = ref_response.json()
-                            # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
+                            if spot.sig_refs and ref_info and "reftype" in ref_info:
-                            # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
-                            # and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry
-                            # to determine if it's a SOTA summit.
-                            if (
-                                spot.sig_refs
-                                and ref_info
-                                and "reftype" in ref_info
-                                and ref_info["reftype"] not in [ActivityName.POTA, ActivityName.WWFF]
-                                and (
-                                    ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info["sota"] == ""
-                                )
-                            ):
                                 match ref_info["reftype"]:
                                     case "Summit":
-                                        spot.sig_refs[0].sig = ActivityName.GMA
+                                        # Summits are a bit complicated, they can be SOTA or GMA depending on the
-                                        spot.sig_refs[0].ref_type = ActivityRefType.SUMMIT
+                                        # separate "sota" field:
-                                        spot.sig = ActivityName.GMA
+                                        if "sota" in ref_info and ref_info["sota"] != "":
+                                            spot.sig_refs[0].sig = ActivityName.SOTA
+                                            spot.sig_refs[0].ref_type = ActivityRefType.SUMMIT
+                                            spot.sig = ActivityName.SOTA
+                                        else:
+                                            spot.sig_refs[0].sig = ActivityName.GMA
+                                            spot.sig_refs[0].ref_type = ActivityRefType.SUMMIT
+                                            spot.sig = ActivityName.GMA
+                                    case "POTA":
+                                        spot.sig_refs[0].sig = ActivityName.POTA
+                                        spot.sig_refs[0].ref_type = ActivityRefType.PARK
+                                        spot.sig = ActivityName.POTA
+                                    case "WWFF":
+                                        spot.sig_refs[0].sig = ActivityName.WWFF
+                                        spot.sig_refs[0].ref_type = ActivityRefType.PARK
+                                        spot.sig = ActivityName.WWFF
                                     case "IOTA Island":
                                         spot.sig_refs[0].sig = ActivityName.IOTA
                                         spot.sig_refs[0].ref_type = ActivityRefType.ISLAND

providers/spot/parksnpeaks.py

@@ -68,21 +68,33 @@ class ParksNPeaks(HTTPSpotProvider):
                 # Record activity information
                 activity = source_spot["actClass"].upper()
                 ref_id = source_spot["actSiteID"]
+
                 if activity:
                     spot.sig = activity
-                    activity_refs = []
+
                     if ref_id:
                         activity_refs = [
                             ActivityRef(
-                                id=source_spot["actSiteID"],
+                                id=ref_id,
                                 sig=activity,
+                                # Free text location is not present in all spots, so only add it if it's set
+                                name=source_spot["actLocation"]
+                                if "actLocation" in source_spot and source_spot["actLocation"] != ""
+                                else None,
                             )
                         ]
                         spot.sig_refs = activity_refs
 
-                    # Free text location is not present in all spots, so only add it if it's set
+                    else:
-                    if "actLocation" in source_spot and source_spot["actLocation"] != "":
+                        # If no actSiteID is set, e.g. because actClass is "QRP", sometimes we still have an actLocation
-                        activity_refs[0].name = source_spot["actLocation"]
+                        # which is free text like "SOTA G/SC-001". If we have that, and not a normal comment field, use
+                        # that location as the comment field so the information doesn't get lost.
+                        if (
+                            "actLocation" in source_spot
+                            and source_spot["actLocation"] != ""
+                            and ("actComments" not in source_spot or source_spot["actComments"] == "")
+                        ):
+                            spot.comment = source_spot["actLocation"]
 
                     # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
                     if activity not in [
@@ -95,8 +107,11 @@ class ParksNPeaks(HTTPSpotProvider):
                         ActivityName.KRMNPA,
                         ActivityName.SANPCPA,
                         ActivityName.LLOTA,
+                        ActivityName.QRP,
                     ]:
-                        logger.warning(f"PNP spot found with activity {activity}, developer needs to add support for this!")
+                        logger.warning(
+                            f"PNP spot found with activity {activity}, developer needs to add support for this!"
+                        )
 
                 # Add new spot to the list
                 new_spots.append(spot)

providers/spot/rbn.py

@@ -10,6 +10,7 @@ import telnetlib3
 from core.config import SERVER_OWNER_CALLSIGN
 from data.spot import Spot
 from providers.spot.spot_provider import SpotProvider
+from core.utils import decode_telnet_bytes
 
 logger = logging.getLogger(__name__)
 
@@ -81,7 +82,7 @@ class RBN(SpotProvider):
                 try:
                     # Check new telnet info against regular expression
                     telnet_output = self._telnet.read_until("\n".encode("latin-1"))
-                    match = self._LINE_PATTERN.match(telnet_output.decode("latin-1"))
+                    match = self._LINE_PATTERN.match(decode_telnet_bytes(telnet_output))
                     if match:
                         spot_time = datetime.strptime(match.group(5), "%H%MZ").replace(tzinfo=pytz.UTC)
                         spot_datetime = datetime.combine(

telnetserver/telnetserver.py

@@ -123,13 +123,19 @@ class TelnetServer:
 
         except asyncio.CancelledError:
             pass
+        except (TimeoutError, ConnectionResetError):
+            logger.debug("Telnet client connection timed out or was reset by client")
         except Exception:
             logger.exception("Exception handling telnet client")
         finally:
             logger.debug("Telnet client disconnected")
             self._clients.remove(writer)
-            writer.close()
+            try:
-            await writer.wait_closed()
+                writer.close()
+                await writer.wait_closed()
+            except (TimeoutError, ConnectionResetError, OSError):
+                # Socket already closed or timed out, don't care as we were getting rid of it anyway
+                pass
 
     def stop(self):
         """Stops the telnet server"""

templates/add_spot.html

@@ -77,7 +77,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1789894090"></script>
+<script src="/static/js/add-spot.js?v=1789933862"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -85,7 +85,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1789894090"></script>
+<script src="/static/js/alerts.js?v=1789933862"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -76,8 +76,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1789894090"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1789933862"></script>
-<script src="/static/js/bands.js?v=1789894090"></script>
+<script src="/static/js/bands.js?v=1789933862"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1789894090" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1789933862" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -16,10 +16,10 @@
     window.fetchEventSource = fetchEventSource;
 </script>
 
-<script src="/static/js/utils.js?v=1789894090"></script>
+<script src="/static/js/utils.js?v=1789933862"></script>
-<script src="/static/js/ui-ham.js?v=1789894090"></script>
+<script src="/static/js/ui-ham.js?v=1789933862"></script>
-<script src="/static/js/geo.js?v=1789894090"></script>
+<script src="/static/js/geo.js?v=1789933862"></script>
-<script src="/static/js/common.js?v=1789894090"></script>
+<script src="/static/js/common.js?v=1789933862"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1789894090"></script>
+<script src="/static/js/conditions.js?v=1789933862"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -113,8 +113,8 @@
     const CARTODB_API_KEY = "{{ web_ui_options.get('cartodb_api_key', '') }}";
 </script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1789894090"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1789933861"></script>
-<script src="/static/js/map.js?v=1789894090"></script>
+<script src="/static/js/map.js?v=1789933861"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -125,8 +125,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1789894090"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1789933861"></script>
-<script src="/static/js/spots.js?v=1789894090"></script>
+<script src="/static/js/spots.js?v=1789933861"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -96,7 +96,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1789894090"></script>
+<script src="/static/js/status.js?v=1789933862"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");