diff --git a/core/activity_lookup_helper.py b/core/activity_lookup_helper.py index 6ec1f5a..88003af 100644 --- a/core/activity_lookup_helper.py +++ b/core/activity_lookup_helper.py @@ -12,7 +12,7 @@ from data.activity_ref import ActivityRef logger = logging.getLogger(__name__) -def get_activity_ref_info(activity_name: str, ref_id: str) -> ActivityRef | None: +def get_activity_ref_info(activity_name: str | None, ref_id: str | None) -> ActivityRef | None: """Look up details of an activity reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in an activity name and a reference ID (both strings) and returns an ActivityRef object populated with as much data as we can find. This makes use of activity ref data in the data store, live lookups from the web, or just @@ -102,12 +102,14 @@ def get_activity_ref_info(activity_name: str, ref_id: str) -> ActivityRef | None # the best result. iota_lookup = get_activity_ref_info(ActivityName.IOTA, ref_id) gma_lookup = get_activity_ref_info(ActivityName.GMA, ref_id) - for key, value in iota_lookup.__dict__.items(): - if value is not None and activity_ref.__dict__.get(key) is None: - activity_ref.__dict__[key] = value - for key, value in gma_lookup.__dict__.items(): - if value is not None and activity_ref.__dict__.get(key) is None: - activity_ref.__dict__[key] = value + if iota_lookup: + for key, value in iota_lookup.__dict__.items(): + if value is not None and activity_ref.__dict__.get(key) is None: + activity_ref.__dict__[key] = value + if gma_lookup: + for key, value in gma_lookup.__dict__.items(): + if value is not None and activity_ref.__dict__.get(key) is None: + activity_ref.__dict__[key] = value activity_ref.ref_type = ActivityRefType.ISLAND return activity_ref diff --git a/core/activity_utils.py b/core/activity_utils.py index c4efab3..9e6a588 100644 --- a/core/activity_utils.py +++ b/core/activity_utils.py @@ -3,7 +3,7 @@ from data.activities import ACTIVITIES from data.activity import Activity -def get_activity_by_name(name: str) -> Activity | None: +def get_activity_by_name(name: str | None) -> Activity | None: """Utility function to resolve an arbitrary, case-insensitive activity name string (e.g. from a spot comment, a provider, or an API request) to the matching known Activity. Returns None if no match is found.""" diff --git a/core/call_lookup_helper.py b/core/call_lookup_helper.py index 75f8a87..5c9d54e 100644 --- a/core/call_lookup_helper.py +++ b/core/call_lookup_helper.py @@ -5,12 +5,12 @@ from data.callsign import Callsign from data.lookup_credentials import LookupCredentials -def get_call_info(callsign: str, lookup_credentials: LookupCredentials | None) -> Callsign: +def get_call_info(callsign: str | None, lookup_credentials: LookupCredentials | None) -> Callsign: """Utility method to get the best set of data for a callsign as we can, using all enabled providers. lookup_credentials is an optional object that carries the user's QRZ.com/HamQTH credentials, if they provided them, to enable lookup using those providers.""" - callsign_data = Callsign(call=callsign) + callsign_data = Callsign(call=callsign or "") # First check our input looks like a real callsign if callsign and re.match(r"^[A-Za-z0-9/\-]*$", callsign): diff --git a/core/data_providers.py b/core/data_providers.py index a72d37f..e5026a1 100644 --- a/core/data_providers.py +++ b/core/data_providers.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging import threading import time +from collections.abc import Sequence from typing import TYPE_CHECKING from core.config import config, create_provider_from_config @@ -48,7 +49,7 @@ class DataProviders: @staticmethod def start_providers( - providers: list[ + providers: Sequence[ SpotProvider | AlertProvider | SolarConditionsProvider | StaticDataProvider | ActivityRefDataProvider | CallsignDataProvider ], provider_type: str, @@ -108,13 +109,13 @@ class DataProviders: logger.exception("Exception stopping provider") threads = [threading.Thread(target=stop_provider, args=(p,), daemon=True) for p in all_providers] - for t in threads: - t.start() + for thread in threads: + thread.start() deadline = time.monotonic() + 15 - for t in threads: - t.join(timeout=max(0.0, deadline - time.monotonic())) - still_running = [t for t in threads if t.is_alive()] + for thread in threads: + thread.join(timeout=max(0.0, deadline - time.monotonic())) + still_running = [thread for thread in threads if thread.is_alive()] if still_running: logger.warning("Some threads did not stop in time!") diff --git a/core/data_store.py b/core/data_store.py index 8a2c542..fb6d3be 100644 --- a/core/data_store.py +++ b/core/data_store.py @@ -14,7 +14,8 @@ from core.single_object_data_cache import SingleObjectDataCache from data.solar_conditions import SolarConditions if TYPE_CHECKING: - # Deferred to avoid a circular import: data.alert and data.spot both import core.data_store at module level. + # Can't find a way to resolve the circular dependency on types but apparently this is a way of managing that + # while still having type safety in method definitions. from data.alert import Alert from data.spot import Spot @@ -33,52 +34,109 @@ class DataStore: self._MAX_ALERT_COUNT = 100000 self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300 self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60 - # Caches - self.alerts: LiveDataCache[Alert] | None = None - self.spots: LiveDataCache[Spot] | None = None - self.callsign_data_countryfiles: diskcache.Cache | None = None - self.callsign_data_clublogxml: diskcache.Cache | None = None - self.callsign_data_clublogapi: diskcache.Cache | None = None - self.callsign_data_qrz: diskcache.Cache | None = None - self.callsign_data_hamqth: diskcache.Cache | None = None - self.dxcc_data: diskcache.Cache | None = None + # Caches. These are all created by setup(), which must be called before use; they are None only in the window + # between construction of this (global, single-instance) object and that call. + self._alerts: LiveDataCache[Alert] | None = None + self._spots: LiveDataCache[Spot] | None = None + self._callsign_data_countryfiles: diskcache.Cache | None = None + self._callsign_data_clublogxml: diskcache.Cache | None = None + self._callsign_data_clublogapi: diskcache.Cache | None = None + self._callsign_data_qrz: diskcache.Cache | None = None + self._callsign_data_hamqth: diskcache.Cache | None = None + self._dxcc_data: diskcache.Cache | None = None self.dxcc_lookup_by_call_regex: list[tuple[re.Pattern[str], Any]] = [] - self.activity_refs: diskcache.Cache | None = None - self.status: SingleObjectDataCache[dict[str, Any]] | None = None - self.solar_conditions: SingleObjectDataCache[SolarConditions] | None = None + self._activity_refs: diskcache.Cache | None = None + self._status: SingleObjectDataCache[dict[str, Any]] | None = None + self._solar_conditions: SingleObjectDataCache[SolarConditions] | None = None # ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be - # caches, they can just be straight objects + # caches, they can just be straight objects. Unlike the caches above, these genuinely may never be populated, + # if the corresponding static data provider isn't configured, so callers must handle None. self.cq_zone_data: geopandas.GeoDataFrame | None = None self.itu_zone_data: geopandas.GeoDataFrame | None = None + @property + def alerts(self) -> LiveDataCache[Alert]: + assert self._alerts is not None, "DataStore.setup() must be called before use" + return self._alerts + + @property + def spots(self) -> LiveDataCache[Spot]: + assert self._spots is not None, "DataStore.setup() must be called before use" + return self._spots + + @property + def callsign_data_countryfiles(self) -> diskcache.Cache: + assert self._callsign_data_countryfiles is not None, "DataStore.setup() must be called before use" + return self._callsign_data_countryfiles + + @property + def callsign_data_clublogxml(self) -> diskcache.Cache: + assert self._callsign_data_clublogxml is not None, "DataStore.setup() must be called before use" + return self._callsign_data_clublogxml + + @property + def callsign_data_clublogapi(self) -> diskcache.Cache: + assert self._callsign_data_clublogapi is not None, "DataStore.setup() must be called before use" + return self._callsign_data_clublogapi + + @property + def callsign_data_qrz(self) -> diskcache.Cache: + assert self._callsign_data_qrz is not None, "DataStore.setup() must be called before use" + return self._callsign_data_qrz + + @property + def callsign_data_hamqth(self) -> diskcache.Cache: + assert self._callsign_data_hamqth is not None, "DataStore.setup() must be called before use" + return self._callsign_data_hamqth + + @property + def dxcc_data(self) -> diskcache.Cache: + assert self._dxcc_data is not None, "DataStore.setup() must be called before use" + return self._dxcc_data + + @property + def activity_refs(self) -> diskcache.Cache: + assert self._activity_refs is not None, "DataStore.setup() must be called before use" + return self._activity_refs + + @property + def status(self) -> SingleObjectDataCache[dict[str, Any]]: + assert self._status is not None, "DataStore.setup() must be called before use" + return self._status + + @property + def solar_conditions(self) -> SingleObjectDataCache[SolarConditions]: + assert self._solar_conditions is not None, "DataStore.setup() must be called before use" + return self._solar_conditions + def setup(self) -> None: Path(CACHE_DIR).mkdir(parents=True, exist_ok=True) # For solar data and status data, we use a wrapper around disk cache where each cache contains only a single # object exposed to the wider application, and provides a store() method for callers to notify diskcache that # the object has changed and needs to be re-cached. - self.solar_conditions = SingleObjectDataCache(f"{CACHE_DIR}solar", SolarConditions()) - self.status = SingleObjectDataCache(f"{CACHE_DIR}status", {}) + self._solar_conditions = SingleObjectDataCache(f"{CACHE_DIR}solar", SolarConditions()) + self._status = SingleObjectDataCache(f"{CACHE_DIR}status", {}) # Standard disk cache for static reference and activity ref data. Separate provider threads will repopulate # these on a regular basis but there's no need for a TTL since old data is better than no data. - self.dxcc_data = diskcache.Cache(f"{CACHE_DIR}dxcc_data") + self._dxcc_data = diskcache.Cache(f"{CACHE_DIR}dxcc_data") self.regenerate_call_regex_to_dxcc_entity_map() # For activity reference data specifically, we need to key on both activity *and* reference, and trying to do # two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an # ugly "activity:ref" syntax for keys to keep it a single level. - self.activity_refs = diskcache.Cache(f"{CACHE_DIR}activity_refs") + self._activity_refs = diskcache.Cache(f"{CACHE_DIR}activity_refs") logger.info(f"Loaded data for {len(self.activity_refs)} activity references.") # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup. # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign # we've seen, so we rely on them timing out and this triggering another lookup. - self.callsign_data_countryfiles = diskcache.Cache(f"{CACHE_DIR}callsign_data_countryfiles") - self.callsign_data_clublogxml = diskcache.Cache(f"{CACHE_DIR}callsign_data_clublogxml") - self.callsign_data_clublogapi = diskcache.Cache(f"{CACHE_DIR}callsign_data_clublogapi") - self.callsign_data_qrz = diskcache.Cache(f"{CACHE_DIR}callsign_data_qrz") - self.callsign_data_hamqth = diskcache.Cache(f"{CACHE_DIR}callsign_data_hamqth") + self._callsign_data_countryfiles = diskcache.Cache(f"{CACHE_DIR}callsign_data_countryfiles") + self._callsign_data_clublogxml = diskcache.Cache(f"{CACHE_DIR}callsign_data_clublogxml") + self._callsign_data_clublogapi = diskcache.Cache(f"{CACHE_DIR}callsign_data_clublogapi") + self._callsign_data_qrz = diskcache.Cache(f"{CACHE_DIR}callsign_data_qrz") + self._callsign_data_hamqth = diskcache.Cache(f"{CACHE_DIR}callsign_data_hamqth") unique_keys = set() for c in [ self.callsign_data_countryfiles, @@ -93,7 +151,7 @@ class DataStore: # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We # specifically load these caches *last* so that any activity ref and callsign data is already loaded from disk # cache before the spots and alerts are live in the system. - self.spots = LiveDataCache( + self._spots = LiveDataCache( maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE, snapshot_dir=f"{CACHE_DIR}spots", @@ -101,7 +159,7 @@ class DataStore: ) logger.info(f"Loaded {len(self.spots.keys())} spots from a previous run.") - self.alerts = LiveDataCache( + self._alerts = LiveDataCache( maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE, snapshot_dir=f"{CACHE_DIR}alerts", diff --git a/core/url_data_cache.py b/core/url_data_cache.py index 10e71ce..f7c98c2 100644 --- a/core/url_data_cache.py +++ b/core/url_data_cache.py @@ -2,8 +2,7 @@ import threading from datetime import timedelta from typing import Any -from requests import Response -from requests_cache import CachedSession +from requests_cache import AnyResponse, CachedSession from core.data_store import CACHE_DIR @@ -22,8 +21,8 @@ class URLDataCache(CachedSession): expire_after=timedelta(days=1), allowable_codes=(200, 400, 401, 403, 404), ) - self._lock = threading.Lock() + self._get_lock = threading.Lock() - def get(self, *args: Any, **kwargs: Any) -> Response: - with self._lock: + def get(self, *args: Any, **kwargs: Any) -> AnyResponse: + with self._get_lock: return super().get(*args, **kwargs) diff --git a/data/spot.py b/data/spot.py index 2b5f5e7..ef81b32 100644 --- a/data/spot.py +++ b/data/spot.py @@ -299,7 +299,7 @@ class Spot: # Now look to see if that activity name was followed by something that looks like a reference ID # for that activity. If so, add that to the sig_refs list for this spot. found_activity_info = get_activity_by_name(found_activity) - if found_activity_info and found_activity_info.has_refs and found_activity_info.ref_regex: + if found_activity and found_activity_info and found_activity_info.has_refs and found_activity_info.ref_regex: ref_matches = re.finditer( r"(^|\W)" + found_activity + r"([ -])(" + found_activity_info.ref_regex + r")($|\W)", self.comment, @@ -314,19 +314,19 @@ class Spot: # name, but where the activity reference is unique-looking enough that we can't confuse it with any other # activity. if self.comment: - for activity in ACTIVITIES.values(): - if activity.has_refs and activity.refs_globally_unique and activity.ref_regex: + for candidate_activity in ACTIVITIES.values(): + if candidate_activity.has_refs and candidate_activity.refs_globally_unique and candidate_activity.ref_regex: ref_matches = re.finditer( - r"(^|\W)(" + activity.ref_regex + r")($|\W)", self.comment, re.IGNORECASE + r"(^|\W)(" + candidate_activity.ref_regex + r")($|\W)", self.comment, re.IGNORECASE ) for ref_match in ref_matches: # First of all, if we haven't got an activity for this spot set yet, now we have. This # covers things like cluster spots where the comment is just "OHFF-1234", now we know # it's WWFF. if not self.sig: - self.sig = activity.name + self.sig = candidate_activity.name self._append_activity_ref_if_missing( - ActivityRef(id=ref_match.group(2).upper(), sig=activity.name) + ActivityRef(id=ref_match.group(2).upper(), sig=candidate_activity.name) ) # Fetch activity data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a @@ -473,12 +473,14 @@ class Spot: self.dx_latitude = dx_call_info.latitude self.dx_longitude = dx_call_info.longitude self.dx_grid = dx_call_info.grid - self.dx_location_source = dx_call_info.location_source + self.dx_location_source = ( + LocationSourceForSpot(dx_call_info.location_source) if dx_call_info.location_source else None + ) # Determine a "QTH" string. If we have an activity ref, pick the first one and turn it into a suitable # string, otherwise see what they have set on an online lookup service. if self.sig_refs: - qth = self.sig_refs[0].id + qth = self.sig_refs[0].id or "" if self.sig_refs[0].name: qth += f" {self.sig_refs[0].name}" self.dx_qth = qth @@ -499,8 +501,8 @@ class Spot: # DXCC lookup from callsign if nothing else has provided it if self.dx_call and not self.dx_dxcc_id: - for regex, entity_code in DATA_STORE.dxcc_lookup_by_call_regex: - if regex.pattern and regex.match(self.dx_call): + for dxcc_regex, entity_code in DATA_STORE.dxcc_lookup_by_call_regex: + if dxcc_regex.pattern and dxcc_regex.match(self.dx_call): self.dx_dxcc_id = entity_code break if self.dx_dxcc_id and not self.dx_flag: @@ -547,7 +549,7 @@ class Spot: def _append_activity_ref_if_missing(self, new_activity_ref: ActivityRef) -> None: """Append an activity ref to the list, so long as it's not already there.""" - new_activity_ref.id = new_activity_ref.id.strip().upper() + new_activity_ref.id = (new_activity_ref.id or "").strip().upper() new_activity_ref.sig = new_activity_ref.sig.strip().upper() if new_activity_ref.id == "": return diff --git a/providers/activityrefdata/pnp_kml_activity_ref_data_provider.py b/providers/activityrefdata/pnp_kml_activity_ref_data_provider.py index 1c771c2..5ed4371 100644 --- a/providers/activityrefdata/pnp_kml_activity_ref_data_provider.py +++ b/providers/activityrefdata/pnp_kml_activity_ref_data_provider.py @@ -4,6 +4,9 @@ from typing import Any import requests from fastkml import kml +from fastkml.containers import Document, Folder +from fastkml.features import Placemark +from fastkml.geometry import Point from pyhamtools.locator import latlong_to_locator from core.enums import ActivityRefType @@ -26,13 +29,19 @@ class ParksNPeaksKMLActivityRefDataProvider(FileDownloadActivityRefDataProvider) def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]: new_data: list[ActivityRef] = [] - k = kml.KML.from_string(http_response.content) + # KML content may carry an XML encoding declaration, which lxml's parser (used internally here) refuses to + # accept as a decoded str, so bytes must be passed even though the type stub only declares str. + k = kml.KML.from_string(http_response.content) # type: ignore[arg-type] for document in k.features: - # noinspection unresolved-references + if not isinstance(document, Document): + continue for folder in document.features: - # noinspection unresolved-references + if not isinstance(folder, Folder): + continue for placemark in folder.features: + if not isinstance(placemark, Placemark): + continue description = placemark.description or "" match = self.REF_PATTERN.search(description) if not match: @@ -40,6 +49,9 @@ class ParksNPeaksKMLActivityRefDataProvider(FileDownloadActivityRefDataProvider) continue ref_id = match.group(0) + if not isinstance(placemark.geometry, Point): + # Not a point location (e.g. a boundary polygon) - skip it, we can't get a single lat/lon from it + continue longitude, latitude = placemark.geometry.x, placemark.geometry.y ref = ActivityRef( diff --git a/providers/alert/alert_provider.py b/providers/alert/alert_provider.py index f72a39e..79bbe82 100644 --- a/providers/alert/alert_provider.py +++ b/providers/alert/alert_provider.py @@ -39,6 +39,7 @@ class AlertProvider: self._add_alert(alert) def _add_alert(self, alert: Alert) -> None: + assert alert.id is not None, "infer_missing() always assigns an id" if not alert.expired(): self._alerts.set(alert.id, alert) diff --git a/providers/callsigndata/qrz.py b/providers/callsigndata/qrz.py index f3f5cea..aeb461b 100644 --- a/providers/callsigndata/qrz.py +++ b/providers/callsigndata/qrz.py @@ -135,7 +135,9 @@ class QRZ(APIQueryCallsignDataProvider): # functions can't deal with multiple calls this way. if isinstance(data, list): data = data[0] + assert isinstance(data, dict) callsign = data["call"] + assert isinstance(data, dict) # Get a name name = None diff --git a/providers/spot/parksnpeaks.py b/providers/spot/parksnpeaks.py index b130c46..7793dce 100644 --- a/providers/spot/parksnpeaks.py +++ b/providers/spot/parksnpeaks.py @@ -128,6 +128,8 @@ class ParksNPeaks(HTTPSpotProvider): raise ValueError( "Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account." ) + if not spot.freq: + raise RuntimeError("The Parks N Peaks API requires a frequency to be set.") ref_id = spot.sig_refs[0].id if spot.sig_refs else "" body = { "actClass": spot.sig or "", diff --git a/providers/spot/pota.py b/providers/spot/pota.py index 3fefeb1..f273aee 100644 --- a/providers/spot/pota.py +++ b/providers/spot/pota.py @@ -64,6 +64,8 @@ class POTA(HTTPSpotProvider): def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None: sig_ref = spot.sig_refs[0].id if spot.sig_refs else None if sig_ref: + if not spot.freq: + raise RuntimeError("The POTA API requires a frequency to be set.") body = { "activator": spot.dx_call, "spotter": spot.de_call, diff --git a/providers/spot/sota.py b/providers/spot/sota.py index f6f1d8e..a650c9f 100644 --- a/providers/spot/sota.py +++ b/providers/spot/sota.py @@ -94,21 +94,24 @@ class SOTA(HTTPSpotProvider): raise ValueError("SOTA API tokens are required. Please log into SOTA in order to spot to it.") sig_ref = spot.sig_refs[0].id if spot.sig_refs else "" if sig_ref: + if not spot.freq: + raise ValueError("SOTA API requires a frequency to be set.") + # Split reference into association and summit codes ref_split = sig_ref.split("/") # Figure out a valid mode. Borrowed this from PoLo :) # https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js - mode = spot.mode - if mode and mode not in self.VALID_MODES: - mode = "Data" + mode_str = spot.mode.value if spot.mode else "" + if spot.mode and spot.mode not in self.VALID_MODES: + mode_str = "Data" body = { "activatorCallsign": spot.dx_call, "associationCode": ref_split[0], "summitCode": ref_split[1], "frequency": spot.freq / 1000000.0, - "mode": mode or "", + "mode": mode_str, "callsign": spot.de_call, "comments": spot.comment or "", "type": "TEST", # todo replatce with NORMAL/QRT once testing complete diff --git a/providers/spot/spot_provider.py b/providers/spot/spot_provider.py index da050c7..50d3f05 100644 --- a/providers/spot/spot_provider.py +++ b/providers/spot/spot_provider.py @@ -37,12 +37,12 @@ class SpotProvider: # off to SSE listeners. spots = sorted(spots, key=lambda s: s.time if s and s.time else 0) for spot in spots: - if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time: + if datetime.fromtimestamp(spot.time or 0, pytz.UTC) > self.last_spot_time: # Fill in any blanks and add to the list spot.infer_missing() self._add_spot(spot) if spots: - self.last_spot_time = datetime.fromtimestamp(max(s.time for s in spots), pytz.UTC) + self.last_spot_time = datetime.fromtimestamp(max(s.time or 0 for s in spots), pytz.UTC) def _submit(self, spot: Spot) -> None: """Submit a single spot retrieved from the provider. This will be added to the list regardless of its age. Spots @@ -52,9 +52,10 @@ class SpotProvider: # Fill in any blanks and add to the list spot.infer_missing() self._add_spot(spot) - self.last_spot_time = datetime.fromtimestamp(spot.time, pytz.UTC) + self.last_spot_time = datetime.fromtimestamp(spot.time or 0, pytz.UTC) def _add_spot(self, spot: Spot) -> None: + assert spot.id is not None, "infer_missing() always assigns an id" if not spot.expired(): self._spots.set(spot.id, spot) diff --git a/providers/spot/tiles.py b/providers/spot/tiles.py index 263804b..7cc3335 100644 --- a/providers/spot/tiles.py +++ b/providers/spot/tiles.py @@ -90,22 +90,25 @@ class Tiles(HTTPSpotProvider): def submit_spot(self, spot: Spot, credentials: dict[str, str]) -> None: # Tiles on the air currently only supports *self* spots if spot.dx_call == spot.de_call: + if not spot.freq: + raise RuntimeError("The Tiles on the Air API requires a frequency to be set.") + # Figure out a valid mode. Borrowed this from PoLo :) # https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js if spot.mode: - mode = spot.mode - if mode not in self.VALID_MODES: - if mode == "OLIVIA": - mode = "Olivia" - elif mode == "JS8": - mode = "JS8Call" + mode_str: str = spot.mode.value + if spot.mode not in self.VALID_MODES: + if spot.mode == "OLIVIA": + mode_str = "Olivia" + elif spot.mode == "JS8": + mode_str = "JS8Call" else: - mode = "Other" + mode_str = "Other" body = { "call_sign": spot.dx_call, "frequency": str(spot.freq / 1000000.0), - "mode": mode or "", + "mode": mode_str or "", "grid": spot.dx_grid or "", "comment": spot.comment or "", "lat": spot.dx_latitude or None, diff --git a/telnetserver/telnetserver.py b/telnetserver/telnetserver.py index 3f29b20..d6aff8b 100644 --- a/telnetserver/telnetserver.py +++ b/telnetserver/telnetserver.py @@ -213,9 +213,9 @@ class TelnetServer: and everything must be renderable in ASCII.""" de_call = f"{callinfo.Callinfo.get_homecall(spot.de_call)[:6] + ':' if spot.de_call else '???:'!s:<7}" - frequency = f"{(spot.freq / 1000.0):10.1f}" + frequency = f"{((spot.freq or 0) / 1000.0):10.1f}" dx_call = f"{spot.dx_call!s:<12}" - comment = f"{spot.comment.encode('ascii', errors='ignore').decode()[:29]:<30}" + comment = f"{(spot.comment or '').encode('ascii', errors='ignore').decode()[:29]:<30}" if spot.time: timestamp = datetime.fromtimestamp(spot.time, tz=pytz.utc).strftime("%H%M") + "Z" else: diff --git a/webserver/handlers/api/addspot.py b/webserver/handlers/api/addspot.py index cb62e55..3cbb1cd 100644 --- a/webserver/handlers/api/addspot.py +++ b/webserver/handlers/api/addspot.py @@ -144,13 +144,14 @@ class APISpotHandler(tornado.web.RequestHandler): return # Reject if activity ref format incorrect for activity + ref_regex = get_ref_regex_for_activity(spot.sig) if spot.sig else None if ( spot.sig and spot.sig_refs and len(spot.sig_refs) > 0 and spot.sig_refs[0].id - and get_ref_regex_for_activity(spot.sig) - and not re.match(get_ref_regex_for_activity(spot.sig), spot.sig_refs[0].id) + and ref_regex + and not re.match(ref_regex, spot.sig_refs[0].id) ): self.set_status(422) self.write( @@ -202,6 +203,8 @@ class APISpotHandler(tornado.web.RequestHandler): # Submit upstream if requested upstream_warning = None if submit_upstream and upstream_provider_name: + # spot.sig was already validated non-empty above, under the same submit_upstream/upstream_provider_name gate + assert spot.sig is not None provider = self._find_provider(upstream_provider_name, spot.sig) if provider: try: @@ -224,6 +227,8 @@ class APISpotHandler(tornado.web.RequestHandler): # we were but it failed, we should still add it to our database anyway. if not submit_upstream or upstream_warning: spot.infer_missing() + assert self._spots is not None, "initialize() must be called before post()" + assert spot.id is not None, "infer_missing() always assigns an id" self._spots.set(spot.id, spot) if upstream_warning: @@ -245,6 +250,7 @@ class APISpotHandler(tornado.web.RequestHandler): def _find_provider(self, provider_name: str, activity: str) -> SpotProvider | None: """Find an enabled provider by name that can submit spots for the given activity.""" + assert self._spot_providers is not None, "initialize() must be called before _find_provider()" for p in self._spot_providers: if p.enabled and p.name == provider_name and p.can_submit_spot(activity): return p diff --git a/webserver/handlers/api/alerts.py b/webserver/handlers/api/alerts.py index 30f6959..14517f9 100644 --- a/webserver/handlers/api/alerts.py +++ b/webserver/handlers/api/alerts.py @@ -51,13 +51,13 @@ class APIAlertsHandler(tornado.web.RequestHandler): # Fetch all alerts matching the query, then optionally enrich with online data credentials = extract_credentials(self.request.headers) - data = get_alert_list_with_filters(self._alerts, query_params) + assert self._alerts is not None, "initialize() must be called before get()" + alerts = get_alert_list_with_filters(self._alerts, query_params) fields = [f.strip() for f in query_params["fields"].split(",")] if "fields" in query_params else [] if credentials: - data = self._enrich(data, credentials) + alerts = self._enrich(alerts, credentials) # Filter for only the required fields, if necessary - if fields: - data = filter_fields(data, fields) + data: list[Alert] | list[dict[str, Any]] = filter_fields(alerts, fields) if fields else alerts self.write(safe_json_dumps(data)) self.set_status(200) except ValueError as e: @@ -104,6 +104,7 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler): # Register to handle new alerts arriving. The callback() method will get called with the new alert as an # argument. + assert self._sse_alert_broadcaster is not None, "initialize() must be called before open()" self._sse_alert_broadcaster.register(self) except Exception: @@ -113,6 +114,7 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler): def close(self) -> None: """When the user closes the socket, deregister ourselves from the alert broadcaster""" + assert self._sse_alert_broadcaster is not None, "initialize() must be called before close()" self._sse_alert_broadcaster.unregister(self) super().close() @@ -121,15 +123,15 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler): try: # If the new alert matches our param filters, send it to the client. If not, ignore it. + assert self._query_params is not None, "open() must be called before callback()" if alert_allowed_by_query(alert, self._query_params): # Add lookup data if we have credentials if self._credentials: alert = copy.deepcopy(alert) alert.infer_missing(self._credentials) # Filter fields returned if necessary - if self._fields: - alert = filter_fields([alert], self._fields)[0] - self.write_message(msg=safe_json_dumps(alert)) + output: Alert | dict[str, Any] = filter_fields([alert], self._fields)[0] if self._fields else alert + self.write_message(msg=safe_json_dumps(output)) except Exception: logger.exception("Exception in SSE callback, connection will be closed") self.close() @@ -151,7 +153,7 @@ def get_alert_list_with_filters(all_alerts: LiveDataCache, query: dict[str, str] alerts = sorted(alerts, key=lambda alert: alert.start_time if alert and alert.start_time else 0) alerts = list(filter(lambda alert: alert_allowed_by_query(alert, query), alerts)) if "limit" in query: - alerts = alerts[: int(query.get("limit"))] + alerts = alerts[: int(query["limit"])] return alerts @@ -162,11 +164,11 @@ def alert_allowed_by_query(alert: Alert, query: dict[str, str]) -> bool: for k in query: match k: case "received_since": - since = datetime.fromtimestamp(float(query.get(k)), pytz.UTC) + since = datetime.fromtimestamp(float(query[k]), pytz.UTC).timestamp() if not alert.received_time or alert.received_time <= since: return False case "max_duration": - max_duration = int(query.get(k)) + max_duration = int(query[k]) # Check the duration if end_time is provided. If end_time is not provided, assume the activation is # "short", i.e. it always passes this check. If dxpeditions_skip_max_duration_check is true and # the alert is a dxpedition, or contests_skip_max_duration_check and the alert is a contest, it also @@ -174,42 +176,42 @@ def alert_allowed_by_query(alert: Alert, query: dict[str, str]) -> bool: if ( alert.sig == ActivityName.DXPEDITION and "dxpeditions_skip_max_duration_check" in query - and query.get("dxpeditions_skip_max_duration_check").upper() == "TRUE" + and query["dxpeditions_skip_max_duration_check"].upper() == "TRUE" ): continue if ( alert.sig == ActivityName.CONTEST and "contests_skip_max_duration_check" in query - and query.get("contests_skip_max_duration_check").upper() == "TRUE" + and query["contests_skip_max_duration_check"].upper() == "TRUE" ): continue if alert.end_time and alert.start_time and alert.end_time - alert.start_time > max_duration: return False case "source": - sources = query.get(k).split(",") + sources = query[k].split(",") if not alert.source or alert.source not in sources: return False case "sig": # If a list of activities is provided, the alert must have an activity and it must match one of them. # The special activity "NO_SIG", when supplied in the list, matches alerts with no activity. - activities = query.get(k).split(",") + activities = query[k].split(",") include_no_activity = "NO_SIG" in activities if not alert.sig and not include_no_activity: return False if alert.sig and alert.sig not in activities: return False case "dx_continent": - dxconts = query.get(k).split(",") + dxconts = query[k].split(",") if not alert.dx_continent or alert.dx_continent not in dxconts: return False case "dx_call_includes": - dx_call_includes = query.get(k).strip() - if not alert.dx_call or dx_call_includes.upper() not in alert.dx_call.upper(): + dx_call_includes = query[k].strip() + if not alert.dx_calls or not any(dx_call_includes.upper() in c.upper() for c in alert.dx_calls if c): return False case "text_includes": - text_includes = query.get(k).strip() + text_includes = query[k].strip() if ( - (not alert.dx_call or text_includes.upper() not in alert.dx_call.upper()) + (not alert.dx_calls or not any(text_includes.upper() in c.upper() for c in alert.dx_calls if c)) and (not alert.comment or text_includes.upper() not in alert.comment.upper()) and (not alert.freqs_modes or text_includes.upper() not in alert.freqs_modes.upper()) ): diff --git a/webserver/handlers/api/dxstats.py b/webserver/handlers/api/dxstats.py index 52b365d..4c5eb13 100644 --- a/webserver/handlers/api/dxstats.py +++ b/webserver/handlers/api/dxstats.py @@ -37,8 +37,9 @@ class APIDxStatsHandler(tornado.web.RequestHandler): def get(self) -> None: try: + assert self._spots is not None, "initialize() must be called before get()" one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp() - counts = Counter() + counts: Counter[tuple[str, str, str]] = Counter() for key in self._spots.keys(): # noqa: SIM118 spot = self._spots.get(key) diff --git a/webserver/handlers/api/lookups.py b/webserver/handlers/api/lookups.py index 7f1fbc5..08e53db 100644 --- a/webserver/handlers/api/lookups.py +++ b/webserver/handlers/api/lookups.py @@ -85,9 +85,8 @@ class APILookupActivityRefHandler(tornado.web.RequestHandler): activity = str(query_params.get("sig")).upper() ref_id = str(query_params.get("id")).upper() if get_activity_by_name(activity): - if not get_ref_regex_for_activity(activity) or re.match( - get_ref_regex_for_activity(activity), ref_id - ): + ref_regex = get_ref_regex_for_activity(activity) + if not ref_regex or re.match(ref_regex, ref_id): data = populate_missing_activity_ref_info(ActivityRef(id=ref_id, sig=activity)) self.write(safe_json_dumps(data)) diff --git a/webserver/handlers/api/options.py b/webserver/handlers/api/options.py index 70274ab..422897d 100644 --- a/webserver/handlers/api/options.py +++ b/webserver/handlers/api/options.py @@ -34,8 +34,10 @@ class APIOptionsHandler(tornado.web.RequestHandler): def get(self) -> None: try: + assert self._status_data is not None, "initialize() must be called before get()" + # Build a map of activity name -> list of provider names that can submit spots for that activity - spot_submit_providers = {} + spot_submit_providers: dict[str, list[str]] = {} # Spothole v2.0 - disable this for now, API changes are in but this functionality is not ready yet. TODO # for provider in self._spot_providers: @@ -47,23 +49,15 @@ class APIOptionsHandler(tornado.web.RequestHandler): # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle # things that aren't even available. - spot_providers: list = [ - p["name"] for p in filter(lambda p: p["enabled"], self._status_data["spot_providers"]) - ] - alert_providers = [p["name"] for p in filter(lambda p: p["enabled"], self._status_data["alert_providers"])] - callsign_data_providers = [ - p["name"] - for p in filter( - lambda p: p["enabled"], - self._status_data["callsign_data_providers"], - ) - ] + spot_provider_configs: list[dict[str, Any]] = self._status_data["spot_providers"] + alert_provider_configs: list[dict[str, Any]] = self._status_data["alert_providers"] + callsign_data_provider_configs: list[dict[str, Any]] = self._status_data["callsign_data_providers"] + + spot_providers: list = [p["name"] for p in spot_provider_configs if p["enabled"]] + alert_providers = [p["name"] for p in alert_provider_configs if p["enabled"]] + callsign_data_providers = [p["name"] for p in callsign_data_provider_configs if p["enabled"]] spot_providers_enabled_by_default = [ - p["name"] - for p in filter( - lambda p: p["enabled"] and p["enabled_by_default_in_web_ui"], - self._status_data["spot_providers"], - ) + p["name"] for p in spot_provider_configs if p["enabled"] and p["enabled_by_default_in_web_ui"] ] # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from diff --git a/webserver/handlers/api/solar_conditions.py b/webserver/handlers/api/solar_conditions.py index 552bc83..c3a3044 100644 --- a/webserver/handlers/api/solar_conditions.py +++ b/webserver/handlers/api/solar_conditions.py @@ -28,6 +28,7 @@ class APISolarConditionsHandler(tornado.web.RequestHandler): def get(self) -> None: try: + assert self._solar_conditions is not None, "initialize() must be called before get()" self.write(self._solar_conditions.to_json()) self.set_status(200) self.set_header("Cache-Control", "no-store") diff --git a/webserver/handlers/api/spots.py b/webserver/handlers/api/spots.py index a6ce3f4..b35ba70 100644 --- a/webserver/handlers/api/spots.py +++ b/webserver/handlers/api/spots.py @@ -51,12 +51,12 @@ class APISpotsHandler(tornado.web.RequestHandler): # Fetch all spots matching the query, then optionally enrich with online data credentials = extract_credentials(self.request.headers) fields = [f.strip() for f in query_params["fields"].split(",")] if "fields" in query_params else [] - data = get_spot_list_with_filters(self._spots, query_params) + assert self._spots is not None, "initialize() must be called before get()" + spots = get_spot_list_with_filters(self._spots, query_params) if credentials: - data = self._enrich(data, credentials) + spots = self._enrich(spots, credentials) # Filter for only the required fields, if necessary - if fields: - data = filter_fields(data, fields) + data: list[Spot] | list[dict[str, Any]] = filter_fields(spots, fields) if fields else spots self.write(safe_json_dumps(data)) self.set_status(200) except ValueError as e: @@ -105,6 +105,7 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler): # Register to handle new spots arriving. The callback() method will get called with the new spot as an # argument. + assert self._sse_spot_broadcaster is not None, "initialize() must be called before open()" self._sse_spot_broadcaster.register(self) except Exception: @@ -114,6 +115,7 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler): def close(self) -> None: """When the user closes the socket, deregister ourselves from the spot broadcaster""" + assert self._sse_spot_broadcaster is not None, "initialize() must be called before close()" self._sse_spot_broadcaster.unregister(self) super().close() @@ -122,15 +124,15 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler): try: # If the new spot matches our param filters, send it to the client. If not, ignore it. + assert self._query_params is not None, "open() must be called before callback()" if spot_allowed_by_query(spot, self._query_params): # Add lookup data if we have credentials if self._credentials: spot = copy.deepcopy(spot) spot.infer_missing(self._credentials) # Filter fields returned if necessary - if self._fields: - spot = filter_fields([spot], self._fields)[0] - self.write_message(msg=safe_json_dumps(spot)) + output: Spot | dict[str, Any] = filter_fields([spot], self._fields)[0] if self._fields else spot + self.write_message(msg=safe_json_dumps(output)) except Exception: logger.exception("Exception in SSE callback, connection will be closed") self.close() @@ -152,7 +154,7 @@ def get_spot_list_with_filters(all_spots: LiveDataCache, query: dict[str, str]) spots = sorted(spots, key=lambda spot: spot.time if spot and spot.time else 0, reverse=True) spots = list(filter(lambda spot: spot_allowed_by_query(spot, query), spots)) if "limit" in query: - spots = spots[: int(query.get("limit"))] + spots = spots[: int(query["limit"])] # Ensure only the latest spot of each callsign-SSID combo is present in the list. This relies on the # list being in reverse time order, so if any future change allows re-ordering the list, that should @@ -162,7 +164,7 @@ def get_spot_list_with_filters(all_spots: LiveDataCache, query: dict[str, str]) # duplicates are fine in the main spot list (e.g. different cluster spots of the same DX) this doesn't # work well for the other views. if "dedupe" in query: - dedupe = query.get("dedupe").upper() == "TRUE" + dedupe = query["dedupe"].upper() == "TRUE" if dedupe: spots_temp = [] already_seen = [] @@ -183,26 +185,26 @@ def spot_allowed_by_query(spot: Spot, query: dict[str, str]) -> bool: for k in query: match k: case "since": - since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC).timestamp() + since = datetime.fromtimestamp(int(query[k]), pytz.UTC).timestamp() if not spot.time or spot.time <= since: return False case "max_age": - max_age = int(query.get(k)) + max_age = int(query[k]) since = (datetime.now(pytz.UTC) - timedelta(seconds=max_age)).timestamp() if not spot.time or spot.time <= since: return False case "received_since": - since = datetime.fromtimestamp(float(query.get(k)), pytz.UTC).timestamp() + since = datetime.fromtimestamp(float(query[k]), pytz.UTC).timestamp() if not spot.received_time or spot.received_time <= since: return False case "source": - sources = query.get(k).split(",") + sources = query[k].split(",") if not spot.source or spot.source not in sources: return False case "sig": # If a list of activities is provided, the spot must have an activity and it must match one of them. # The special activity "NO_SIG", when supplied in the list, matches spots with no activity. - activities = query.get(k).split(",") + activities = query[k].split(",") include_no_activity = "NO_SIG" in activities if not spot.sig and not include_no_activity: return False @@ -211,56 +213,56 @@ def spot_allowed_by_query(spot: Spot, query: dict[str, str]) -> bool: case "needs_sig": # If true, an activity is required, regardless of what it is, it just can't be missing. Mutually # exclusive with supplying the special "NO_SIG" parameter to the "sig" query param. - needs_activity = query.get(k).upper() == "TRUE" + needs_activity = query[k].upper() == "TRUE" if needs_activity and not spot.sig: return False case "needs_sig_ref": # If true, at least one activity ref is required, regardless of what it is, it just can't be missing. - needs_activity_ref = query.get(k).upper() == "TRUE" + needs_activity_ref = query[k].upper() == "TRUE" if needs_activity_ref and (not spot.sig_refs or len(spot.sig_refs) == 0): return False case "band": - bands = query.get(k).split(",") + bands = query[k].split(",") if not spot.band or spot.band not in bands: return False case "mode": - modes = query.get(k).split(",") + modes = query[k].split(",") if not spot.mode or spot.mode not in modes: return False case "mode_type": - mode_types = query.get(k).split(",") + mode_types = query[k].split(",") if not spot.mode_type or spot.mode_type not in mode_types: return False case "dx_continent": - dxconts = query.get(k).split(",") + dxconts = query[k].split(",") if not spot.dx_continent or spot.dx_continent not in dxconts: return False case "de_continent": - deconts = query.get(k).split(",") + deconts = query[k].split(",") if not spot.de_continent or spot.de_continent not in deconts: return False case "comment_includes": - comment_includes = query.get(k).strip() + comment_includes = query[k].strip() if not spot.comment or comment_includes.upper() not in spot.comment.upper(): return False case "dx_call_includes": - dx_call_includes = query.get(k).strip() + dx_call_includes = query[k].strip() if not spot.dx_call or dx_call_includes.upper() not in spot.dx_call.upper(): return False case "text_includes": - text_includes = query.get(k).strip() + text_includes = query[k].strip() if (not spot.dx_call or text_includes.upper() not in spot.dx_call.upper()) and ( not spot.comment or text_includes.upper() not in spot.comment.upper() ): return False case "allow_qrt": # If false, spots that are flagged as QRT are not returned. - prevent_qrt = query.get(k).upper() == "FALSE" + prevent_qrt = query[k].upper() == "FALSE" if prevent_qrt and spot.qrt: return False case "needs_good_location": # If true, spots require a "good" location to be returned - needs_good_location = query.get(k).upper() == "TRUE" + needs_good_location = query[k].upper() == "TRUE" if needs_good_location and not spot.dx_location_good: return False return True diff --git a/webserver/handlers/api/v1_addspot.py b/webserver/handlers/api/v1_addspot.py index 85994c7..33941bb 100644 --- a/webserver/handlers/api/v1_addspot.py +++ b/webserver/handlers/api/v1_addspot.py @@ -106,13 +106,14 @@ class V1APISpotHandler(tornado.web.RequestHandler): return # Reject if activity ref format incorrect for activity + ref_regex = get_ref_regex_for_activity(spot.sig) if spot.sig else None if ( spot.sig and spot.sig_refs and len(spot.sig_refs) > 0 and spot.sig_refs[0].id - and get_ref_regex_for_activity(spot.sig) - and not re.match(get_ref_regex_for_activity(spot.sig), spot.sig_refs[0].id) + and ref_regex + and not re.match(ref_regex, spot.sig_refs[0].id) ): self.set_status(422) self.write( @@ -127,6 +128,8 @@ class V1APISpotHandler(tornado.web.RequestHandler): # infer missing data, and add it to our database. spot.source = "API" spot.infer_missing() + assert self._spots is not None, "initialize() must be called before post()" + assert spot.id is not None, "infer_missing() always assigns an id" self._spots.set(spot.id, spot) self.write(safe_json_dumps("OK")) diff --git a/webserver/sse_broadcaster.py b/webserver/sse_broadcaster.py index b5de188..a01cffe 100644 --- a/webserver/sse_broadcaster.py +++ b/webserver/sse_broadcaster.py @@ -34,6 +34,7 @@ class SSEBroadcaster: return len(self._handlers) def publish(self, value: Any) -> None: + assert self._loop is not None, "bind_to_web_server_loop() must be called before publish()" self._loop.add_callback(self._broadcast, value) def _broadcast(self, value: Any) -> None: