diff --git a/config-example.yml b/config-example.yml index 4492e5c..00bfa34 100644 --- a/config-example.yml +++ b/config-example.yml @@ -67,7 +67,7 @@ spot_providers: - class: "DXCluster" # Clusters have a "name" property in case you want to say which cluster node the data is from, or connect to - # several clusters and name them separately. + # several clusters and name them separately. The name will be used as the "source" field for the spots received. name: "HRD Cluster" enabled: true host: "hrd.wa9pie.net" @@ -88,7 +88,7 @@ spot_providers: enabled: false host: "w3lpl.net" port: 7373 - login_prompt: "Please enter your call:" + login_prompt: "login:" login_callsign: "N0CALL-99" allow_rbn_spots: false diff --git a/core/call_lookup_helper.py b/core/call_lookup_helper.py index 19e8788..e1528cb 100644 --- a/core/call_lookup_helper.py +++ b/core/call_lookup_helper.py @@ -1,4 +1,5 @@ -from core.constants import CALL_ONLY_PATTERN +import re + from core.data_providers import DATA_PROVIDERS from data.callsign import Callsign @@ -11,7 +12,7 @@ def get_call_info(callsign, lookup_credentials): callsign_data = Callsign(call=callsign) # First check our input looks like a real callsign - if callsign and CALL_ONLY_PATTERN.match(callsign): + if callsign and re.match(r"^[A-Za-z0-9/\-]*$", callsign): # Sort callsign providers by priority order, so we query the highest priority (lowest numbers) first, and only # query other providers for data we are missing as we go along. for p in sorted(DATA_PROVIDERS.callsign_data_providers, key=lambda p2: p2.priority): diff --git a/core/constants.py b/core/constants.py index b1a6fdd..77125fd 100644 --- a/core/constants.py +++ b/core/constants.py @@ -1,5 +1,3 @@ -import re - from core.config import SERVER_OWNER_CALLSIGN from data.band import Band from data.sig import SIG @@ -11,11 +9,6 @@ SOFTWARE_VERSION = "2.0.2-pre" HTTP_HEADERS = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"} HAMQTH_PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}".replace(" ", "_") -# Generally useful regexes -CALL_REGEX = r"[A-Za-z0-9/\-]+" -CALL_PATTERN = re.compile(CALL_REGEX) -CALL_ONLY_PATTERN = re.compile(rf"^{CALL_REGEX}$") - # Special Interest Groups SIGS = [ SIG( @@ -60,6 +53,12 @@ SIGS = [ description="Islands on the Air", ref_regex=r"[A-Z]{2}\-\d{3}", ), + SIG( + name="GMA Island", + comment_names=[], + description="Global Mountain Activity - Islands", + ref_regex=r"(([A-Z]{2}\-\d{3})|([A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}))", + ), SIG( name="MOTA", comment_names=["MOTA"], diff --git a/core/sig_lookup_helper.py b/core/sig_lookup_helper.py index e6b11cf..32a4b0e 100644 --- a/core/sig_lookup_helper.py +++ b/core/sig_lookup_helper.py @@ -73,6 +73,20 @@ def get_sig_ref_info(sig, ref_id): sig_ref.url = f"https://www.beachesontheair.com/beaches/{sig_ref.name.lower().replace(' ', '-')}" return sig_ref + elif sig.upper() == "GMA Islands": + # GMA Islands is a bit of a mess of GMA and IOTA references. Try looking them both up and see what returns + # the best result. + iota_lookup = get_sig_ref_info("IOTA", ref_id) + gma_lookup = get_sig_ref_info("GMA", ref_id) + for key, value in iota_lookup.__dict__.items(): + if value is not None and sig_ref.__dict__.get(key) is None: + sig_ref.__dict__[key] = value + for key, value in gma_lookup.__dict__.items(): + if value is not None and sig_ref.__dict__.get(key) is None: + sig_ref.__dict__[key] = value + sig_ref.ref_type = "Island" + return sig_ref + ### ACTUAL LOOKUP ### # # OK, this is something we have to look up. Now check to see if our data store contains reference data and use diff --git a/data/spot.py b/data/spot.py index ba7a1e8..62e1bba 100644 --- a/data/spot.py +++ b/data/spot.py @@ -11,7 +11,7 @@ from pyhamtools.locator import latlong_to_locator, locator_to_latlong from core.call_lookup_helper import get_call_info from core.config import MAX_SPOT_AGE -from core.constants import CALL_REGEX, MODE_ALIASES, PROPAGATION_MODES +from core.constants import MODE_ALIASES, PROPAGATION_MODES from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone from core.sig_lookup_helper import populate_missing_sig_ref_info from core.sig_utils import ( @@ -201,14 +201,14 @@ class Spot: # If we have a spotter of "RBNHOLE", we should have the actual spotter callsign in the comment, so extract it. # RBNHole posts come from a number of providers, so it's dealt with here in the generic spot handling code. if self.de_call == "RBNHOLE" and self.comment: - rbnhole_call_match = re.search(rf"\Wat ({CALL_REGEX})\W", self.comment, re.IGNORECASE) + rbnhole_call_match = re.search(r"\Wat ([a-z0-9/]+)\W", self.comment, re.IGNORECASE) if rbnhole_call_match: self.de_call = rbnhole_call_match.group(1).upper() # If we have a spotter of "SOTAMAT", we might have the actual spotter callsign in the comment, if so extract it. # SOTAMAT can do POTA as well as SOTA, so it's dealt with here in the generic spot handling code. if self.de_call == "SOTAMAT" and self.comment: - sotamat_call_match = re.search(rf"\Wfrom ({CALL_REGEX})]", self.comment, re.IGNORECASE) + sotamat_call_match = re.search(r"\Wfrom ([a-z0-9/]+)]", self.comment, re.IGNORECASE) if sotamat_call_match: self.de_call = sotamat_call_match.group(1).upper() diff --git a/providers/alert/ng3k.py b/providers/alert/ng3k.py index cc2f6ed..161cd6a 100644 --- a/providers/alert/ng3k.py +++ b/providers/alert/ng3k.py @@ -6,7 +6,6 @@ import pytz from rss_parser import Parser from rss_parser.models.rss import RSS -from core.constants import CALL_REGEX from data.alert import Alert from providers.alert.http_alert_provider import HTTPAlertProvider @@ -16,7 +15,7 @@ class NG3K(HTTPAlertProvider): POLL_INTERVAL_SEC = 1800 ALERTS_URL = "https://www.ng3k.com/adxo.xml" - AS_CALL_PATTERN = re.compile(rf"as ({CALL_REGEX})", re.IGNORECASE) + AS_CALL_PATTERN = re.compile("as ([a-z0-9/]+)", re.IGNORECASE) def __init__(self, provider_config): super().__init__("NG3K", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC) diff --git a/providers/spot/dxcluster.py b/providers/spot/dxcluster.py index 379f14d..5cbe78f 100644 --- a/providers/spot/dxcluster.py +++ b/providers/spot/dxcluster.py @@ -8,7 +8,6 @@ import pytz import telnetlib3 from core.config import SERVER_OWNER_CALLSIGN -from core.constants import CALL_REGEX from data.spot import Spot from providers.spot.spot_provider import SpotProvider @@ -20,11 +19,11 @@ class DXCluster(SpotProvider): See config-example.yml for examples.""" _LINE_PATTERN_EXCLUDE_RBN = re.compile( - rf"^DX de ({CALL_REGEX}):\s+([0-9.]+)\s+({CALL_REGEX})\s+(.*)\s+(\d{4}Z)", + r"^DX de ([a-z0-9/]+):\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)", re.IGNORECASE, ) _LINE_PATTERN_ALLOW_RBN = re.compile( - rf"^DX de ({CALL_REGEX})-?#?:\s+([0-9.]+)\s+({CALL_REGEX})\s+(.*)\s+(\d{4}Z)", + r"^DX de ([a-z0-9/]+)-?#?:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)", re.IGNORECASE, ) @@ -51,7 +50,8 @@ class DXCluster(SpotProvider): def stop(self): self._running = False - self._telnet.close() + if self._telnet: + self._telnet.close() self._thread.join() def _handle(self): diff --git a/providers/spot/gma.py b/providers/spot/gma.py index 33c7036..e36a334 100644 --- a/providers/spot/gma.py +++ b/providers/spot/gma.py @@ -117,6 +117,9 @@ class GMA(HTTPSpotProvider): case "IOTA Island": spot.sig_refs[0].sig = "IOTA" spot.sig = "IOTA" + case "GMA Island": + spot.sig_refs[0].sig = "GMA Island" + spot.sig = "GMA Island" case "Lighthouse (ILLW)": spot.sig_refs[0].sig = "ILLW" spot.sig = "ILLW" diff --git a/providers/spot/parksnpeaks.py b/providers/spot/parksnpeaks.py index 2570e7a..9ae4f7c 100644 --- a/providers/spot/parksnpeaks.py +++ b/providers/spot/parksnpeaks.py @@ -5,7 +5,7 @@ from datetime import datetime import pytz import requests -from core.constants import HTTP_HEADERS, CALL_REGEX +from core.constants import HTTP_HEADERS from data.sig_ref import SIGRef from data.spot import Spot from providers.spot.http_spot_provider import HTTPSpotProvider @@ -60,7 +60,7 @@ class ParksNPeaks(HTTPSpotProvider): ) # Extract a de_call if it's in the comment but not in the "actSpoter" field - m = re.search(rf"\(de ({CALL_REGEX})\)", spot.comment or "") + m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment or "") if not spot.de_call and m: spot.de_call = str(m.group(1)) diff --git a/providers/spot/rbn.py b/providers/spot/rbn.py index 2f28626..6b26238 100644 --- a/providers/spot/rbn.py +++ b/providers/spot/rbn.py @@ -8,7 +8,6 @@ import pytz import telnetlib3 from core.config import SERVER_OWNER_CALLSIGN -from core.constants import CALL_REGEX from data.spot import Spot from providers.spot.spot_provider import SpotProvider @@ -20,7 +19,7 @@ class RBN(SpotProvider): (port 7001) you need to instantiate two copies of this. The port is provided as an argument to the constructor.""" _LINE_PATTERN = re.compile( - rf"^DX de ({CALL_REGEX})-.*:\s+([0-9.]+)\s+({CALL_REGEX})\s+(.*)\s+(\d{4}Z)", + r"^DX de ([a-z0-9/]+)-.*:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)", re.IGNORECASE, ) @@ -40,7 +39,8 @@ class RBN(SpotProvider): def stop(self): self._running = False - self._telnet.close() + if self._telnet: + self._telnet.close() self._thread.join() def _handle(self): diff --git a/server/handlers/api/addspot.py b/server/handlers/api/addspot.py index e59f4c7..72504c1 100644 --- a/server/handlers/api/addspot.py +++ b/server/handlers/api/addspot.py @@ -11,7 +11,7 @@ from tornado import httputil from tornado.web import Application from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRET_KEY -from core.constants import CALL_ONLY_PATTERN, UNKNOWN_BAND +from core.constants import UNKNOWN_BAND from core.prometheus_metrics_handler import api_requests_counter from core.sig_utils import get_ref_regex_for_sig from core.utils import infer_band_from_freq, safe_json_dumps @@ -121,13 +121,13 @@ class APISpotHandler(tornado.web.RequestHandler): return # Reject invalid-looking callsigns - if not CALL_ONLY_PATTERN.match(spot.dx_call): + if not re.match(r"^[A-Za-z0-9/\-]*$", spot.dx_call): self.set_status(422) self.write(safe_json_dumps(f"Error - '{spot.dx_call}' does not look like a valid callsign.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return - if not CALL_ONLY_PATTERN.match(spot.de_call): + if not re.match(r"^[A-Za-z0-9/\-]*$", spot.de_call): self.set_status(422) self.write(safe_json_dumps(f"Error - '{spot.de_call}' does not look like a valid callsign.")) self.set_header("Cache-Control", "no-store") diff --git a/server/handlers/api/alerts.py b/server/handlers/api/alerts.py index 2e999e3..6c1de40 100644 --- a/server/handlers/api/alerts.py +++ b/server/handlers/api/alerts.py @@ -82,8 +82,8 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler): self._credentials = None super().__init__(application, request, **kwargs) - def initialize(self, _sse_alert_broadcaster, web_server_metrics): - self._sse_alert_broadcaster = _sse_alert_broadcaster + def initialize(self, sse_alert_broadcaster, web_server_metrics): + self._sse_alert_broadcaster = sse_alert_broadcaster self._web_server_metrics = web_server_metrics def custom_headers(self): diff --git a/server/handlers/api/lookups.py b/server/handlers/api/lookups.py index c700cbe..ccc06ed 100644 --- a/server/handlers/api/lookups.py +++ b/server/handlers/api/lookups.py @@ -9,7 +9,7 @@ from tornado import httputil from tornado.web import Application from core.call_lookup_helper import get_call_info -from core.constants import CALL_ONLY_PATTERN, SIGS +from core.constants import SIGS from core.geo_utils import ( lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, @@ -55,7 +55,7 @@ class APILookupCallHandler(tornado.web.RequestHandler): # The "call" query param must exist and look like a callsign if "call" in query_params: call = str(query_params.get("call")).upper() - if CALL_ONLY_PATTERN.match(call): + if re.match(r"^[A-Z0-9/\-]*$", call): credentials = extract_credentials(self.request.headers) callsign_data = get_call_info(call, credentials) self.write(safe_json_dumps(callsign_data)) diff --git a/server/handlers/api/v1_addspot.py b/server/handlers/api/v1_addspot.py index 33394ab..04cc455 100644 --- a/server/handlers/api/v1_addspot.py +++ b/server/handlers/api/v1_addspot.py @@ -9,7 +9,7 @@ from tornado import httputil from tornado.web import Application from core.config import ALLOW_SPOTTING -from core.constants import CALL_ONLY_PATTERN, UNKNOWN_BAND +from core.constants import UNKNOWN_BAND from core.prometheus_metrics_handler import api_requests_counter from core.sig_utils import get_ref_regex_for_sig from core.utils import infer_band_from_freq, safe_json_dumps @@ -83,13 +83,13 @@ class V1APISpotHandler(tornado.web.RequestHandler): return # Reject invalid-looking callsigns - if not CALL_ONLY_PATTERN.match(spot.dx_call): + if not re.match(r"^[A-Za-z0-9/\-]*$", spot.dx_call): self.set_status(422) self.write(safe_json_dumps(f"Error - '{spot.dx_call}' does not look like a valid callsign.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return - if not CALL_ONLY_PATTERN.match(spot.de_call): + if not re.match(r"^[A-Za-z0-9/\-]*$", spot.de_call): self.set_status(422) self.write(safe_json_dumps(f"Error - '{spot.de_call}' does not look like a valid callsign.")) self.set_header("Cache-Control", "no-store") diff --git a/server/handlers/api/v1_compatability.py b/server/handlers/api/v1_compatability.py index 1fed156..fa6bcbb 100644 --- a/server/handlers/api/v1_compatability.py +++ b/server/handlers/api/v1_compatability.py @@ -2,6 +2,7 @@ import tornado from tornado.httpclient import AsyncHTTPClient from tornado.httputil import HTTPHeaders + _LEGACY_PARAM_TO_HEADER_MAP = { "qrz_username": "X-QRZ-Username", "qrz_password": "X-QRZ-Password", @@ -11,7 +12,6 @@ _LEGACY_PARAM_TO_HEADER_MAP = { "hamqth_session_id": "X-HamQTH-Session-ID", } - class V1RedirectHandler(tornado.web.RequestHandler): """Transparently proxies requests from the old API to the new one, returning whatever the v2 endpoint returns, for endpoints with no breaking changes.""" @@ -34,7 +34,7 @@ class V1RedirectHandler(tornado.web.RequestHandler): response = await client.fetch( new_url, method=self.request.method, - headers=headers, + headers=self.request.headers, body=None if self.request.method == "GET" else (self.request.body or b""), raise_error=False, follow_redirects=False, @@ -60,4 +60,4 @@ class V1RedirectHandler(tornado.web.RequestHandler): await self._proxy(path) async def post(self, path): - await self._proxy(path) + await self._proxy(path) \ No newline at end of file diff --git a/server/webserver.py b/server/webserver.py index 59c296c..9db01f8 100644 --- a/server/webserver.py +++ b/server/webserver.py @@ -160,7 +160,6 @@ class WebServer: V1APISpotHandler, { "spots": self._data_store.spots, - "spot_providers": self._data_providers, **handler_opts, }, ), diff --git a/spothole.py b/spothole.py index 55f82b5..64ef753 100644 --- a/spothole.py +++ b/spothole.py @@ -33,7 +33,7 @@ if __name__ == "__main__": root.setLevel(LOG_LEVEL) handler = logging.StreamHandler(sys.stdout) handler.setLevel(LOG_LEVEL) - formatter = logging.Formatter("%(levelname)s : %(message)s") + formatter = logging.Formatter("%(asctime)s | %(levelname)7s | %(message)s", "%Y-%m-%d %H:%M:%S") handler.setFormatter(formatter) root.handlers.clear() root.addHandler(handler) diff --git a/static/apidocs/openapi.yml b/static/apidocs/openapi.yml index d7be310..cbcc3cc 100644 --- a/static/apidocs/openapi.yml +++ b/static/apidocs/openapi.yml @@ -527,7 +527,8 @@ components: in: query description: > Limit the spots to only ones from one or more sources. To select more than one source, supply a - comma-separated list. + comma-separated list. Note that this enumeration is not exhaustive; in some cases the server owner may have + named some sources like clusters differently, and provided multiple options. schema: $ref: "#/components/schemas/Source" SpotSig: diff --git a/static/js/ui-ham.js b/static/js/ui-ham.js index e63224e..f365784 100644 --- a/static/js/ui-ham.js +++ b/static/js/ui-ham.js @@ -345,6 +345,7 @@ const SIG_ICONS = { "SOTA": "fa-mountain-sun", "WWFF": "fa-seedling", "GMA": "fa-person-hiking", + "GMA Island": "fa-person-hiking", "WWBOTA": "fa-radiation", "HEMA": "fa-mound", "IOTA": "fa-book-atlas", @@ -372,6 +373,7 @@ const SIG_NAMES = { "SOTA": "Summits on the Air", "WWFF": "Worldwide Flora & Fauna", "GMA": "Global Mountain Activity", + "GMA Island": "Global Mountain Activity - Islands", "WWBOTA": "Bunkers on the Air", "HEMA": "Humps Excluding Marilyns Award", "IOTA": "Islands on the Air", diff --git a/templates/about.html b/templates/about.html index 28753a4..e853d41 100644 --- a/templates/about.html +++ b/templates/about.html @@ -98,7 +98,7 @@

Between the various data sources, the following Special Interest Groups (SIGs) are supported: Parks on the Air (POTA), Summits on the Air (SOTA), Worldwide Flora & Fauna (WWFF), Global Mountain Activity (GMA), Worldwide Bunkers on the Air (WWBOTA), HuMPs Excluding Marilyns Award (HEMA), Islands on the Air (IOTA), Mills on the Air - (MOTA), the Amateur Radio Lighthouse Socirty (ARLHS), International Lighthouse Lightship Weekend (ILLW), Silos + (MOTA), the Amateur Radio Lighthouse Society (ARLHS), International Lighthouse Lightship Weekend (ILLW), Silos on the Air (SIOTA), World Castles Award (WCA), New Zealand on the Air (ZLOTA), Keith Roget Memorial National Parks Award (KRMNPA), South Australia National Parks and Conservation Parks Award (SANPCPA), Wainwrights on the Air (WOTA), Beaches on the Air (BOTA), Lagos y Lagunas On the Air (LLOTA), Towers on the Air (WWTOTA), Tiles on @@ -171,10 +171,6 @@ section, Privacy, for details of how these are handled. If you are looking at the map and see lots of spots in the geographic centre of countries, allowing these lookups will help. For QRZ.com a paid account is required to look up operator locations.

-

IOTA in particular causes a mapping problem, firstly that there is no official set of geodata beyond the names of - islands. We also have no way of telling whether priority should be given to IOTA reference or e.g. QRZ lookup, - for example for a G callsign in EU-005 Great Britain the QRZ home address is probably more accurate, but in - EU-002 Aaland islands, it's more likely a DXpedition and the IOTA would be more accurate.

Privacy

Spothole collects no data about you on a permanent basis. All spots and alerts are "timed out" and deleted from the system after a set interval, which by default is one hour for spots and one week for alerts.

@@ -182,11 +178,13 @@ filters. They are also stored in your browser's local storage, so that your preferences are remembered between sessions.

The data you provide can optionally include your login credentials for QRZ.com and HamQTH. You can provide these - in the "Data" menu of most pages. If you do, Spothole will augment the data it produces with lookups from these + in the "Your Data" menu of most pages. If you do, Spothole will augment the data it produces with lookups from these services, which can for example provide more accurate markers on the map tab, and operator names when you mouse over a DX callsign. Spothole will still work fine if you don't provide these. The values you enter are sent to Spothole via HTTPS so are protected in transit, though of course you do have to trust Spothole with this sensitive data in order to use this feature.

+

Any data you send as part of a query, such as your QRZ or HamQTH credentials, is used only for the lifetime of that + query and is not saved anywhere apart from your own device.

Spothole uses no trackers, no ads, and no cookies.

{% if len(web_ui_options["support_button_html"]) > 0 %}

Caveat: The owner of this server has chosen to inject their own content into the "spots" page. diff --git a/templates/add_spot.html b/templates/add_spot.html index 01129a8..934bd04 100644 --- a/templates/add_spot.html +++ b/templates/add_spot.html @@ -76,7 +76,7 @@ - + diff --git a/templates/alerts.html b/templates/alerts.html index 13c326d..a0e1ca6 100644 --- a/templates/alerts.html +++ b/templates/alerts.html @@ -84,7 +84,7 @@ - + diff --git a/templates/bands.html b/templates/bands.html index 59cf682..2f0d194 100644 --- a/templates/bands.html +++ b/templates/bands.html @@ -76,8 +76,8 @@ - - + + diff --git a/templates/base.html b/templates/base.html index abec8fe..261b349 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,6 +1,6 @@ {% extends "skeleton.html" %} {% block head_extra %} - + @@ -15,10 +15,10 @@ window.fetchEventSource = fetchEventSource; - - - - + + + + {% end %} {% block body %}

diff --git a/templates/cards/spot_age.html b/templates/cards/spot_age.html index 36cf5d3..eb5e625 100644 --- a/templates/cards/spot_age.html +++ b/templates/cards/spot_age.html @@ -6,7 +6,7 @@ oninput="filtersUpdated();" style="width: 5em; display: inline-block;"> {% for a in web_ui_options["max_spot_age"] %} + "max_spot_age_default"] == a %}selected{% end %}>{{a}} {% end %} minutes diff --git a/templates/conditions.html b/templates/conditions.html index 3c16c3e..9ac8027 100644 --- a/templates/conditions.html +++ b/templates/conditions.html @@ -284,7 +284,7 @@
- + diff --git a/templates/map.html b/templates/map.html index e8a06ba..cb89722 100644 --- a/templates/map.html +++ b/templates/map.html @@ -109,11 +109,12 @@ - - + + + diff --git a/templates/spots.html b/templates/spots.html index 7cc0ad0..941345a 100644 --- a/templates/spots.html +++ b/templates/spots.html @@ -113,8 +113,8 @@ - - + + diff --git a/templates/status.html b/templates/status.html index c3d71ae..1e92b51 100644 --- a/templates/status.html +++ b/templates/status.html @@ -86,7 +86,7 @@ - +