2 Commits
Author SHA1 Message Date
Ian Renton 818fd2d504 Refactor of caching & data storage part 2 #118 2026-07-31 15:18:23 +01:00
Ian Renton d26ddff7d1 Refactor of caching & data storage part 1 2026-07-31 14:12:55 +01:00
40 changed files with 449 additions and 1636 deletions
-7
View File
@@ -1,7 +0,0 @@
/.venv
__pycache__
*.pyc
/config.yml
/cache/
.git
.gitignore
-8
View File
@@ -1,8 +0,0 @@
FROM python:3.13-slim
WORKDIR /app
COPY . .
RUN pip install --no-cache-dir -r requirements.txt
EXPOSE 8080
CMD ["python3", "spothole.py"]
-30
View File
@@ -172,36 +172,6 @@ countries. This is normal, don't panic!
If you see some errors on startup, check your configuration, e.g. in case you have specified a port for the web server
that is already in use by something else.
### Running using Docker
Spothole comes with a Docker configuration to make it easy to run it in a containerised environment. To set it up using
Docker, the easiest way is to use a Docker Compose file. Create a new directory such as `/opt/docker/spothole` and
create a `compose.yaml` file inside it with the following contents:
```yaml
services:
spothole:
container_name: spothole
build:
context: https://git.ianrenton.com/ian/spothole.git#main
restart: unless-stopped
ports:
- "8080:8080"
volumes:
- ./config.yml:/app/config.yml
- ./cache:/app/cache
```
You can replace `#main` with any other branch or tag reference, for example `#1.5` to pin the build to tagged version
1.5.
Save the file. You will still need to create a copy of `config-example.yml` and name it `config.yml`, though with the
Docker setup nothing has actually been downloaded yet, so you will have to copy the example from the repository some
other way, e.g. [from the repo in a web browser](https://git.ianrenton.com/ian/spothole/src/branch/main/config-example.yml).
With that in place, run `docker compose up` and you should be good to go. To detach, press `d` or run the command with
the `-d` flag.
### Multiple cluster nodes with different settings
Dan, S50U has written in with his Spothole cluster settings. He is using a cluster node which provides RBN spots, and
+3 -13
View File
@@ -2,7 +2,7 @@ from datetime import datetime
import pytz
from core.config import MAX_ALERT_AGE
from core.data_store import DATA_STORE
class AlertProvider:
@@ -15,14 +15,7 @@ class AlertProvider:
self.enabled = provider_config["enabled"]
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
self.status = "Not Started" if self.enabled else "Disabled"
self._alerts = None
self._web_server = None
def setup(self, alerts, web_server):
"""Set up the provider, e.g. giving it the alert list to work from"""
self._alerts = alerts
self._web_server = web_server
self._alerts = DATA_STORE.alerts
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -44,10 +37,7 @@ class AlertProvider:
def _add_alert(self, alert):
if not alert.expired():
self._alerts.add(alert.id, alert, expire=MAX_ALERT_AGE)
# Ping the web server in case we have any SSE connections that need to see this immediately
if self._web_server:
self._web_server.notify_new_alert(alert)
self._alerts.set(alert.id, alert)
def stop(self):
"""Stop any threads and prepare for application shutdown"""
-27
View File
@@ -1,27 +0,0 @@
import threading
from datetime import timedelta
from requests_cache import CachedSession
# Cache for "semi-static" data such as the locations of parks, CSVs of reference lists, etc.
# This has an expiry time of 30 days, so will re-request from the source after that amount
# of time has passed. This is used throughout Spothole to cache data that does not change
# rapidly. The ThreadSafeSession construct here protects it against some multithreading
# contention weirdness we sometimes used to see on startup where the cache was hammered
# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type
# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote
# servers for data they've told us they can't provide.
_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30),
allowable_codes=(200, 400, 401, 403, 404))
_lock = threading.Lock()
class _ThreadSafeSession:
"""Wraps CachedSession with a lock to prevent concurrent SQLite access across threads."""
def get(self, *args, **kwargs):
with _lock:
return _session.get(*args, **kwargs)
SEMI_STATIC_URL_DATA_CACHE = _ThreadSafeSession()
-73
View File
@@ -1,73 +0,0 @@
import logging
from datetime import datetime
from threading import Event, Thread
import pytz
class CleanupTimer:
"""Provides a timed cleanup of the spot list."""
def __init__(self, spots, alerts, web_server, cleanup_interval):
"""Constructor"""
self._spots = spots
self._alerts = alerts
self._web_server = web_server
self._cleanup_interval = cleanup_interval
self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC)
self.status = "Starting"
self._thread = None
self._stop_event = Event()
def start(self):
"""Start the cleanup timer"""
self._thread = Thread(target=self._run, daemon=True)
self._thread.start()
def stop(self):
"""Stop any threads and prepare for application shutdown"""
self._stop_event.set()
def _run(self):
while not self._stop_event.wait(timeout=self._cleanup_interval):
self._cleanup()
def _cleanup(self):
"""Perform cleanup and reschedule next timer"""
try:
# Perform cleanup via letting the data expire
self._spots.expire()
self._alerts.expire()
# Explicitly clean up any spots and alerts that have expired
for i in list(self._spots.iterkeys()):
try:
spot = self._spots[i]
if spot.expired():
self._spots.delete(i)
except KeyError:
# Must have already been deleted, OK with that
pass
for i in list(self._alerts.iterkeys()):
try:
alert = self._alerts[i]
if alert.expired():
self._alerts.delete(i)
except KeyError:
# Must have already been deleted, OK with that
pass
# Clean up web server SSE spot/alert queues
self._web_server.clean_up_sse_queues()
self.status = "OK"
self.last_cleanup_time = datetime.now(pytz.UTC)
except Exception:
self.status = "Error"
logging.exception("Exception in Cleanup thread")
self._stop_event.wait(timeout=1)
+1 -1
View File
@@ -3,7 +3,7 @@ from data.band import Band
from data.sig import SIG
# General software
SOFTWARE_VERSION = "1.4"
SOFTWARE_VERSION = "1.4-pre"
# HTTP headers used for spot providers that use HTTP
HTTP_HEADERS = {"User-Agent": "Spothole v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
+69
View File
@@ -0,0 +1,69 @@
from pathlib import Path
import diskcache
from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
from core.live_data_cache import LiveDataCache
from data.solar_conditions import SolarConditions
class DataStore:
"""Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
lookup data using different caching strategies for each."""
def __init__(self):
cache_dir = "./cache"
self.MAX_SPOT_COUNT = 10000
self.MAX_ALERT_COUNT = 10000
self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
Path(cache_dir).mkdir(parents=True, exist_ok=True)
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval
self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
snapshot_dir=cache_dir + "/spots",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
snapshot_dir=cache_dir + "/alerts",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
# Standard disk cache for solar data and status data, but each cache contains only a single object which we
# expose to the wider application
self.solar = diskcache.Cache(cache_dir + "/solar")
if "solar_conditions" not in self.solar:
self.solar.add("solar_conditions", SolarConditions())
self.solar_conditions = self.solar.get("solar_conditions")
self.status = diskcache.Cache(cache_dir + "/status")
if "status_data" not in self.status:
self.status.add("status_data", {})
self.status_data = self.status.get("status_data")
# Standard disk caches for SIG ref data. Separate provider threads will repopulate these on a regular basis
# but there's no need for a TTL since old data is better than no data.
self.sigrefs_wwff = diskcache.Cache(cache_dir + "/sigrefs/wwff")
self.sigrefs_siota = diskcache.Cache(cache_dir + "/sigrefs/siota")
self.sigrefs_wota = diskcache.Cache(cache_dir + "/sigrefs/wota")
self.sigrefs_zlota = diskcache.Cache(cache_dir + "/sigrefs/zlota")
self.sigrefs_llota = diskcache.Cache(cache_dir + "/sigrefs/llota")
self.sigrefs_dme = diskcache.Cache(cache_dir + "/sigrefs/dme")
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
# we've seen, so we rely on them timing out and this triggering another lookup.
self.callsigns = diskcache.Cache(cache_dir + "/callsigns")
def close(self):
self.spots.close()
self.alerts.close()
self.solar.close()
self.status.close()
self.sigrefs_wwff.close()
self.sigrefs_siota.close()
self.sigrefs_wota.close()
self.sigrefs_zlota.close()
self.sigrefs_llota.close()
self.callsigns.close()
# Global object
DATA_STORE = DataStore()
+100
View File
@@ -0,0 +1,100 @@
import logging
import threading
import time
import diskcache
from cachetools import TTLCache
class LiveDataCache:
"""Cache for spots and alerts. Uses the fast in-memory TTLCache for normal data I/O, including the TTL to enforce
maximum lifetime, and adds a separate diskcache to which we can save and load the TTLCache to provide persistence.
Also adds thread safety so spots and alerts can come from any thread, and a listener mechanism so the web server
can get a callback when new spots/alerts are added, and send them to any SSE clients."""
def __init__(self, maxsize, ttl, snapshot_dir, snapshot_interval_sec):
self._cache = TTLCache(maxsize=maxsize, ttl=ttl)
self._lock = threading.Lock()
self._ttl = ttl
self._listeners = []
self._listeners_lock = threading.Lock()
self._snapshot_dir = snapshot_dir
self._disk_cache = diskcache.Cache(str(snapshot_dir))
self._load_snapshot()
self._start_periodic_snapshot(snapshot_interval_sec)
def set(self, key, value):
with self._lock:
self._cache[key] = value
# Notify listeners
with self._listeners_lock:
listeners = list(self._listeners)
for callback in listeners:
try:
callback(value)
except Exception:
logging.error("Listener raised an exception for key %s", key, exc_info=True)
def get(self, key, default=None):
with self._lock:
return self._cache.get(key, default)
def delete(self, key):
with self._lock:
self._cache.pop(key, None)
def keys(self):
with self._lock:
return list(self._cache.keys())
def values(self):
with self._lock:
return list(self._cache.values())
def add_listener(self, callback):
"""Register callback(value) which will be called whenever a new spot/alert item is added via set(). Used by the
web server (via SSEBroadcaster) to send SSE clients an update on every new spot."""
with self._listeners_lock:
self._listeners.append(callback)
def remove_listener(self, callback):
with self._listeners_lock:
self._listeners.remove(callback)
def save_snapshot(self):
with self._lock:
# Store the time with the data so we can avoid loading anything nxt time that's older than TTL
data = [(k, v, time.time()) for k, v in self._cache.items()]
try:
self._disk_cache.set("snapshot", data)
except Exception as e:
logging.error("Failed to write snapshot to %s", self._snapshot_dir, e, exc_info=True)
def _load_snapshot(self):
data = self._disk_cache.get("snapshot")
if not data:
return
now = time.time()
with self._lock:
for key, value, saved_at in data:
# Only restore entries that would still be within TTL
if now - saved_at < self._ttl:
self._cache[key] = value
logging.info("Loaded snapshot from %s", self._snapshot_dir)
def _start_periodic_snapshot(self, interval):
def loop():
while True:
time.sleep(interval)
self.save_snapshot()
t = threading.Thread(target=loop, daemon=True, name=f"snapshot-{self._snapshot_dir}")
t.start()
def close(self):
self.save_snapshot()
self._disk_cache.close()
+5 -5
View File
@@ -14,10 +14,10 @@ from pyhamtools.locator import latlong_to_locator
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests_cache import CachedSession
from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
from core.config import config
from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
from core.url_data_cache import URL_DATA_CACHE
# QRZ XML field names differ from pyhamtools' normalised names; map them here.
_QRZ_FIELD_MAP = {
@@ -142,7 +142,7 @@ class LookupHelper:
try:
logging.info("Downloading Country-files.com cty.plist...")
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
response = URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
headers=HTTP_HEADERS)
if response.ok:
@@ -167,7 +167,7 @@ class LookupHelper:
try:
logging.info("Downloading dxcc.json...")
response = SEMI_STATIC_URL_DATA_CACHE.get(
response = URL_DATA_CACHE.get(
"https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
headers=HTTP_HEADERS)
@@ -515,7 +515,7 @@ class LookupHelper:
for lookup_call in calls_to_try:
try:
response = SEMI_STATIC_URL_DATA_CACHE.get(
response = URL_DATA_CACHE.get(
self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
headers=HTTP_HEADERS, timeout=10)
if response.ok:
@@ -593,7 +593,7 @@ class LookupHelper:
for lookup_call in calls_to_try:
try:
response = SEMI_STATIC_URL_DATA_CACHE.get(
response = URL_DATA_CACHE.get(
self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
if response.ok:
+39 -49
View File
@@ -4,23 +4,16 @@ import logging
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
from core.constants import SIGS, HTTP_HEADERS
from core.data_store import DATA_STORE
from core.geo_utils import wab_wai_square_to_lat_lon
from core.url_data_cache import URL_DATA_CACHE
# Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data
# file in Spothole and load it on startup.
with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
_DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
_WWFF_INDEX_CACHE = {}
_SIOTA_INDEX_CACHE = {}
_WOTA_INDEX_CACHE = {}
_ZLOTA_INDEX_CACHE = {}
_LLOTA_INDEX_CACHE = {}
for row in csv.DictReader(_f, delimiter=";"):
DATA_STORE.sigrefs_dme.add(row["COD_INE"][:5], row)
def get_ref_regex_for_sig(sig):
"""Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
@@ -53,7 +46,7 @@ def populate_sig_ref_info(sig_ref):
ref_id = sig_ref.id
try:
if sig.upper() == "POTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
response = URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
@@ -71,7 +64,7 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "SOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
response = URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
@@ -88,7 +81,7 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WWBOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
response = URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
@@ -104,7 +97,7 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
@@ -130,15 +123,14 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WWFF":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
response = URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
headers=HTTP_HEADERS)
if response.ok:
global _WWFF_INDEX_CACHE
if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
if not bool(DATA_STORE.sigrefs_wwff) or not response.from_cache:
# New data from WWFF, update our internal map
_WWFF_INDEX_CACHE = {row["reference"]: row for row in
csv.DictReader(response.content.decode().splitlines())}
row = _WWFF_INDEX_CACHE.get(ref_id)
for row in csv.DictReader(response.content.decode().splitlines()):
DATA_STORE.sigrefs_wwff.add(row["reference"], row)
row = DATA_STORE.sigrefs_wwff.get(ref_id)
if row:
sig_ref.name = row["name"] if "name" in row else None
sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
@@ -152,15 +144,14 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "SIOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
response = URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
headers=HTTP_HEADERS)
if response.ok:
global _SIOTA_INDEX_CACHE
if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
if not bool(DATA_STORE.sigrefs_siota) or not response.from_cache:
# New data from SIOTA, update our internal map
_SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
csv.DictReader(response.content.decode().splitlines())}
row = _SIOTA_INDEX_CACHE.get(ref_id)
for row in csv.DictReader(response.content.decode().splitlines()):
DATA_STORE.sigrefs_siota.add(row["SILO_CODE"], row)
row = DATA_STORE.sigrefs_siota.get(ref_id)
if row:
sig_ref.name = row["NAME"] if "NAME" in row else None
sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
@@ -172,17 +163,16 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "WOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
response = URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
global _WOTA_INDEX_CACHE
if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
if not bool(DATA_STORE.sigrefs_wota) or not response.from_cache:
# New data from WOTA, update our internal map
_WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
data.get("features", [])}
feature = _WOTA_INDEX_CACHE.get(ref_id)
for feature in data.get("features", []):
DATA_STORE.sigrefs_wota.add(feature["properties"]["wotaId"], feature)
feature = DATA_STORE.sigrefs_wota.get(ref_id)
if feature:
sig_ref.name = feature["properties"]["title"]
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -200,24 +190,24 @@ def populate_sig_ref_info(sig_ref):
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
elif sig.upper() == "ZLOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
response = URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if isinstance(data, list):
global _ZLOTA_INDEX_CACHE
if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
if not bool(DATA_STORE.sigrefs_zlota) or not response.from_cache:
# New data from ZLOTA, update our internal map
_ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
asset = _ZLOTA_INDEX_CACHE.get(ref_id)
if asset:
sig_ref.name = asset["name"]
for ref in data:
DATA_STORE.sigrefs_zlota.add(ref["code"], ref)
ref = DATA_STORE.sigrefs_zlota.get(ref_id)
if ref:
sig_ref.name = ref["name"]
sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
try:
sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
sig_ref.grid = latlong_to_locator(ref["y"], ref["x"], 6)
except:
logging.debug("Invalid lat/lon received for reference")
sig_ref.latitude = asset["y"]
sig_ref.longitude = asset["x"]
sig_ref.latitude = ref["y"]
sig_ref.longitude = ref["x"]
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
elif not response.from_cache:
@@ -229,16 +219,16 @@ def populate_sig_ref_info(sig_ref):
sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
elif sig.upper() == "LLOTA":
response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
response = URL_DATA_CACHE.get("https://llota.app/api/public/references",
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if isinstance(data, list):
global _LLOTA_INDEX_CACHE
if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
if not bool(DATA_STORE.sigrefs_llota) or not response.from_cache:
# New data from LLOTA, update our internal map
_LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
ref = _LLOTA_INDEX_CACHE.get(ref_id)
for ref in data:
DATA_STORE.sigrefs_llota.add(ref["reference_code"], ref)
ref = DATA_STORE.sigrefs_llota.get(ref_id)
if ref:
sig_ref.name = str(ref["name"])
sig_ref.url = "https://llota.app/list/ref/" + ref_id
@@ -280,7 +270,7 @@ def populate_sig_ref_info(sig_ref):
elif sig.upper() == "DME":
# Zero-pad to 5 digits to match our source data
row = _DME_INDEX.get(ref_id.zfill(5))
row = DATA_STORE.sigrefs_dme.get(ref_id.zfill(5))
if row:
sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+14 -21
View File
@@ -7,31 +7,27 @@ import pytz
from core.config import SERVER_OWNER_CALLSIGN
from core.constants import SOFTWARE_VERSION
from core.data_store import DATA_STORE
from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge
class StatusReporter:
"""Provides a timed update of the application's status data."""
def __init__(self, status_data, run_interval, web_server, cleanup_timer, spots, spot_providers, alerts,
alert_providers, solar_condition_providers):
def __init__(self, run_interval, web_server,spot_providers, alert_providers, solar_condition_providers):
"""Constructor"""
self._status_data = status_data
self._run_interval = run_interval
self._web_server = web_server
self._cleanup_timer = cleanup_timer
self._spots = spots
self._spot_providers = spot_providers
self._alerts = alerts
self._alert_providers = alert_providers
self._solar_condition_providers = solar_condition_providers
self._thread = None
self._stop_event = Event()
self._startup_time = datetime.now(pytz.UTC)
self._status_data["software-version"] = SOFTWARE_VERSION
self._status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
DATA_STORE.status_data["software-version"] = SOFTWARE_VERSION
DATA_STORE.status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
def start(self):
"""Start the reporter thread"""
@@ -55,31 +51,28 @@ class StatusReporter:
def _report(self):
"""Write status information"""
self._status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
self._status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
self._status_data["num_spots"] = len(self._spots)
self._status_data["num_alerts"] = len(self._alerts)
self._status_data["spot_providers"] = list(
DATA_STORE.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
DATA_STORE.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
DATA_STORE.status_data["spot_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
"last_spot": p.last_spot_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
self._spot_providers))
self._status_data["alert_providers"] = list(
DATA_STORE.status_data["alert_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
self._alert_providers))
self._status_data["solar_condition_providers"] = list(
DATA_STORE.status_data["solar_condition_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
self._solar_condition_providers))
self._status_data["cleanup"] = {"status": self._cleanup_timer.status,
"last_ran": self._cleanup_timer.last_cleanup_time.replace(
tzinfo=pytz.UTC).timestamp() if self._cleanup_timer.last_cleanup_time else 0}
self._status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
"last_api_access": self._web_server.web_server_metrics[
"last_api_access_time"].replace(
tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
@@ -94,5 +87,5 @@ class StatusReporter:
# Update Prometheus metrics
memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)
spots_gauge.set(len(self._spots))
alerts_gauge.set(len(self._alerts))
spots_gauge.set(len(DATA_STORE.spots.values()))
alerts_gauge.set(len(DATA_STORE.alerts.values()))
+23
View File
@@ -0,0 +1,23 @@
import threading
from datetime import timedelta
from requests_cache import CachedSession
# Cache for "semi-static" data retrieved from a URL. This is a layet of caching in addition to the normal caching of
# spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints frequently when
# e.g. restarting Spothole many times during testing.
_session = CachedSession("cache/semi_static_urls", expire_after=timedelta(days=1),
allowable_codes=(200, 400, 401, 403, 404))
_lock = threading.Lock()
class _ThreadSafeSession:
"""Wraps CachedSession with a lock to prevent concurrent SQLite access across threads. This allows a single object
to be used freely across the application."""
def get(self, *args, **kwargs):
with _lock:
return _session.get(*args, **kwargs)
# Global object
URL_DATA_CACHE = _ThreadSafeSession()
-10
View File
@@ -6,13 +6,3 @@ def safe_json_dumps(obj):
which are invalid in JSON."""
return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
def empty_queue(q):
"""Empty a queue"""
while not q.empty():
try:
q.get_nowait()
except:
break
+1
View File
@@ -18,3 +18,4 @@ tornado~=6.4.2
tornado_eventsource~=3.0.0
geopandas~=0.13.2
simplejson~=4.1.1
cachetools~=7.1.6
+2 -2
View File
@@ -8,7 +8,7 @@ import tornado
from tornado import httputil
from tornado.web import Application
from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
from core.config import ALLOW_SPOTTING
from core.constants import UNKNOWN_BAND
from core.lookup_helper import infer_band_from_freq
from core.prometheus_metrics_handler import api_requests_counter
@@ -119,7 +119,7 @@ class APISpotHandler(tornado.web.RequestHandler):
# infer missing data, and add it to our database.
spot.source = "API"
spot.infer_missing()
self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
self._spots.set(spot.id, spot)
self.write(safe_json_dumps("OK"))
self.set_status(201)
+13 -46
View File
@@ -1,7 +1,6 @@
import copy
import logging
from datetime import datetime
from queue import Queue
from typing import Any
import pytz
@@ -11,12 +10,9 @@ from tornado import httputil
from tornado.web import Application
from core.prometheus_metrics_handler import api_requests_counter
from core.utils import safe_json_dumps, empty_queue
from core.utils import safe_json_dumps
from data.lookup_credentials import extract_credentials
SSE_HANDLER_MAX_QUEUE_SIZE = 100
SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
class APIAlertsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v1/alerts"""
@@ -73,16 +69,14 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
"""API request handler for /api/v1/alerts/stream"""
def __init__(self, application, request, **kwargs: Any):
self._sse_alert_queues = None
self._sse_alert_broadcaster = None
self._web_server_metrics = None
self._query_params = None
self._credentials = None
self._alert_queue = None
self._heartbeat = None
super().__init__(application, request, **kwargs)
def initialize(self, sse_alert_queues, web_server_metrics):
self._sse_alert_queues = sse_alert_queues
def initialize(self, _sse_alert_broadcaster, web_server_metrics):
self._sse_alert_broadcaster = _sse_alert_broadcaster
self._web_server_metrics = web_server_metrics
def custom_headers(self):
@@ -104,59 +98,32 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
self._credentials = extract_credentials(self._query_params)
# Create a alert queue and add it to the web server's list. The web server will fill this when alerts arrive
self._alert_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
self._sse_alert_queues.append(self._alert_queue)
# Set up a timed callback to check if anything is in the queue
self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
self._heartbeat.start()
# Flush headers immediately so nginx doesn't time out waiting for a response
self.write_message("keepalive", "")
# Register to handle new alerts arriving. The callback() method will get called with the new alert as an
# argument.
self._sse_alert_broadcaster.register(self)
except Exception as e:
logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
self.close()
def close(self):
"""When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
"""When the user closes the socket, deregister ourselves from the alert broadcaster"""
try:
if self._alert_queue in self._sse_alert_queues:
self._sse_alert_queues.remove(self._alert_queue)
empty_queue(self._alert_queue)
except:
pass
try:
self._heartbeat.stop()
except:
pass
self._alert_queue = None
self._sse_alert_broadcaster.unregister(self)
super().close()
def _callback(self):
"""Callback to check if anything has arrived in the queue, and if so send it to the client"""
def callback(self, alert):
"""Callback when a new alert arrives"""
try:
if self._alert_queue:
if not self._alert_queue.empty():
while not self._alert_queue.empty():
alert = self._alert_queue.get()
# If the new alert matches our param filters, send it to the client. If not, ignore it.
if alert_allowed_by_query(alert, self._query_params):
if self._credentials:
alert = copy.deepcopy(alert)
alert.infer_missing(self._credentials)
self.write_message(msg=safe_json_dumps(alert))
else:
# Send a keepalive comment if the queue was empty
self.write_message("keepalive", "")
if self._alert_queue not in self._sse_alert_queues:
logging.error("Web server cleared up a queue of an active connection!")
self.close()
except Exception as e:
logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
self.close()
@@ -169,7 +136,7 @@ def get_alert_list_with_filters(all_alerts, query):
# Create a shallow copy of the alert list ordered by start time, then filter the list to reduce it only to alerts
# that match the filter parameters in the query string. Finally, apply a limit to the number of alerts returned.
# The list of query string filters is defined in the API docs.
alert_ids = list(all_alerts.iterkeys())
alert_ids = all_alerts.keys()
alerts = []
for k in alert_ids:
a = all_alerts.get(k)
+1 -1
View File
@@ -40,7 +40,7 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
counts = Counter()
for key in self._spots.iterkeys():
for key in self._spots.keys():
spot = self._spots.get(key)
if spot is None:
continue
+13 -45
View File
@@ -1,7 +1,6 @@
import copy
import logging
from datetime import datetime, timedelta
from queue import Queue
from typing import Any
import pytz
@@ -11,12 +10,9 @@ from tornado import httputil
from tornado.web import Application
from core.prometheus_metrics_handler import api_requests_counter
from core.utils import safe_json_dumps, empty_queue
from core.utils import safe_json_dumps
from data.lookup_credentials import extract_credentials
SSE_HANDLER_MAX_QUEUE_SIZE = 1000
SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
class APISpotsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v1/spots"""
@@ -73,16 +69,14 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
"""API request handler for /api/v1/spots/stream"""
def __init__(self, application, request, **kwargs: Any):
self._sse_spot_queues = None
self._sse_spot_broadcaster = None
self._web_server_metrics = None
self._query_params = None
self._credentials = None
self._spot_queue = None
self._heartbeat = None
super().__init__(application, request, **kwargs)
def initialize(self, sse_spot_queues, web_server_metrics):
self._sse_spot_queues = sse_spot_queues
def initialize(self, sse_spot_broadcaster, web_server_metrics):
self._sse_spot_broadcaster = sse_spot_broadcaster
self._web_server_metrics = web_server_metrics
def custom_headers(self):
@@ -106,59 +100,33 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
self._credentials = extract_credentials(self._query_params)
# Create a spot queue and add it to the web server's list. The web server will fill this when spots arrive
self._spot_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
self._sse_spot_queues.append(self._spot_queue)
# Set up a timed callback to check if anything is in the queue
self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
self._heartbeat.start()
# Flush headers immediately so nginx doesn't time out waiting for a response
self.write_message("keepalive", "")
# Register to handle new spots arriving. The callback() method will get called with the new spot as an
# argument.
self._sse_spot_broadcaster.register(self)
except Exception as e:
logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
self.close()
def close(self):
"""When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
"""When the user closes the socket, deregister ourselves from the spot broadcaster"""
try:
if self._spot_queue in self._sse_spot_queues:
self._sse_spot_queues.remove(self._spot_queue)
empty_queue(self._spot_queue)
except:
pass
try:
self._heartbeat.stop()
except:
pass
self._spot_queue = None
self._sse_spot_broadcaster.unregister(self)
super().close()
def _callback(self):
"""Callback to check if anything has arrived in the queue, and if so send it to the client"""
def callback(self, spot):
"""Callback when a new spot arrives"""
try:
if self._spot_queue:
if not self._spot_queue.empty():
while not self._spot_queue.empty():
spot = self._spot_queue.get()
# If the new spot matches our param filters, send it to the client. If not, ignore it.
if spot_allowed_by_query(spot, self._query_params):
if self._credentials:
spot = copy.deepcopy(spot)
spot.infer_missing(self._credentials)
self.write_message(msg=safe_json_dumps(spot))
else:
# Send a keepalive comment if the queue was empty
self.write_message("keepalive", "")
if self._spot_queue not in self._sse_spot_queues:
logging.error("Web server cleared up a queue of an active connection!")
self.close()
except Exception as e:
logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
self.close()
@@ -171,7 +139,7 @@ def get_spot_list_with_filters(all_spots, query):
# Create a shallow copy of the spot list, ordered by spot time, then filter the list to reduce it only to spots
# that match the filter parameters in the query string. Finally, apply a limit to the number of spots returned.
# The list of query string filters is defined in the API docs.
spot_ids = list(all_spots.iterkeys())
spot_ids = all_spots.keys()
spots = []
for k in spot_ids:
s = all_spots.get(k)
+36
View File
@@ -0,0 +1,36 @@
import logging
import threading
from tornado.ioloop import IOLoop
class SSEBroadcaster:
"""Bridge between DataStore listener callbacks (which fire on provider threads) to Tornado's async SSE handlers
(which live on the IOLoop thread) to avoid any interdependency between them."""
def __init__(self):
self._handlers = set()
self._lock = threading.Lock()
self._loop = IOLoop.current()
def register(self, handler):
with self._lock:
self._handlers.add(handler)
def unregister(self, handler):
with self._lock:
self._handlers.discard(handler)
def publish(self, value):
self._loop.add_callback(self._fan_out, value)
def _fan_out(self, value):
with self._lock:
handlers = list(self._handlers)
for handler in handlers:
try:
handler.callback(value)
except Exception:
# Connection probably dropped, ignore and de-register the handler to stop getting future items.
logging.debug("Failed to push to an SSE client; dropping it")
self.unregister(handler)
+19 -66
View File
@@ -6,7 +6,7 @@ import tornado
from tornado.web import StaticFileHandler
from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL
from core.utils import empty_queue
from core.data_store import DATA_STORE
from server.handlers.api.addspot import APISpotHandler
from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
from server.handlers.api.dxstats import APIDxStatsHandler
@@ -18,6 +18,7 @@ from server.handlers.api.status import APIStatusHandler
from server.handlers.manifesthandler import ManifestHandler
from server.handlers.metrics import PrometheusMetricsHandler
from server.handlers.pagetemplate import PageTemplateHandler
from server.sse_broadcaster import SSEBroadcaster
_HERE = os.path.dirname(__file__ or "")
@@ -25,15 +26,12 @@ _HERE = os.path.dirname(__file__ or "")
class WebServer:
"""Provides the public-facing web server."""
def __init__(self, spots, alerts, solar_conditions, status_data):
def __init__(self):
"""Constructor"""
self._spots = spots
self._alerts = alerts
self._solar_conditions = solar_conditions
self._sse_spot_queues = []
self._sse_alert_queues = []
self._status_data = status_data
self._data_store = DATA_STORE
self._spot_broadcaster = SSEBroadcaster()
self._alert_broadcaster = SSEBroadcaster()
self._port = WEB_SERVER_PORT
self._api_only_mode = API_ONLY_MODE
self._shutdown_event = asyncio.Event()
@@ -45,6 +43,10 @@ class WebServer:
"status": "Starting"
}
# Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
DATA_STORE.spots.add_listener(self._spot_broadcaster.publish)
DATA_STORE.alerts.add_listener(self._alert_broadcaster.publish)
def start(self):
"""Start the web server"""
@@ -64,20 +66,20 @@ class WebServer:
# API endpoints are always enabled
api_routes = [
(r"/api/v1/spots", APISpotsHandler, {"spots": self._spots, **handler_opts}),
(r"/api/v1/alerts", APIAlertsHandler, {"alerts": self._alerts, **handler_opts}),
(r"/api/v1/spots", APISpotsHandler, {"spots": self._data_store.spots, **handler_opts}),
(r"/api/v1/alerts", APIAlertsHandler, {"alerts": self._data_store.alerts, **handler_opts}),
(r"/api/v1/spots/stream", APISpotsStreamHandler,
{"sse_spot_queues": self._sse_spot_queues, **handler_opts}),
{"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts}),
(r"/api/v1/alerts/stream", APIAlertsStreamHandler,
{"sse_alert_queues": self._sse_alert_queues, **handler_opts}),
(r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._solar_conditions, **handler_opts}),
(r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._spots, **handler_opts}),
(r"/api/v1/options", APIOptionsHandler, {"status_data": self._status_data, **handler_opts}),
(r"/api/v1/status", APIStatusHandler, {"status_data": self._status_data, **handler_opts}),
{"sse_alert_broadcaster": self._alert_broadcaster, **handler_opts}),
(r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._data_store.solar, **handler_opts}),
(r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._data_store.spots, **handler_opts}),
(r"/api/v1/options", APIOptionsHandler, {"status_data": self._data_store.status_data, **handler_opts}),
(r"/api/v1/status", APIStatusHandler, {"status_data": self._data_store.status_data, **handler_opts}),
(r"/api/v1/lookup/call", APILookupCallHandler, {**handler_opts}),
(r"/api/v1/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
(r"/api/v1/lookup/grid", APILookupGridHandler, {**handler_opts}),
(r"/api/v1/spot", APISpotHandler, {"spots": self._spots, **handler_opts}),
(r"/api/v1/spot", APISpotHandler, {"spots": self._data_store.spots, **handler_opts}),
]
# If in API-only mode, serve a basic homepage; in normal mode, serve the usual UI routes
@@ -121,55 +123,6 @@ class WebServer:
logging.info("You can access your copy of Spothole at " + BASE_URL)
await self._shutdown_event.wait()
def notify_new_spot(self, spot):
"""Internal method called when a new spot is added to the system. This is used to ping any SSE clients that are
awaiting a server-sent message with new spots."""
for queue in self._sse_spot_queues:
try:
queue.put(spot)
except:
# Cleanup thread was probably deleting the queue, that's fine
pass
pass
def notify_new_alert(self, alert):
"""Internal method called when a new alert is added to the system. This is used to ping any SSE clients that are
awaiting a server-sent message with new spots."""
for queue in self._sse_alert_queues:
try:
queue.put(alert)
except:
# Cleanup thread was probably deleting the queue, that's fine
pass
pass
def clean_up_sse_queues(self):
"""Clean up any SSE queues that are growing too large; probably their client disconnected and we didn't catch it
properly for some reason."""
for q in self._sse_spot_queues:
try:
if q.full():
logging.warning(
"A full SSE spot queue was found, presumably because the client disconnected strangely. It has been removed.")
self._sse_spot_queues.remove(q)
empty_queue(q)
except:
# Probably got deleted already on another thread
pass
for q in self._sse_alert_queues:
try:
if q.full():
logging.warning(
"A full SSE alert queue was found, presumably because the client disconnected strangely. It has been removed.")
self._sse_alert_queues.remove(q)
empty_queue(q)
except:
# Probably got deleted already on another thread
pass
pass
def request_log(handler):
"""Custom log function to provide more data about requests when enabled, and to provide the ability to turn off
@@ -0,0 +1,28 @@
from datetime import datetime
import pytz
class SIGRefDataProvider:
"""Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data."""
def __init__(self, name, provider_config):
"""Constructor"""
self.name = name
self.enabled = provider_config["enabled"]
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
self.status = "Not Started" if self.enabled else "Disabled"
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
raise NotImplementedError("Subclasses must implement this method")
def stop(self):
"""Stop any threads and prepare for application shutdown"""
raise NotImplementedError("Subclasses must implement this method")
+12 -15
View File
@@ -35,6 +35,18 @@ class GIROIonosonde(SolarConditionsProvider):
self._thread = None
self._stop_event = Event()
# Pre-populate ionosonde_data with known station names for stations not already present,
# so the station dropdown is available before the first poll. Does not overwrite existing
# entries so KC2G cache data is preserved.
existing = self._solar_conditions.ionosonde_data or {}
new_entries = {
s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
"luf": None, "band_states": None}
for s in self._stations if s["ursi"] not in existing
}
if new_entries:
self.update_data({"ionosonde_data": {**existing, **new_entries}})
@staticmethod
def _load_stations():
stations = []
@@ -44,21 +56,6 @@ class GIROIonosonde(SolarConditionsProvider):
stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
return stations
def setup(self, solar_conditions, solar_conditions_cache):
"""Pre-populate ionosonde_data with known station names for stations not already present,
so the station dropdown is available before the first poll. Does not overwrite existing
entries so KC2G cache data is preserved."""
super().setup(solar_conditions, solar_conditions_cache)
existing = solar_conditions.ionosonde_data or {}
new_entries = {
s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
"luf": None, "band_states": None}
for s in self._stations if s["ursi"] not in existing
}
if new_entries:
self.update_data({"ionosonde_data": {**existing, **new_entries}})
def start(self):
logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
self._thread = Thread(target=self._run, daemon=True)
@@ -2,6 +2,8 @@ from datetime import datetime
import pytz
from core.data_store import DATA_STORE
class SolarConditionsProvider:
"""Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
@@ -10,18 +12,11 @@ class SolarConditionsProvider:
def __init__(self, name, provider_config):
"""Constructor"""
self._solar_conditions_cache = None
self.name = name
self.enabled = provider_config["enabled"]
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
self.status = "Not Started" if self.enabled else "Disabled"
self._solar_conditions = None
def setup(self, solar_conditions, solar_conditions_cache):
"""Set up the provider, giving it the solar conditions object and its backing cache"""
self._solar_conditions = solar_conditions
self._solar_conditions_cache = solar_conditions_cache
self._solar_conditions = DATA_STORE.solar_conditions
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -41,4 +36,3 @@ class SolarConditionsProvider:
if hasattr(self._solar_conditions, key):
setattr(self._solar_conditions, key, value)
self._solar_conditions.infer_descriptions()
self._solar_conditions_cache['solar_conditions'] = self._solar_conditions
+4 -26
View File
@@ -5,23 +5,15 @@ import os
import signal
import sys
from diskcache import Cache
from core.cleanup import CleanupTimer
from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL
from core.constants import SOFTWARE_VERSION
from core.data_store import DATA_STORE
from core.lookup_helper import lookup_helper
from core.status_reporter import StatusReporter
from data.solar_conditions import SolarConditions
from server.webserver import WebServer
# Globals
spots = Cache('cache/spots_cache')
alerts = Cache('cache/alerts_cache')
solar_conditions_cache = Cache('cache/solar_conditions_cache')
solar_conditions = solar_conditions_cache.get('solar_conditions', SolarConditions())
web_server = None
status_data = {}
spot_providers = []
alert_providers = []
solar_condition_providers = []
@@ -46,13 +38,7 @@ def shutdown(_signum=None, _frame=None):
for scp in solar_condition_providers:
if scp.enabled:
scp.stop()
if cleanup_timer:
cleanup_timer.stop()
if lookup_helper:
lookup_helper.stop()
spots.close()
alerts.close()
solar_conditions_cache.close()
DATA_STORE.close()
os._exit(0)
@@ -103,13 +89,12 @@ if __name__ == '__main__':
lookup_helper.start()
# Set up web server
web_server = WebServer(spots=spots, alerts=alerts, solar_conditions=solar_conditions, status_data=status_data)
web_server = WebServer()
# Fetch, set up and start spot providers
for entry in config["spot-providers"]:
spot_providers.append(get_spot_provider_from_config(entry))
for p in spot_providers:
p.setup(spots=spots, web_server=web_server)
if p.enabled:
p.start()
@@ -117,7 +102,6 @@ if __name__ == '__main__':
for entry in config["alert-providers"]:
alert_providers.append(get_alert_provider_from_config(entry))
for p in alert_providers:
p.setup(alerts=alerts, web_server=web_server)
if p.enabled:
p.start()
@@ -125,17 +109,11 @@ if __name__ == '__main__':
for entry in config.get("solar-condition-providers", []):
solar_condition_providers.append(get_solar_conditions_provider_from_config(entry))
for p in solar_condition_providers:
p.setup(solar_conditions=solar_conditions, solar_conditions_cache=solar_conditions_cache)
if p.enabled:
p.start()
# Set up timer to clear spot list of old data
cleanup_timer = CleanupTimer(spots=spots, alerts=alerts, web_server=web_server, cleanup_interval=60)
cleanup_timer.start()
# Set up status reporter
status_reporter = StatusReporter(status_data=status_data, spots=spots, alerts=alerts, web_server=web_server,
cleanup_timer=cleanup_timer, spot_providers=spot_providers,
status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers,
alert_providers=alert_providers,
solar_condition_providers=solar_condition_providers, run_interval=5)
status_reporter.start()
+2 -2
View File
@@ -3,8 +3,8 @@ from datetime import datetime
import pytz
from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
from core.constants import HTTP_HEADERS
from core.url_data_cache import URL_DATA_CACHE
from data.sig_ref import SIGRef
from data.spot import Spot
from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -56,7 +56,7 @@ class GMA(HTTPSpotProvider):
# GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
if "REF" in source_spot:
try:
ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
ref_response = URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
headers=HTTP_HEADERS)
# Sometimes this is blank even if it's a 200 response, so handle that
if ref_response.ok and ref_response.text is not None and ref_response.text != "":
+3 -13
View File
@@ -2,7 +2,7 @@ from datetime import datetime
import pytz
from core.config import MAX_SPOT_AGE
from core.data_store import DATA_STORE
class SpotProvider:
@@ -16,14 +16,7 @@ class SpotProvider:
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
self.status = "Not Started" if self.enabled else "Disabled"
self._spots = None
self._web_server = None
def setup(self, spots, web_server):
"""Set up the provider, e.g. giving it the spot list to work from"""
self._spots = spots
self._web_server = web_server
self._spots = DATA_STORE.spots
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -59,10 +52,7 @@ class SpotProvider:
def _add_spot(self, spot):
if not spot.expired():
self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
# Ping the web server in case we have any SSE connections that need to see this immediately
if self._web_server:
self._web_server.notify_new_spot(spot)
self._spots.set(spot.id, spot)
def stop(self):
"""Stop any threads and prepare for application shutdown"""
+3 -3
View File
@@ -16,7 +16,7 @@ L.WorkedAllBritainIreland = L.LayerGroup.extend({
// Workaround to load the geodesy modules in non-modular code. Once we have loaded all three modules, trigger a
// first draw.
import(new URL('./modules/geodesy/osgridref.js', import.meta.url).href)
import("https://misc.ianrenton.com/Leaflet.WorkedAllBritainIreland/modules/geodesy/osgridref.js")
.then(module => {
this._osGridLibrary = module;
if (this._ieGridLibrary && this._utmLibrary) {
@@ -27,7 +27,7 @@ L.WorkedAllBritainIreland = L.LayerGroup.extend({
console.log("Error loading OS Grid Ref library, GB WAB squares may not be available.");
console.log(error);
});
import(new URL('./modules/geodesy/iegridref.js', import.meta.url).href)
import("https://misc.ianrenton.com/Leaflet.WorkedAllBritainIreland/modules/geodesy/iegridref.js")
.then(module => {
this._ieGridLibrary = module;
if (this._osGridLibrary && this._utmLibrary) {
@@ -38,7 +38,7 @@ L.WorkedAllBritainIreland = L.LayerGroup.extend({
console.log("Error loading IE Grid Ref library, NI WAB squares may not be available.");
console.log(error);
});
import(new URL('./modules/geodesy/utm_ci.js', import.meta.url).href)
import("https://misc.ianrenton.com/Leaflet.WorkedAllBritainIreland/modules/geodesy/utm_ci.js")
.then(module => {
this._utmLibrary = module;
if (this._osGridLibrary && this._ieGridLibrary) {
-22
View File
@@ -1,22 +0,0 @@
The MIT License (MIT)
Copyright (c) 2014 Chris Veness
With some additional code & modifications by Ian Renton, 2025
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-326
View File
@@ -1,326 +0,0 @@
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* Ordnance Survey of Ireland Grid Reference funcs (c) Chris Veness 2005-2021 & Ian Renton 2025 */
/* MIT Licence */
/* www.movable-type.co.uk/scripts/latlong-gridref.html */
/* www.movable-type.co.uk/scripts/geodesy-library.html#IeGridRef */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
import LatLonEllipsoidal, { Dms } from 'https://cdn.jsdelivr.net/npm/geodesy@2/latlon-ellipsoidal-datum.js';
/**
* Ordnance Survey of Ireland & Northern Ireland grid reference calculations, based on the
* IeGridRef class in the geodesy library at https://github.com/chrisveness/geodesy
*/
/* IeGridRef - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
const nationalGrid = {
trueOrigin: { lat: 53.5, lon: -8 }, // true origin of Irish grid 53°30N, 8°W
falseOrigin: { easting: -200e3, northing: -250e3 }, // easting & northing of false origin, metres from true origin
scaleFactor: 1.000035, // scale factor on central meridian
ellipsoid: LatLonEllipsoidal.ellipsoids.Airy1830,
};
/**
* Irish Grid References with methods to parse and convert them to latitude/longitude points.
*/
class IeGridRef {
/**
* Creates an IeGridRef object.
*
* @param {number} easting - Easting in metres from OS Grid false origin.
* @param {number} northing - Northing in metres from OS Grid false origin.
*
* @example
* import IeGridRef from '/js/geodesy/IeGridRef.js';
* const gridref = new IeGridRef(651409, 313177);
*/
constructor(easting, northing) {
this.easting = Number(easting);
this.northing = Number(northing);
if (isNaN(easting) || this.easting<0 || this.easting>7000e3) throw new RangeError(`invalid easting ${easting}`);
if (isNaN(northing) || this.northing<0 || this.northing>13000e3) throw new RangeError(`invalid northing ${northing}`);
}
/**
* Converts this Irish Grid Reference easting/northing coordinate to latitude/longitude
* (SW corner of grid square).
*
* While OS Grid References are based on OSGB-36, the Ordnance Survey have deprecated the use of
* OSGB-36 for latitude/longitude coordinates (in favour of WGS-84), hence this function returns
* WGS-84 by default, with OSGB-36 as an option. See www.ordnancesurvey.co.uk/blog/2014/12/2.
*
* Note formulation implemented here due to Thomas, Redfearn, etc is as published by OS, but is
* inferior to Krüger as used by e.g. Karney 2011.
*
* @param {LatLon.datum} [datum=WGS84] - Datum to convert grid reference into.
* @returns {LatLon} Latitude/longitude of supplied grid reference.
*
* @example
* const gridref = new IeGridRef(651409.903, 313177.270);
* const pWgs84 = gridref.toLatLon(); // 52°3928.723″N, 001°4257.787″E
* // to obtain (historical) OSGB36 lat/lon point:
* const pOsgb = gridref.toLatLon(LatLon.datums.OSGB36); // 52°3927.253″N, 001°4304.518″E
*/
toLatLon(datum=LatLonEllipsoidal.datums.WGS84) {
const { easting: E, northing: N } = this;
const { a, b } = nationalGrid.ellipsoid; // a = 6377563.396, b = 6356256.909
const φ0 = nationalGrid.trueOrigin.lat.toRadians(); // latitude of true origin
const λ0 = nationalGrid.trueOrigin.lon.toRadians(); // longitude of true origin
const E0 = -nationalGrid.falseOrigin.easting; // easting of true origin
const N0 = -nationalGrid.falseOrigin.northing; // northing of true origin
const F0 = nationalGrid.scaleFactor; // scale factor
const e2 = 1 - (b*b)/(a*a); // eccentricity squared
const n = (a-b)/(a+b), n2 = n*n, n3 = n*n*n; // n, n², n³
let φ=φ0, M=0;
do {
φ = (N-N0-M)/(a*F0) + φ;
const Ma = (1 + n + (5/4)*n2 + (5/4)*n3) * (φ-φ0);
const Mb = (3*n + 3*n2 + (21/8)*n3) * Math.sin(φ-φ0) * Math.cos(φ+φ0);
const Mc = ((15/8)*n2 + (15/8)*n3) * Math.sin(2*(φ-φ0)) * Math.cos(2*(φ+φ0));
const Md = (35/24)*n3 * Math.sin(3*(φ-φ0)) * Math.cos(3*(φ+φ0));
M = b * F0 * (Ma - Mb + Mc - Md); // meridional arc
} while (Math.abs(N-N0-M) >= 0.00001); // ie until < 0.01mm
const cosφ = Math.cos(φ), sinφ = Math.sin(φ);
const ν = a*F0/Math.sqrt(1-e2*sinφ*sinφ); // nu = transverse radius of curvature
const ρ = a*F0*(1-e2)/Math.pow(1-e2*sinφ*sinφ, 1.5); // rho = meridional radius of curvature
const η2 = ν/ρ-1; // eta = ?
const tanφ = Math.tan(φ);
const tan2φ = tanφ*tanφ, tan4φ = tan2φ*tan2φ, tan6φ = tan4φ*tan2φ;
const secφ = 1/cosφ;
const ν3 = ν*ν*ν, ν5 = ν3*ν*ν, ν7 = ν5*ν*ν;
const VII = tanφ/(2*ρ*ν);
const VIII = tanφ/(24*ρ*ν3)*(5+3*tan2φ+η2-9*tan2φ*η2);
const IX = tanφ/(720*ρ*ν5)*(61+90*tan2φ+45*tan4φ);
const X = secφ/ν;
const XI = secφ/(6*ν3)*(ν/ρ+2*tan2φ);
const XII = secφ/(120*ν5)*(5+28*tan2φ+24*tan4φ);
const XIIA = secφ/(5040*ν7)*(61+662*tan2φ+1320*tan4φ+720*tan6φ);
const dE = (E-E0), dE2 = dE*dE, dE3 = dE2*dE, dE4 = dE2*dE2, dE5 = dE3*dE2, dE6 = dE4*dE2, dE7 = dE5*dE2;
φ = φ - VII*dE2 + VIII*dE4 - IX*dE6;
const λ = λ0 + X*dE - XI*dE3 + XII*dE5 - XIIA*dE7;
let point = new LatLon_IeGridRef(φ.toDegrees(), λ.toDegrees(), 0, LatLonEllipsoidal.datums.OSGB36);
if (datum != LatLonEllipsoidal.datums.OSGB36) {
// if point is required in datum other than OSGB36, convert it
point = point.convertDatum(datum);
// convertDatum() gives us a LatLon: convert to LatLon_IeGridRef which includes toOsGrid()
point = new LatLon_IeGridRef(point.lat, point.lon, point.height, point.datum);
}
return point;
}
/**
* Parses grid reference to IeGridRef object.
*
* Accepts standard grid references (eg 'G 387 148'), with or without whitespace separators, from
* two-digit references up to 10-digit references (1m × 1m square), or fully numeric comma-separated
* references in metres (eg '438700,114800').
*
* @param {string} gridref - Standard format OS Grid Reference.
* @returns {IeGridRef} Numeric version of grid reference in metres from false origin (SW corner of
* supplied grid square).
* @throws {Error} Invalid grid reference.
*
* @example
* const grid = IeGridRef.parse('G 51409 13177'); // grid: { easting: 651409, northing: 313177 }
*/
static parse(gridref) {
gridref = String(gridref).trim();
// check for fully numeric comma-separated gridref format
let match = gridref.match(/^(\d+),\s*(\d+)$/);
if (match) return new IeGridRef(match[1], match[2]);
// validate format
match = gridref.match(/^[ABCDEFGHJKLMNOPQRSTUVWXYZ]\s*[0-9]+\s*[0-9]+$/i);
if (!match) throw new Error(`invalid grid reference ${gridref}`);
// get numeric values of letter references, mapping A->0, B->1, C->2, etc:
let l1 = gridref.toUpperCase().charCodeAt(0) - 'A'.charCodeAt(0); // 100km square
// shuffle down letters after 'I' since 'I' is not used in grid:
if (l1 > 7) l1--;
// convert grid letters into 100km-square indexes from false origin (grid square SV):
const e100km = l1 % 5;
const n100km = 4 - Math.floor(l1 / 5);
// skip grid letters to get numeric (easting/northing) part of ref
let en = gridref.slice(1).trim().split(/\s+/);
// if e/n not whitespace separated, split half way
if (en.length == 1) en = [ en[0].slice(0, en[0].length / 2), en[0].slice(en[0].length / 2) ];
// validation
if (en[0].length != en[1].length) throw new Error(`invalid grid reference ${gridref}`);
// standardise to 10-digit refs (metres)
en[0] = en[0].padEnd(5, '0');
en[1] = en[1].padEnd(5, '0');
const e = e100km + en[0];
const n = n100km + en[1];
return new IeGridRef(e, n);
}
/**
* Converts this numeric grid reference to standard OS of Ireland Grid Reference.
*
* @param {number} [digits=10] - Precision of returned grid reference (10 digits = metres);
* digits=0 will return grid reference in numeric format.
* @returns {string} This grid reference in standard format.
*
* @example
* const gridref = new IeGridRef(651409, 313177).toString(8); // 'TG 5140 1317'
* const gridref = new IeGridRef(651409, 313177).toString(0); // '651409,313177'
*/
toString(digits=10) {
if (![ 0,2,4,6,8,10,12,14,16 ].includes(Number(digits))) throw new RangeError(`invalid precision ${digits}`); // eslint-disable-line comma-spacing
let { easting: e, northing: n } = this;
// use digits = 0 to return numeric format (in metres) - note northing may be >= 1e7
if (digits == 0) {
const format = { useGrouping: false, minimumIntegerDigits: 6, maximumFractionDigits: 3 };
const ePad = e.toLocaleString('en', format);
const nPad = n.toLocaleString('en', format);
return `${ePad},${nPad}`;
}
// get the 100km-grid indices
const e100km = Math.floor(e / 100000), n100km = Math.floor(n / 100000);
// translate those into the numeric equivalent of the grid letters
let l1 = (n100km) * 5 % 25 + e100km % 5;
return null; // haven't done this maths yet
// compensate for skipped 'I' and calculate grid letter
if (l1 > 7) l1++;
const letter = String.fromCharCode(l1 + 'A'.charCodeAt(0));
// strip 100km-grid indices from easting & northing, and reduce precision
e = Math.floor((e % 100000) / Math.pow(10, 5 - digits / 2));
n = Math.floor((n % 100000) / Math.pow(10, 5 - digits / 2));
// pad eastings & northings with leading zeros
e = e.toString().padStart(digits/2, '0');
n = n.toString().padStart(digits/2, '0');
return `${letter} ${e} ${n}`;
}
}
/* LatLon_IeGridRef - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/**
* Extends LatLon class with method to convert LatLon point to Irish Grid Reference.
*
* @extends LatLonEllipsoidal
*/
class LatLon_IeGridRef extends LatLonEllipsoidal {
/**
* Converts latitude/longitude to Ordnance Survey of Ireland grid reference easting/northing coordinate.
*
* @returns {IeGridRef} Irish Grid Reference easting/northing.
*
* @example
* const grid = new LatLon(52.65798, 1.71605).toOsGrid(); // TG 51409 13177
* // for conversion of (historical) OSGB36 latitude/longitude point:
* const grid = new LatLon(52.65798, 1.71605).toOsGrid(LatLon.datums.OSGB36);
*/
toOsGrid() {
// if necessary convert to OSGB36 first
const point = this.datum == LatLonEllipsoidal.datums.OSGB36
? this
: this.convertDatum(LatLonEllipsoidal.datums.OSGB36);
const φ = point.lat.toRadians();
const λ = point.lon.toRadians();
const { a, b } = nationalGrid.ellipsoid; // a = 6377563.396, b = 6356256.909
const φ0 = nationalGrid.trueOrigin.lat.toRadians(); // latitude of true origin
const λ0 = nationalGrid.trueOrigin.lon.toRadians(); // longitude of true origin
const E0 = -nationalGrid.falseOrigin.easting; // easting of true origin
const N0 = -nationalGrid.falseOrigin.northing; // northing of true origin
const F0 = nationalGrid.scaleFactor; // scale factor
const e2 = 1 - (b*b)/(a*a); // eccentricity squared
const n = (a-b)/(a+b), n2 = n*n, n3 = n*n*n; // n, n², n³
const cosφ = Math.cos(φ), sinφ = Math.sin(φ);
const ν = a*F0/Math.sqrt(1-e2*sinφ*sinφ); // nu = transverse radius of curvature
const ρ = a*F0*(1-e2)/Math.pow(1-e2*sinφ*sinφ, 1.5); // rho = meridional radius of curvature
const η2 = ν/ρ-1; // eta = ?
const Ma = (1 + n + (5/4)*n2 + (5/4)*n3) * (φ-φ0);
const Mb = (3*n + 3*n2 + (21/8)*n3) * Math.sin(φ-φ0) * Math.cos(φ+φ0);
const Mc = ((15/8)*n2 + (15/8)*n3) * Math.sin(2*(φ-φ0)) * Math.cos(2*(φ+φ0));
const Md = (35/24)*n3 * Math.sin(3*(φ-φ0)) * Math.cos(3*(φ+φ0));
const M = b * F0 * (Ma - Mb + Mc - Md); // meridional arc
const cos3φ = cosφ*cosφ*cosφ;
const cos5φ = cos3φ*cosφ*cosφ;
const tan2φ = Math.tan(φ)*Math.tan(φ);
const tan4φ = tan2φ*tan2φ;
const I = M + N0;
const II = (ν/2)*sinφ*cosφ;
const III = (ν/24)*sinφ*cos3φ*(5-tan2φ+9*η2);
const IIIA = (ν/720)*sinφ*cos5φ*(61-58*tan2φ+tan4φ);
const IV = ν*cosφ;
const V = (ν/6)*cos3φ*(ν/ρ-tan2φ);
const VI = (ν/120) * cos5φ * (5 - 18*tan2φ + tan4φ + 14*η2 - 58*tan2φ*η2);
const Δλ = λ-λ0;
const Δλ2 = Δλ*Δλ, Δλ3 = Δλ2*Δλ, Δλ4 = Δλ3*Δλ, Δλ5 = Δλ4*Δλ, Δλ6 = Δλ5*Δλ;
let N = I + II*Δλ2 + III*Δλ4 + IIIA*Δλ6;
let E = E0 + IV*Δλ + V*Δλ3 + VI*Δλ5;
N = Number(N.toFixed(3)); // round to mm precision
E = Number(E.toFixed(3));
try {
return new IeGridRef(E, N); // note: gets truncated to SW corner of 1m grid square
} catch (e) {
throw new Error(`${e.message} from (${point.lat.toFixed(6)},${point.lon.toFixed(6)}).toOsGrid()`);
}
}
/**
* Override LatLonEllipsoidal.convertDatum() with version which returns LatLon_IeGridRef.
*/
convertDatum(toDatum) {
const osieED = super.convertDatum(toDatum); // returns LatLonEllipsoidal_Datum
const osieOSGR = new LatLon_IeGridRef(osieED.lat, osieED.lon, osieED.height, osieED.datum);
return osieOSGR;
}
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
export { IeGridRef as default, LatLon_IeGridRef as LatLon, Dms };
-348
View File
@@ -1,348 +0,0 @@
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* Ordnance Survey Grid Reference functions (c) Chris Veness 2005-2021 */
/* MIT Licence */
/* www.movable-type.co.uk/scripts/latlong-gridref.html */
/* www.movable-type.co.uk/scripts/geodesy-library.html#osgridref */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
import LatLonEllipsoidal, { Dms } from 'https://cdn.jsdelivr.net/npm/geodesy@2/latlon-ellipsoidal-datum.js';
/**
* Ordnance Survey OSGB grid references provide geocoordinate references for UK mapping purposes.
*
* Formulation implemented here due to Thomas, Redfearn, etc is as published by OS, but is inferior
* to Krüger as used by e.g. Karney 2011.
*
* www.ordnancesurvey.co.uk/documents/resources/guide-coordinate-systems-great-britain.pdf.
*
* Note OSGB grid references cover Great Britain only; Ireland and the Channel Islands have their
* own references.
*
* Note that these formulae are based on ellipsoidal calculations, and according to the OS are
* accurate to about 45 metres for greater accuracy, a geoid-based transformation (OSTN15) must
* be used.
*/
/*
* Converted 2015 to work with WGS84 by default, OSGB36 as option;
* www.ordnancesurvey.co.uk/blog/2014/12/confirmation-on-changes-to-latitude-and-longitude
*/
/* OsGridRef - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
const nationalGrid = {
trueOrigin: { lat: 49, lon: -2 }, // true origin of grid 49°N,2°W on OSGB36 datum
falseOrigin: { easting: -400e3, northing: 100e3 }, // easting & northing of false origin, metres from true origin
scaleFactor: 0.9996012717, // scale factor on central meridian
ellipsoid: LatLonEllipsoidal.ellipsoids.Airy1830,
};
// note Irish National Grid uses t/o 53°30N, 8°W, f/o 200kmW, 250kmS, scale factor 1.000035, on Airy 1830 Modified ellipsoid
/**
* OS Grid References with methods to parse and convert them to latitude/longitude points.
*/
class OsGridRef {
/**
* Creates an OsGridRef object.
*
* @param {number} easting - Easting in metres from OS Grid false origin.
* @param {number} northing - Northing in metres from OS Grid false origin.
*
* @example
* import OsGridRef from '/js/geodesy/osgridref.js';
* const gridref = new OsGridRef(651409, 313177);
*/
constructor(easting, northing) {
this.easting = Number(easting);
this.northing = Number(northing);
if (isNaN(easting) || this.easting<0 || this.easting>700e3) throw new RangeError(`invalid easting ${easting}`);
if (isNaN(northing) || this.northing<0 || this.northing>1300e3) throw new RangeError(`invalid northing ${northing}`);
}
/**
* Converts this Ordnance Survey Grid Reference easting/northing coordinate to latitude/longitude
* (SW corner of grid square).
*
* While OS Grid References are based on OSGB-36, the Ordnance Survey have deprecated the use of
* OSGB-36 for latitude/longitude coordinates (in favour of WGS-84), hence this function returns
* WGS-84 by default, with OSGB-36 as an option. See www.ordnancesurvey.co.uk/blog/2014/12/2.
*
* Note formulation implemented here due to Thomas, Redfearn, etc is as published by OS, but is
* inferior to Krüger as used by e.g. Karney 2011.
*
* @param {LatLon.datum} [datum=WGS84] - Datum to convert grid reference into.
* @returns {LatLon} Latitude/longitude of supplied grid reference.
*
* @example
* const gridref = new OsGridRef(651409.903, 313177.270);
* const pWgs84 = gridref.toLatLon(); // 52°3928.723″N, 001°4257.787″E
* // to obtain (historical) OSGB36 lat/lon point:
* const pOsgb = gridref.toLatLon(LatLon.datums.OSGB36); // 52°3927.253″N, 001°4304.518″E
*/
toLatLon(datum=LatLonEllipsoidal.datums.WGS84) {
const { easting: E, northing: N } = this;
const { a, b } = nationalGrid.ellipsoid; // a = 6377563.396, b = 6356256.909
const φ0 = nationalGrid.trueOrigin.lat.toRadians(); // latitude of true origin, 49°N
const λ0 = nationalGrid.trueOrigin.lon.toRadians(); // longitude of true origin, 2°W
const E0 = -nationalGrid.falseOrigin.easting; // easting of true origin, 400km
const N0 = -nationalGrid.falseOrigin.northing; // northing of true origin, -100km
const F0 = nationalGrid.scaleFactor; // 0.9996012717
const e2 = 1 - (b*b)/(a*a); // eccentricity squared
const n = (a-b)/(a+b), n2 = n*n, n3 = n*n*n; // n, n², n³
let φ=φ0, M=0;
do {
φ = (N-N0-M)/(a*F0) + φ;
const Ma = (1 + n + (5/4)*n2 + (5/4)*n3) * (φ-φ0);
const Mb = (3*n + 3*n2 + (21/8)*n3) * Math.sin(φ-φ0) * Math.cos(φ+φ0);
const Mc = ((15/8)*n2 + (15/8)*n3) * Math.sin(2*(φ-φ0)) * Math.cos(2*(φ+φ0));
const Md = (35/24)*n3 * Math.sin(3*(φ-φ0)) * Math.cos(3*(φ+φ0));
M = b * F0 * (Ma - Mb + Mc - Md); // meridional arc
} while (Math.abs(N-N0-M) >= 0.00001); // ie until < 0.01mm
const cosφ = Math.cos(φ), sinφ = Math.sin(φ);
const ν = a*F0/Math.sqrt(1-e2*sinφ*sinφ); // nu = transverse radius of curvature
const ρ = a*F0*(1-e2)/Math.pow(1-e2*sinφ*sinφ, 1.5); // rho = meridional radius of curvature
const η2 = ν/ρ-1; // eta = ?
const tanφ = Math.tan(φ);
const tan2φ = tanφ*tanφ, tan4φ = tan2φ*tan2φ, tan6φ = tan4φ*tan2φ;
const secφ = 1/cosφ;
const ν3 = ν*ν*ν, ν5 = ν3*ν*ν, ν7 = ν5*ν*ν;
const VII = tanφ/(2*ρ*ν);
const VIII = tanφ/(24*ρ*ν3)*(5+3*tan2φ+η2-9*tan2φ*η2);
const IX = tanφ/(720*ρ*ν5)*(61+90*tan2φ+45*tan4φ);
const X = secφ/ν;
const XI = secφ/(6*ν3)*(ν/ρ+2*tan2φ);
const XII = secφ/(120*ν5)*(5+28*tan2φ+24*tan4φ);
const XIIA = secφ/(5040*ν7)*(61+662*tan2φ+1320*tan4φ+720*tan6φ);
const dE = (E-E0), dE2 = dE*dE, dE3 = dE2*dE, dE4 = dE2*dE2, dE5 = dE3*dE2, dE6 = dE4*dE2, dE7 = dE5*dE2;
φ = φ - VII*dE2 + VIII*dE4 - IX*dE6;
const λ = λ0 + X*dE - XI*dE3 + XII*dE5 - XIIA*dE7;
let point = new LatLon_OsGridRef(φ.toDegrees(), λ.toDegrees(), 0, LatLonEllipsoidal.datums.OSGB36);
if (datum != LatLonEllipsoidal.datums.OSGB36) {
// if point is required in datum other than OSGB36, convert it
point = point.convertDatum(datum);
// convertDatum() gives us a LatLon: convert to LatLon_OsGridRef which includes toOsGrid()
point = new LatLon_OsGridRef(point.lat, point.lon, point.height, point.datum);
}
return point;
}
/**
* Parses grid reference to OsGridRef object.
*
* Accepts standard grid references (eg 'SU 387 148'), with or without whitespace separators, from
* two-digit references up to 10-digit references (1m × 1m square), or fully numeric comma-separated
* references in metres (eg '438700,114800').
*
* @param {string} gridref - Standard format OS Grid Reference.
* @returns {OsGridRef} Numeric version of grid reference in metres from false origin (SW corner of
* supplied grid square).
* @throws {Error} Invalid grid reference.
*
* @example
* const grid = OsGridRef.parse('TG 51409 13177'); // grid: { easting: 651409, northing: 313177 }
*/
static parse(gridref) {
gridref = String(gridref).trim();
// check for fully numeric comma-separated gridref format
let match = gridref.match(/^(\d+),\s*(\d+)$/);
if (match) return new OsGridRef(match[1], match[2]);
// validate format
match = gridref.match(/^[HNOST][ABCDEFGHJKLMNOPQRSTUVWXYZ]\s*[0-9]+\s*[0-9]+$/i);
if (!match) throw new Error(`invalid grid reference ${gridref}`);
// get numeric values of letter references, mapping A->0, B->1, C->2, etc:
let l1 = gridref.toUpperCase().charCodeAt(0) - 'A'.charCodeAt(0); // 500km square
let l2 = gridref.toUpperCase().charCodeAt(1) - 'A'.charCodeAt(0); // 100km square
// shuffle down letters after 'I' since 'I' is not used in grid:
if (l1 > 7) l1--;
if (l2 > 7) l2--;
// convert grid letters into 100km-square indexes from false origin (grid square SV):
const e100km = ((l1 - 2) % 5) * 5 + (l2 % 5);
const n100km = (19 - Math.floor(l1 / 5) * 5) - Math.floor(l2 / 5);
// skip grid letters to get numeric (easting/northing) part of ref
let en = gridref.slice(2).trim().split(/\s+/);
// if e/n not whitespace separated, split half way
if (en.length == 1) en = [ en[0].slice(0, en[0].length / 2), en[0].slice(en[0].length / 2) ];
// validation
if (en[0].length != en[1].length) throw new Error(`invalid grid reference ${gridref}`);
// standardise to 10-digit refs (metres)
en[0] = en[0].padEnd(5, '0');
en[1] = en[1].padEnd(5, '0');
const e = e100km + en[0];
const n = n100km + en[1];
return new OsGridRef(e, n);
}
/**
* Converts this numeric grid reference to standard OS Grid Reference.
*
* @param {number} [digits=10] - Precision of returned grid reference (10 digits = metres);
* digits=0 will return grid reference in numeric format.
* @returns {string} This grid reference in standard format.
*
* @example
* const gridref = new OsGridRef(651409, 313177).toString(8); // 'TG 5140 1317'
* const gridref = new OsGridRef(651409, 313177).toString(0); // '651409,313177'
*/
toString(digits=10) {
if (![ 0,2,4,6,8,10,12,14,16 ].includes(Number(digits))) throw new RangeError(`invalid precision ${digits}`); // eslint-disable-line comma-spacing
let { easting: e, northing: n } = this;
// use digits = 0 to return numeric format (in metres) - note northing may be >= 1e7
if (digits == 0) {
const format = { useGrouping: false, minimumIntegerDigits: 6, maximumFractionDigits: 3 };
const ePad = e.toLocaleString('en', format);
const nPad = n.toLocaleString('en', format);
return `${ePad},${nPad}`;
}
// get the 100km-grid indices
const e100km = Math.floor(e / 100000), n100km = Math.floor(n / 100000);
// translate those into numeric equivalents of the grid letters
let l1 = (19 - n100km) - (19 - n100km) % 5 + Math.floor((e100km + 10) / 5);
let l2 = (19 - n100km) * 5 % 25 + e100km % 5;
// compensate for skipped 'I' and build grid letter-pairs
if (l1 > 7) l1++;
if (l2 > 7) l2++;
const letterPair = String.fromCharCode(l1 + 'A'.charCodeAt(0), l2 + 'A'.charCodeAt(0));
// strip 100km-grid indices from easting & northing, and reduce precision
e = Math.floor((e % 100000) / Math.pow(10, 5 - digits / 2));
n = Math.floor((n % 100000) / Math.pow(10, 5 - digits / 2));
// pad eastings & northings with leading zeros
e = e.toString().padStart(digits/2, '0');
n = n.toString().padStart(digits/2, '0');
return `${letterPair} ${e} ${n}`;
}
}
/* LatLon_OsGridRef - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/**
* Extends LatLon class with method to convert LatLon point to OS Grid Reference.
*
* @extends LatLonEllipsoidal
*/
class LatLon_OsGridRef extends LatLonEllipsoidal {
/**
* Converts latitude/longitude to Ordnance Survey grid reference easting/northing coordinate.
*
* @returns {OsGridRef} OS Grid Reference easting/northing.
*
* @example
* const grid = new LatLon(52.65798, 1.71605).toOsGrid(); // TG 51409 13177
* // for conversion of (historical) OSGB36 latitude/longitude point:
* const grid = new LatLon(52.65798, 1.71605).toOsGrid(LatLon.datums.OSGB36);
*/
toOsGrid() {
// if necessary convert to OSGB36 first
const point = this.datum == LatLonEllipsoidal.datums.OSGB36
? this
: this.convertDatum(LatLonEllipsoidal.datums.OSGB36);
const φ = point.lat.toRadians();
const λ = point.lon.toRadians();
const { a, b } = nationalGrid.ellipsoid; // a = 6377563.396, b = 6356256.909
const φ0 = nationalGrid.trueOrigin.lat.toRadians(); // latitude of true origin, 49°N
const λ0 = nationalGrid.trueOrigin.lon.toRadians(); // longitude of true origin, 2°W
const E0 = -nationalGrid.falseOrigin.easting; // easting of true origin, 400km
const N0 = -nationalGrid.falseOrigin.northing; // northing of true origin, -100km
const F0 = nationalGrid.scaleFactor; // 0.9996012717
const e2 = 1 - (b*b)/(a*a); // eccentricity squared
const n = (a-b)/(a+b), n2 = n*n, n3 = n*n*n; // n, n², n³
const cosφ = Math.cos(φ), sinφ = Math.sin(φ);
const ν = a*F0/Math.sqrt(1-e2*sinφ*sinφ); // nu = transverse radius of curvature
const ρ = a*F0*(1-e2)/Math.pow(1-e2*sinφ*sinφ, 1.5); // rho = meridional radius of curvature
const η2 = ν/ρ-1; // eta = ?
const Ma = (1 + n + (5/4)*n2 + (5/4)*n3) * (φ-φ0);
const Mb = (3*n + 3*n2 + (21/8)*n3) * Math.sin(φ-φ0) * Math.cos(φ+φ0);
const Mc = ((15/8)*n2 + (15/8)*n3) * Math.sin(2*(φ-φ0)) * Math.cos(2*(φ+φ0));
const Md = (35/24)*n3 * Math.sin(3*(φ-φ0)) * Math.cos(3*(φ+φ0));
const M = b * F0 * (Ma - Mb + Mc - Md); // meridional arc
const cos3φ = cosφ*cosφ*cosφ;
const cos5φ = cos3φ*cosφ*cosφ;
const tan2φ = Math.tan(φ)*Math.tan(φ);
const tan4φ = tan2φ*tan2φ;
const I = M + N0;
const II = (ν/2)*sinφ*cosφ;
const III = (ν/24)*sinφ*cos3φ*(5-tan2φ+9*η2);
const IIIA = (ν/720)*sinφ*cos5φ*(61-58*tan2φ+tan4φ);
const IV = ν*cosφ;
const V = (ν/6)*cos3φ*(ν/ρ-tan2φ);
const VI = (ν/120) * cos5φ * (5 - 18*tan2φ + tan4φ + 14*η2 - 58*tan2φ*η2);
const Δλ = λ-λ0;
const Δλ2 = Δλ*Δλ, Δλ3 = Δλ2*Δλ, Δλ4 = Δλ3*Δλ, Δλ5 = Δλ4*Δλ, Δλ6 = Δλ5*Δλ;
let N = I + II*Δλ2 + III*Δλ4 + IIIA*Δλ6;
let E = E0 + IV*Δλ + V*Δλ3 + VI*Δλ5;
N = Number(N.toFixed(3)); // round to mm precision
E = Number(E.toFixed(3));
try {
return new OsGridRef(E, N); // note: gets truncated to SW corner of 1m grid square
} catch (e) {
throw new Error(`${e.message} from (${point.lat.toFixed(6)},${point.lon.toFixed(6)}).toOsGrid()`);
}
}
/**
* Override LatLonEllipsoidal.convertDatum() with version which returns LatLon_OsGridRef.
*/
convertDatum(toDatum) {
const osgbED = super.convertDatum(toDatum); // returns LatLonEllipsoidal_Datum
const osgbOSGR = new LatLon_OsGridRef(osgbED.lat, osgbED.lon, osgbED.height, osgbED.datum);
return osgbOSGR;
}
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
export { OsGridRef as default, LatLon_OsGridRef as LatLon, Dms };
-413
View File
@@ -1,413 +0,0 @@
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* UTM / WGS-84 Conversion Functions (c) Chris Veness 2014-2022 & Ian Renton 2025 */
/* MIT Licence */
/* www.movable-type.co.uk/scripts/latlong-utm-mgrs.html */
/* www.movable-type.co.uk/scripts/geodesy-library.html#utm */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* eslint-disable indent */
import LatLonEllipsoidal, { Dms } from 'https://cdn.jsdelivr.net/npm/geodesy@2/latlon-ellipsoidal-datum.js';
/**
* The Universal Transverse Mercator (UTM) system is a 2-dimensional Cartesian coordinate system
* providing locations on the surface of the Earth.
*
* UTM is a set of 60 transverse Mercator projections, normally based on the WGS-84 ellipsoid.
* Within each zone, coordinates are represented as eastings and northings, measures in metres; e.g.
* 31 N 448251 5411932.
*
* This method based on Karney 2011 Transverse Mercator with an accuracy of a few nanometers,
* building on Krüger 1912 Konforme Abbildung des Erdellipsoids in der Ebene.
*
* @module utm
*/
/* Utm - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/**
* UTM coordinates, with functions to parse them and convert them to LatLon points.
*/
class Utm {
/**
* Creates a Utm coordinate object comprising zone, hemisphere, easting, northing on a given
* datum (normally WGS84).
*
* @param {number} zone - UTM 6° longitudinal zone (1..60 covering 180°W..180°E).
* @param {string} hemisphere - N for northern hemisphere, S for southern hemisphere.
* @param {number} easting - Easting in metres from false easting (-500km from central meridian).
* @param {number} northing - Northing in metres from equator (N) or from false northing -10,000km (S).
* @param {LatLon.datums} [datum=WGS84] - Datum UTM coordinate is based on.
* @param {number} [convergence=null] - Meridian convergence (bearing of grid north
* clockwise from true north), in degrees.
* @param {number} [scale=null] - Grid scale factor.
* @params {boolean=true} verifyEN - Check easting/northing is within 'normal' values (may be
* suppressed for extended coherent coordinates or alternative datums
* e.g. ED50 (epsg.io/23029).
* @throws {TypeError} Invalid UTM coordinate.
*
* @example
* import Utm from '/js/geodesy/utm.js';
* const utmCoord = new Utm(31, 'N', 448251, 5411932);
*/
constructor(zone, hemisphere, easting, northing, datum=LatLonEllipsoidal.datums.WGS84, convergence=null, scale=null, verifyEN=true) {
if (!(1<=zone && zone<=60)) throw new RangeError(`invalid UTM zone ${zone}`);
if (zone != parseInt(zone)) throw new RangeError(`invalid UTM zone ${zone}`);
if (typeof hemisphere != 'string' || !hemisphere.match(/[NS]/i)) throw new RangeError(`invalid UTM hemisphere ${hemisphere}`);
if (verifyEN) { // (rough) range-check of E/N values
if (!(0<=easting && easting<=1000e3)) throw new RangeError(`invalid UTM easting ${easting}`);
if (hemisphere.toUpperCase()=='N' && !(0<=northing && northing<9329006)) throw new RangeError(`invalid UTM northing ${northing}`);
if (hemisphere.toUpperCase()=='S' && !(1116914<northing && northing<=10000e3)) throw new RangeError(`invalid UTM northing ${northing}`);
}
if (!datum || datum.ellipsoid==undefined) throw new TypeError(`unrecognised datum ${datum}`);
this.zone = Number(zone);
this.hemisphere = hemisphere.toUpperCase();
this.easting = Number(easting);
this.northing = Number(northing);
this.datum = datum;
this.convergence = convergence===null ? null : Number(convergence);
this.scale = scale===null ? null : Number(scale);
}
/**
* Converts UTM zone/easting/northing coordinate to latitude/longitude.
*
* Implements Karneys method, using Krüger series to order n⁶, giving results accurate to 5nm
* for distances up to 3900km from the central meridian.
*
* @param {Utm} utmCoord - UTM coordinate to be converted to latitude/longitude.
* @returns {LatLon} Latitude/longitude of supplied grid reference.
*
* @example
* const grid = new Utm(31, 'N', 448251.795, 5411932.678);
* const latlong = grid.toLatLon(); // 48°5129.52″N, 002°1740.20″E
*/
toLatLon() {
const { zone: z, hemisphere: h } = this;
const falseEasting = 500e3, falseNorthing = 10000e3;
const { a, f } = this.datum.ellipsoid; // WGS-84: a = 6378137, f = 1/298.257223563;
const k0 = 0.9996; // UTM scale on the central meridian
const x = this.easting - falseEasting; // make x ± relative to central meridian
const y = h=='S' ? this.northing - falseNorthing : this.northing; // make y ± relative to equator
// ---- from Karney 2011 Eq 15-22, 36:
const e = Math.sqrt(f*(2-f)); // eccentricity
const n = f / (2 - f); // 3rd flattening
const n2 = n*n, n3 = n*n2, n4 = n*n3, n5 = n*n4, n6 = n*n5;
const A = a/(1+n) * (1 + 1/4*n2 + 1/64*n4 + 1/256*n6); // 2πA is the circumference of a meridian
const η = x / (k0*A);
const ξ = y / (k0*A);
const β = [ null, // note β is one-based array (6th order Krüger expressions)
1/2*n - 2/3*n2 + 37/96*n3 - 1/360*n4 - 81/512*n5 + 96199/604800*n6,
1/48*n2 + 1/15*n3 - 437/1440*n4 + 46/105*n5 - 1118711/3870720*n6,
17/480*n3 - 37/840*n4 - 209/4480*n5 + 5569/90720*n6,
4397/161280*n4 - 11/504*n5 - 830251/7257600*n6,
4583/161280*n5 - 108847/3991680*n6,
20648693/638668800*n6 ];
let ξʹ = ξ;
for (let j=1; j<=6; j++) ξʹ -= β[j] * Math.sin(2*j*ξ) * Math.cosh(2*j*η);
let ηʹ = η;
for (let j=1; j<=6; j++) ηʹ -= β[j] * Math.cos(2*j*ξ) * Math.sinh(2*j*η);
const sinhηʹ = Math.sinh(ηʹ);
const sinξʹ = Math.sin(ξʹ), cosξʹ = Math.cos(ξʹ);
const τʹ = sinξʹ / Math.sqrt(sinhηʹ*sinhηʹ + cosξʹ*cosξʹ);
let δτi = null;
let τi = τʹ;
do {
const σi = Math.sinh(e*Math.atanh(e*τi/Math.sqrt(1+τi*τi)));
const τiʹ = τi * Math.sqrt(1+σi*σi) - σi * Math.sqrt(1+τi*τi);
δτi = (τʹ - τiʹ)/Math.sqrt(1+τiʹ*τiʹ)
* (1 + (1-e*e)*τi*τi) / ((1-e*e)*Math.sqrt(1+τi*τi));
τi += δτi;
} while (Math.abs(δτi) > 1e-12); // using IEEE 754 δτi -> 0 after 2-3 iterations
// note relatively large convergence test as δτi toggles on ±1.12e-16 for eg 31 N 400000 5000000
const τ = τi;
const φ = Math.atan(τ);
let λ = Math.atan2(sinhηʹ, cosξʹ);
// ---- convergence: Karney 2011 Eq 26, 27
let p = 1;
for (let j=1; j<=6; j++) p -= 2*j*β[j] * Math.cos(2*j*ξ) * Math.cosh(2*j*η);
let q = 0;
for (let j=1; j<=6; j++) q += 2*j*β[j] * Math.sin(2*j*ξ) * Math.sinh(2*j*η);
const γʹ = Math.atan(Math.tan(ξʹ) * Math.tanh(ηʹ));
const γʺ = Math.atan2(q, p);
const γ = γʹ + γʺ;
// ---- scale: Karney 2011 Eq 28
const sinφ = Math.sin(φ);
const kʹ = Math.sqrt(1 - e*e*sinφ*sinφ) * Math.sqrt(1 + τ*τ) * Math.sqrt(sinhηʹ*sinhηʹ + cosξʹ*cosξʹ);
const = A / a / Math.sqrt(p*p + q*q);
const k = k0 * kʹ * ;
// ------------
const λ0 = ((z-1)*6 - 180 + 3).toRadians(); // longitude of central meridian
λ += λ0; // move λ from zonal to global coordinates
// round to reasonable precision
const lat = Number(φ.toDegrees().toFixed(14)); // nm precision (1nm = 10^-14°)
const lon = Number(λ.toDegrees().toFixed(14)); // (strictly lat rounding should be φ⋅cosφ!)
const convergence = Number(γ.toDegrees().toFixed(9));
const scale = Number(k.toFixed(12));
const latLong = new LatLon_Utm(lat, lon, 0, this.datum);
// ... and add the convergence and scale into the LatLon object ... wonderful JavaScript!
latLong.convergence = convergence;
latLong.scale = scale;
return latLong;
}
/**
* Parses a Channel Islands (WA/WV) grid reference.
*/
static parseChannelIslandGrid(gridref) {
// validate format
let match = gridref.match(/^W[AV]\s*[0-9]+\s*[0-9]+$/i);
if (!match) throw new Error(`invalid grid reference ${gridref}`);
// skip grid letters to get numeric (easting/northing) part of ref
let en = gridref.slice(2).trim().split(/\s+/);
// if e/n not whitespace separated, split half way
if (en.length == 1) en = [ en[0].slice(0, en[0].length / 2), en[0].slice(en[0].length / 2) ];
// validation
if (en[0].length != en[1].length) throw new Error(`invalid grid reference ${gridref}`);
// standardise to 10-digit refs (metres)
en[0] = en[0].padEnd(5, '0');
en[1] = en[1].padEnd(5, '0');
let utmCoord = "30 N ";
const e = 5 + en[0];
utmCoord += e + " ";
if (gridref.substring(0, 2) === "WA") {
const n = 55 + en[1];
utmCoord += n;
} else if (gridref.substring(0, 2) === "WV") {
const n = 54 + en[1];
utmCoord += n;
}
return Utm.parse(utmCoord);
}
/**
* Parses string representation of UTM coordinate.
*
* A UTM coordinate comprises (space-separated)
* - zone
* - hemisphere
* - easting
* - northing.
*
* @param {string} utmCoord - UTM coordinate (WGS 84).
* @param {Datum} [datum=WGS84] - Datum coordinate is defined in (default WGS 84).
* @returns {Utm} Parsed UTM coordinate.
* @throws {TypeError} Invalid UTM coordinate.
*
* @example
* const utmCoord = Utm.parse('31 N 448251 5411932');
* // utmCoord: {zone: 31, hemisphere: 'N', easting: 448251, northing: 5411932 }
*/
static parse(utmCoord, datum=LatLonEllipsoidal.datums.WGS84) {
// match separate elements (separated by whitespace)
utmCoord = utmCoord.trim().match(/\S+/g);
if (utmCoord==null || utmCoord.length!=4) throw new Error(`invalid UTM coordinate ${utmCoord}`);
const zone = utmCoord[0], hemisphere = utmCoord[1], easting = utmCoord[2], northing = utmCoord[3];
return new this(zone, hemisphere, easting, northing, datum); // 'new this' as may return subclassed types
}
/**
* Returns a string representation of a UTM coordinate.
*
* To distinguish from MGRS grid zone designators, a space is left between the zone and the
* hemisphere.
*
* Note that UTM coordinates get rounded, not truncated (unlike MGRS grid references).
*
* @param {number} [digits=0] - Number of digits to appear after the decimal point (3 ≡ mm).
* @returns {string} A string representation of the coordinate.
*
* @example
* const utm = new Utm('31', 'N', 448251, 5411932).toString(4); // 31 N 448251.0000 5411932.0000
*/
toString(digits=0) {
const z = this.zone.toString().padStart(2, '0');
const h = this.hemisphere;
const e = this.easting.toFixed(digits);
const n = this.northing.toFixed(digits);
return `${z} ${h} ${e} ${n}`;
}
}
/* LatLon_Utm - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/**
* Extends LatLon with method to convert LatLon points to UTM coordinates.
*
* @extends LatLon
*/
class LatLon_Utm extends LatLonEllipsoidal {
/**
* Converts latitude/longitude to UTM coordinate.
*
* Implements Karneys method, using Krüger series to order n⁶, giving results accurate to 5nm
* for distances up to 3900km from the central meridian.
*
* @param {number} [zoneOverride] - Use specified zone rather than zone within which point lies;
* note overriding the UTM zone has the potential to result in negative eastings, and
* perverse results within Norway/Svalbard exceptions.
* @returns {Utm} UTM coordinate.
* @throws {TypeError} Latitude outside UTM limits.
*
* @example
* const latlong = new LatLon(48.8582, 2.2945);
* const utmCoord = latlong.toUtm(); // 31 N 448252 5411933
*/
toUtm(zoneOverride=undefined) {
if (!(-80<=this.lat && this.lat<=84)) throw new RangeError(`latitude ${this.lat} outside UTM limits`);
const falseEasting = 500e3, falseNorthing = 10000e3;
let zone = zoneOverride || Math.floor((this.lon+180)/6) + 1; // longitudinal zone
let λ0 = ((zone-1)*6 - 180 + 3).toRadians(); // longitude of central meridian
// ---- handle Norway/Svalbard exceptions
// grid zones are 8° tall; 0°N is offset 10 into latitude bands array
const mgrsLatBands = 'CDEFGHJKLMNPQRSTUVWXX'; // X is repeated for 80-84°N
const latBand = mgrsLatBands.charAt(Math.floor(this.lat/8+10));
// adjust zone & central meridian for Norway
if (zone==31 && latBand=='V' && this.lon>= 3) { zone++; λ0 += (6).toRadians(); }
// adjust zone & central meridian for Svalbard
if (zone==32 && latBand=='X' && this.lon< 9) { zone--; λ0 -= (6).toRadians(); }
if (zone==32 && latBand=='X' && this.lon>= 9) { zone++; λ0 += (6).toRadians(); }
if (zone==34 && latBand=='X' && this.lon< 21) { zone--; λ0 -= (6).toRadians(); }
if (zone==34 && latBand=='X' && this.lon>=21) { zone++; λ0 += (6).toRadians(); }
if (zone==36 && latBand=='X' && this.lon< 33) { zone--; λ0 -= (6).toRadians(); }
if (zone==36 && latBand=='X' && this.lon>=33) { zone++; λ0 += (6).toRadians(); }
const φ = this.lat.toRadians(); // latitude ± from equator
const λ = this.lon.toRadians() - λ0; // longitude ± from central meridian
// allow alternative ellipsoid to be specified
const ellipsoid = this.datum ? this.datum.ellipsoid : LatLonEllipsoidal.ellipsoids.WGS84;
const { a, f } = ellipsoid; // WGS-84: a = 6378137, f = 1/298.257223563;
const k0 = 0.9996; // UTM scale on the central meridian
// ---- easting, northing: Karney 2011 Eq 7-14, 29, 35:
const e = Math.sqrt(f*(2-f)); // eccentricity
const n = f / (2 - f); // 3rd flattening
const n2 = n*n, n3 = n*n2, n4 = n*n3, n5 = n*n4, n6 = n*n5;
const cosλ = Math.cos(λ), sinλ = Math.sin(λ), tanλ = Math.tan(λ);
const τ = Math.tan(φ); // τ ≡ tanφ, τʹ ≡ tanφʹ; prime (ʹ) indicates angles on the conformal sphere
const σ = Math.sinh(e*Math.atanh(e*τ/Math.sqrt(1+τ*τ)));
const τʹ = τ*Math.sqrt(1+σ*σ) - σ*Math.sqrt(1+τ*τ);
const ξʹ = Math.atan2(τʹ, cosλ);
const ηʹ = Math.asinh(sinλ / Math.sqrt(τʹ*τʹ + cosλ*cosλ));
const A = a/(1+n) * (1 + 1/4*n2 + 1/64*n4 + 1/256*n6); // 2πA is the circumference of a meridian
const α = [ null, // note α is one-based array (6th order Krüger expressions)
1/2*n - 2/3*n2 + 5/16*n3 + 41/180*n4 - 127/288*n5 + 7891/37800*n6,
13/48*n2 - 3/5*n3 + 557/1440*n4 + 281/630*n5 - 1983433/1935360*n6,
61/240*n3 - 103/140*n4 + 15061/26880*n5 + 167603/181440*n6,
49561/161280*n4 - 179/168*n5 + 6601661/7257600*n6,
34729/80640*n5 - 3418889/1995840*n6,
212378941/319334400*n6 ];
let ξ = ξʹ;
for (let j=1; j<=6; j++) ξ += α[j] * Math.sin(2*j*ξʹ) * Math.cosh(2*j*ηʹ);
let η = ηʹ;
for (let j=1; j<=6; j++) η += α[j] * Math.cos(2*j*ξʹ) * Math.sinh(2*j*ηʹ);
let x = k0 * A * η;
let y = k0 * A * ξ;
// ---- convergence: Karney 2011 Eq 23, 24
let pʹ = 1;
for (let j=1; j<=6; j++) pʹ += 2*j*α[j] * Math.cos(2*j*ξʹ) * Math.cosh(2*j*ηʹ);
let qʹ = 0;
for (let j=1; j<=6; j++) qʹ += 2*j*α[j] * Math.sin(2*j*ξʹ) * Math.sinh(2*j*ηʹ);
const γʹ = Math.atan(τʹ / Math.sqrt(1+τʹ*τʹ)*tanλ);
const γʺ = Math.atan2(qʹ, pʹ);
const γ = γʹ + γʺ;
// ---- scale: Karney 2011 Eq 25
const sinφ = Math.sin(φ);
const kʹ = Math.sqrt(1 - e*e*sinφ*sinφ) * Math.sqrt(1 + τ*τ) / Math.sqrt(τʹ*τʹ + cosλ*cosλ);
const = A / a * Math.sqrt(pʹ*pʹ + qʹ*qʹ);
const k = k0 * kʹ * ;
// ------------
// shift x/y to false origins
x = x + falseEasting; // make x relative to false easting
if (y < 0) y = y + falseNorthing; // make y in southern hemisphere relative to false northing
// round to reasonable precision
x = Number(x.toFixed(9)); // nm precision
y = Number(y.toFixed(9)); // nm precision
const convergence = Number(γ.toDegrees().toFixed(9));
const scale = Number(k.toFixed(12));
const h = this.lat>=0 ? 'N' : 'S'; // hemisphere
return new Utm(zone, h, x, y, this.datum, convergence, scale, !!zoneOverride);
}
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
export { Utm as default, LatLon_Utm as LatLon, Dms };
+1 -1
View File
@@ -76,7 +76,7 @@
</div>
<script src="/static/js/add-spot.js?v=1786081935"></script>
<script src="/static/js/add-spot.js?v=1785434214"></script>
<script>$(document).ready(function () {
$("#nav-link-add-spot").addClass("active");
}); <!-- highlight active page in nav --></script>
+1 -1
View File
@@ -75,7 +75,7 @@
</div>
<script src="/static/js/alerts.js?v=1786081935"></script>
<script src="/static/js/alerts.js?v=1785434213"></script>
<script>$(document).ready(function () {
$("#nav-link-alerts").addClass("active");
}); <!-- highlight active page in nav --></script>
+2 -2
View File
@@ -75,8 +75,8 @@
<script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script>
<script src="/static/js/spotsbandsandmap.js?v=1786081935"></script>
<script src="/static/js/bands.js?v=1786081935"></script>
<script src="/static/js/spotsbandsandmap.js?v=1785434214"></script>
<script src="/static/js/bands.js?v=1785434214"></script>
<script>$(document).ready(function () {
$("#nav-link-bands").addClass("active");
}); <!-- highlight active page in nav --></script>
+5 -5
View File
@@ -1,6 +1,6 @@
{% extends "skeleton.html" %}
{% block head_extra %}
<link rel="stylesheet" href="/static/css/style.css?v=1786081935" type="text/css">
<link rel="stylesheet" href="/static/css/style.css?v=1785434213" type="text/css">
<link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
<link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
<link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
<script src="/static/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
<script src="/static/vendor/js/tinycolor2-1.6.0.min.js"></script>
<script src="/static/js/utils.js?v=1786081935"></script>
<script src="/static/js/ui-ham.js?v=1786081935"></script>
<script src="/static/js/geo.js?v=1786081935"></script>
<script src="/static/js/common.js?v=1786081935"></script>
<script src="/static/js/utils.js?v=1785434213"></script>
<script src="/static/js/ui-ham.js?v=1785434213"></script>
<script src="/static/js/geo.js?v=1785434213"></script>
<script src="/static/js/common.js?v=1785434213"></script>
{% end %}
{% block body %}
<div class="container">
+1 -1
View File
@@ -284,7 +284,7 @@
</div>
<script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
<script src="/static/js/conditions.js?v=1786081935"></script>
<script src="/static/js/conditions.js?v=1785434213"></script>
<script>$(document).ready(function () {
$("#nav-link-conditions").addClass("active");
}); <!-- highlight active page in nav --></script>
+2 -2
View File
@@ -108,8 +108,8 @@
<script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script>
<script src="/static/js/spotsbandsandmap.js?v=1786081935"></script>
<script src="/static/js/map.js?v=1786081935"></script>
<script src="/static/js/spotsbandsandmap.js?v=1785434213"></script>
<script src="/static/js/map.js?v=1785434213"></script>
<script>$(document).ready(function () {
$("#nav-link-map").addClass("active");
}); <!-- highlight active page in nav --></script>
+2 -2
View File
@@ -116,8 +116,8 @@
<script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script>
<script src="/static/js/spotsbandsandmap.js?v=1786081935"></script>
<script src="/static/js/spots.js?v=1786081935"></script>
<script src="/static/js/spotsbandsandmap.js?v=1785434213"></script>
<script src="/static/js/spots.js?v=1785434213"></script>
<script>$(document).ready(function () {
$("#nav-link-spots").addClass("active");
}); <!-- highlight active page in nav --></script>
+1 -1
View File
@@ -59,7 +59,7 @@
</div>
</div>
<script src="/static/js/status.js?v=1786081935"></script>
<script src="/static/js/status.js?v=1785434213"></script>
<script>
$(document).ready(function () {
$("#nav-link-status").addClass("active");