w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-05 18:11:41 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Fix a bug where some NaN values could leak into the JSON output, causing it to be invalid. Created a new safe_json_dumps method since we would otherwise now be providing multiple parameters all over the code

Browse Source
This commit is contained in:
Ian Renton
2026-06-28 18:09:44 +01:00
parent 72f367f607
commit b09108b41b
16 changed files with 66 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/utils.py
+8 -5
View File
@@ -1,8 +1,11 @@
def serialize_everything(obj):
"""Convert objects to serialisable things. Used by JSON serialiser as a default when it encounters unserializable things.
Just converts objects to dict. Try to avoid doing anything clever here when serialising spots, because we also need
to receive spots without complex handling."""
return obj.__dict__
import simplejson
def safe_json_dumps(obj):
"""Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
which are invalid in JSON."""
return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
def empty_queue(q):
requirements.txt
+2 -1
View File
@@ -16,4 +16,5 @@ beautifulsoup4~=4.14.2
websocket-client~=1.8.0
tornado~=6.4.2
tornado_eventsource~=3.0.0
geopandas~=0.13.2
geopandas~=0.13.2
simplejson~=4.1.1
server/handlers/api/addspot.py
+13 -21
View File
@@ -14,7 +14,7 @@ from core.constants import UNKNOWN_BAND
from core.lookup_helper import infer_band_from_freq
from core.prometheus_metrics_handler import api_requests_counter
from core.sig_utils import get_ref_regex_for_sig
from core.utils import serialize_everything
from core.utils import safe_json_dumps
from data.sig_ref import SIGRef
from data.spot import Spot
@@ -42,8 +42,7 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject if not allowed
if not ALLOW_SPOTTING:
self.set_status(401)
self.write(json.dumps("Error - this server does not allow new spots to be added via the API.",
default=serialize_everything))
self.write(safe_json_dumps("Error - this server does not allow new spots to be added via the API."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -51,8 +50,7 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject if format not json
if not self.request.headers.get('Content-Type', '').startswith("application/json"):
self.set_status(415)
self.write(
json.dumps("Error - request Content-Type must be application/json", default=serialize_everything))
self.write(safe_json_dumps("Error - request Content-Type must be application/json"))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -61,7 +59,7 @@ class APISpotHandler(tornado.web.RequestHandler):
post_data = self.request.body
if not post_data:
self.set_status(422)
self.write(json.dumps("Error - request body is empty", default=serialize_everything))
self.write(safe_json_dumps("Error - request body is empty"))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -81,8 +79,7 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject if no timestamp, frequency, dx_call or de_call
if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
self.set_status(422)
self.write(json.dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.",
default=serialize_everything))
self.write(safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -90,15 +87,13 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject invalid-looking callsigns
if not re.match(r"^[A-Za-z0-9/\-]*$", spot.dx_call):
self.set_status(422)
self.write(json.dumps("Error - '" + spot.dx_call + "' does not look like a valid callsign.",
default=serialize_everything))
self.write(safe_json_dumps("Error - '" + spot.dx_call + "' does not look like a valid callsign."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
if not re.match(r"^[A-Za-z0-9/\-]*$", spot.de_call):
self.set_status(422)
self.write(json.dumps("Error - '" + spot.de_call + "' does not look like a valid callsign.",
default=serialize_everything))
self.write(safe_json_dumps("Error - '" + spot.de_call + "' does not look like a valid callsign."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -106,8 +101,7 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject if frequency not in a known band
if infer_band_from_freq(spot.freq) == UNKNOWN_BAND:
self.set_status(422)
self.write(json.dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band.",
default=serialize_everything))
self.write(safe_json_dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -117,8 +111,7 @@ class APISpotHandler(tornado.web.RequestHandler):
r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
spot.dx_grid.upper()):
self.set_status(422)
self.write(json.dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid.",
default=serialize_everything))
self.write(safe_json_dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -127,9 +120,8 @@ class APISpotHandler(tornado.web.RequestHandler):
if spot.sig and spot.sig_refs and len(spot.sig_refs) > 0 and spot.sig_refs[0].id and get_ref_regex_for_sig(
spot.sig) and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id):
self.set_status(422)
self.write(json.dumps(
"Error - '" + spot.sig_refs[0].id + "' does not look like a valid reference for " + spot.sig + ".",
default=serialize_everything))
self.write(safe_json_dumps(
"Error - '" + spot.sig_refs[0].id + "' does not look like a valid reference for " + spot.sig + "."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -139,14 +131,14 @@ class APISpotHandler(tornado.web.RequestHandler):
spot.infer_missing()
self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
self.write(json.dumps("OK", default=serialize_everything))
self.write(safe_json_dumps("OK"))
self.set_status(201)
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
except Exception as e:
logging.error(e)
self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
self.write(safe_json_dumps("Error - an internal server error occurred."))
self.set_status(500)
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
server/handlers/api/alerts.py
+5 -6
View File
@@ -1,5 +1,4 @@
import copy
import json
import logging
from datetime import datetime
from queue import Queue
@@ -12,7 +11,7 @@ from tornado import httputil
from tornado.web import Application
from core.prometheus_metrics_handler import api_requests_counter
from core.utils import serialize_everything, empty_queue
from core.utils import safe_json_dumps, empty_queue
from data.lookup_credentials import extract_credentials
SSE_HANDLER_MAX_QUEUE_SIZE = 100
@@ -57,15 +56,15 @@ class APIAlertsHandler(tornado.web.RequestHandler):
data = get_alert_list_with_filters(self._alerts, query_params)
if credentials:
data = self._enrich(data, credentials)
self.write(json.dumps(data, default=serialize_everything))
self.write(safe_json_dumps(data))
self.set_status(200)
except ValueError as e:
logging.error(e)
self.write(json.dumps("Bad request - " + str(e), default=serialize_everything))
self.write(safe_json_dumps("Bad request - " + str(e)))
self.set_status(400)
except Exception as e:
logging.error(e)
self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
self.write(safe_json_dumps("Error - an internal server error occurred."))
self.set_status(500)
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
@@ -150,7 +149,7 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
if self._credentials:
alert = copy.deepcopy(alert)
alert.infer_missing(self._credentials)
self.write_message(msg=json.dumps(alert, default=serialize_everything))
self.write_message(msg=safe_json_dumps(alert))
else:
# Send a keepalive comment if the queue was empty
server/handlers/api/lookups.py
+14 -18
View File
@@ -1,4 +1,3 @@
import json
import logging
import re
from datetime import datetime
@@ -13,7 +12,7 @@ from core.constants import SIGS
from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone
from core.prometheus_metrics_handler import api_requests_counter
from core.sig_utils import get_ref_regex_for_sig, populate_sig_ref_info
from core.utils import serialize_everything
from core.utils import safe_json_dumps
from data.lookup_credentials import extract_credentials
from data.sig_ref import SIGRef
from data.spot import Spot
@@ -65,19 +64,18 @@ class APILookupCallHandler(tornado.web.RequestHandler):
"longitude": fake_spot.dx_longitude,
"location_source": fake_spot.dx_location_source
}
self.write(json.dumps(data, default=serialize_everything))
self.write(safe_json_dumps(data))
else:
self.write(json.dumps("Error - '" + call + "' does not look like a valid callsign.",
default=serialize_everything))
self.write(safe_json_dumps("Error - '" + call + "' does not look like a valid callsign."))
self.set_status(422)
else:
self.write(json.dumps("Error - call must be provided", default=serialize_everything))
self.write(safe_json_dumps("Error - call must be provided"))
self.set_status(422)
except Exception as e:
logging.error(e)
self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
self.write(safe_json_dumps("Error - an internal server error occurred."))
self.set_status(500)
self.set_header("Cache-Control", "no-store")
@@ -114,24 +112,22 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
if sig in list(map(lambda p: p.name.upper(), SIGS)):
if not get_ref_regex_for_sig(sig) or re.match(get_ref_regex_for_sig(sig), ref_id):
data = populate_sig_ref_info(SIGRef(id=ref_id, sig=sig))
self.write(json.dumps(data, default=serialize_everything))
self.write(safe_json_dumps(data))
else:
self.write(
json.dumps(
"Error - '" + ref_id + "' does not look like a valid reference ID for " + sig + ".",
default=serialize_everything))
self.write(safe_json_dumps(
"Error - '" + ref_id + "' does not look like a valid reference ID for " + sig + "."))
self.set_status(422)
else:
self.write(json.dumps("Error - sig '" + sig + "' is not known.", default=serialize_everything))
self.write(safe_json_dumps("Error - sig '" + sig + "' is not known."))
self.set_status(422)
else:
self.write(json.dumps("Error - sig and id must be provided", default=serialize_everything))
self.write(safe_json_dumps("Error - sig and id must be provided"))
self.set_status(422)
except Exception as e:
logging.error(e)
self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
self.write(safe_json_dumps("Error - an internal server error occurred."))
self.set_status(500)
self.set_header("Cache-Control", "no-store")
@@ -185,15 +181,15 @@ class APILookupGridHandler(tornado.web.RequestHandler):
"latitude": lat + lat_cell_size,
"longitude": lon + lon_cell_size,
}}
self.write(json.dumps(response, default=serialize_everything))
self.write(safe_json_dumps(response))
else:
self.write(json.dumps("Error - grid must be provided", default=serialize_everything))
self.write(safe_json_dumps("Error - grid must be provided"))
self.set_status(422)
except Exception as e:
logging.error(e)
self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
self.write(safe_json_dumps("Error - an internal server error occurred."))
self.set_status(500)
self.set_header("Cache-Control", "no-store")
server/handlers/api/options.py
+2 -3
View File
@@ -1,4 +1,3 @@
import json
from datetime import datetime
from typing import Any
@@ -10,7 +9,7 @@ from tornado.web import Application
from core.config import MAX_SPOT_AGE, ALLOW_SPOTTING
from core.constants import BANDS, ALL_MODES, MODE_TYPES, SIGS, CONTINENTS, PROPAGATION_MODES
from core.prometheus_metrics_handler import api_requests_counter
from core.utils import serialize_everything
from core.utils import safe_json_dumps
class APIOptionsHandler(tornado.web.RequestHandler):
@@ -50,7 +49,7 @@ class APIOptionsHandler(tornado.web.RequestHandler):
if ALLOW_SPOTTING:
options["spot_sources"].append("API")
self.write(json.dumps(options, default=serialize_everything))
self.write(safe_json_dumps(options))
self.set_status(200)
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
server/handlers/api/spots.py
+5 -6
View File
@@ -1,5 +1,4 @@
import copy
import json
import logging
from datetime import datetime, timedelta
from queue import Queue
@@ -12,7 +11,7 @@ from tornado import httputil
from tornado.web import Application
from core.prometheus_metrics_handler import api_requests_counter
from core.utils import serialize_everything, empty_queue
from core.utils import safe_json_dumps, empty_queue
from data.lookup_credentials import extract_credentials
SSE_HANDLER_MAX_QUEUE_SIZE = 1000
@@ -57,15 +56,15 @@ class APISpotsHandler(tornado.web.RequestHandler):
data = get_spot_list_with_filters(self._spots, query_params)
if credentials:
data = self._enrich(data, credentials)
self.write(json.dumps(data, default=serialize_everything))
self.write(safe_json_dumps(data))
self.set_status(200)
except ValueError as e:
logging.error(e)
self.write(json.dumps("Bad request - " + str(e), default=serialize_everything))
self.write(safe_json_dumps("Bad request - " + str(e)))
self.set_status(400)
except Exception as e:
logging.error(e)
self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
self.write(safe_json_dumps("Error - an internal server error occurred."))
self.set_status(500)
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
@@ -152,7 +151,7 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
if self._credentials:
spot = copy.deepcopy(spot)
spot.infer_missing(self._credentials)
self.write_message(msg=json.dumps(spot, default=serialize_everything))
self.write_message(msg=safe_json_dumps(spot))
else:
# Send a keepalive comment if the queue was empty
server/handlers/api/status.py
+2 -3
View File
@@ -1,4 +1,3 @@
import json
from datetime import datetime
from typing import Any
@@ -8,7 +7,7 @@ from tornado import httputil
from tornado.web import Application
from core.prometheus_metrics_handler import api_requests_counter
from core.utils import serialize_everything
from core.utils import safe_json_dumps
class APIStatusHandler(tornado.web.RequestHandler):
@@ -30,7 +29,7 @@ class APIStatusHandler(tornado.web.RequestHandler):
self._web_server_metrics["status"] = "OK"
api_requests_counter.inc()
self.write(json.dumps(self._status_data, default=serialize_everything))
self.write(safe_json_dumps(self._status_data))
self.set_status(200)
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
templates/add_spot.html
+1 -1
View File
@@ -76,7 +76,7 @@
</div>
<script src="/js/add-spot.js?v=1782461355"></script>
<script src="/js/add-spot.js?v=1782666584"></script>
<script>$(document).ready(function () {
$("#nav-link-add-spot").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/alerts.html
+1 -1
View File
@@ -75,7 +75,7 @@
</div>
<script src="/js/alerts.js?v=1782461355"></script>
<script src="/js/alerts.js?v=1782666584"></script>
<script>$(document).ready(function () {
$("#nav-link-alerts").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/bands.html
+2 -2
View File
@@ -75,8 +75,8 @@
<script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script>
<script src="/js/spotsbandsandmap.js?v=1782461355"></script>
<script src="/js/bands.js?v=1782461355"></script>
<script src="/js/spotsbandsandmap.js?v=1782666584"></script>
<script src="/js/bands.js?v=1782666584"></script>
<script>$(document).ready(function () {
$("#nav-link-bands").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/base.html
+5 -5
View File
@@ -1,6 +1,6 @@
{% extends "skeleton.html" %}
{% block head_extra %}
<link rel="stylesheet" href="/css/style.css?v=1782461355" type="text/css">
<link rel="stylesheet" href="/css/style.css?v=1782666584" type="text/css">
<link href="/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
<link href="/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
<link href="/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
<script src="/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
<script src="/vendor/js/tinycolor2-1.6.0.min.js"></script>
<script src="/js/utils.js?v=1782461355"></script>
<script src="/js/ui-ham.js?v=1782461355"></script>
<script src="/js/geo.js?v=1782461355"></script>
<script src="/js/common.js?v=1782461355"></script>
<script src="/js/utils.js?v=1782666584"></script>
<script src="/js/ui-ham.js?v=1782666584"></script>
<script src="/js/geo.js?v=1782666584"></script>
<script src="/js/common.js?v=1782666584"></script>
{% end %}
{% block body %}
<div class="container">
templates/conditions.html
+1 -1
View File
@@ -284,7 +284,7 @@
</div>
<script src="/vendor/js/chart-4.4.9.umd.min.js"></script>
<script src="/js/conditions.js?v=1782461355"></script>
<script src="/js/conditions.js?v=1782666584"></script>
<script>$(document).ready(function () {
$("#nav-link-conditions").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/map.html
+2 -2
View File
@@ -95,8 +95,8 @@
<script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script>
<script src="/js/spotsbandsandmap.js?v=1782461355"></script>
<script src="/js/map.js?v=1782461355"></script>
<script src="/js/spotsbandsandmap.js?v=1782666584"></script>
<script src="/js/map.js?v=1782666584"></script>
<script>$(document).ready(function () {
$("#nav-link-map").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/spots.html
+2 -2
View File
@@ -116,8 +116,8 @@
<script>
let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
</script>
<script src="/js/spotsbandsandmap.js?v=1782461355"></script>
<script src="/js/spots.js?v=1782461355"></script>
<script src="/js/spotsbandsandmap.js?v=1782666584"></script>
<script src="/js/spots.js?v=1782666584"></script>
<script>$(document).ready(function () {
$("#nav-link-spots").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/status.html
+1 -1
View File
@@ -59,7 +59,7 @@
</div>
</div>
<script src="/js/status.js?v=1782461355"></script>
<script src="/js/status.js?v=1782666584"></script>
<script>
$(document).ready(function () {
$("#nav-link-status").addClass("active");