w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2025-10-27 08:49:27 +00:00
Code Issues 11 Packages Projects Releases Wiki Activity

Further alert implementation #17

Browse Source
This commit is contained in:
Ian Renton
2025-10-06 21:06:54 +01:00
parent e0675a01f0
commit 6ee4fe288a
14 changed files with 171 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
alertproviders/alert_provider.py
View File

@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import datetime, timedelta
import pytz
@@ -35,7 +35,11 @@ class AlertProvider:
for alert in alerts:
# Fill in any blanks
alert.infer_missing()
# Add to the list
# Add to the list, provided it meets the semsible date test. If alerts have an end time, it must be in the
# future, or if not, then the start date must be at least in the last 24 hours.
if (alert.end_time and alert.end_time > datetime.now(pytz.UTC).timestamp()) or (
not alert.end_time and alert.start_time > (datetime.now(
pytz.UTC) - timedelta(days=1)).timestamp()):
self.alerts.add(alert.id, alert, expire=MAX_ALERT_AGE)
# Stop any threads and prepare for application shutdown

13
core/cleanup.py
View File

@@ -1,5 +1,5 @@
import logging
from datetime import datetime
from datetime import datetime, timedelta
from threading import Timer
from time import sleep
@@ -32,6 +32,17 @@ class CleanupTimer:
# Perform cleanup
self.spots.expire()
self.alerts.expire()
# Alerts can persist in the system for a while, so we want to explicitly clean up any alerts that have
# definitively ended, or if they have no definite end time, then if the start time was more than 24 hours
# ago.
for id in list(self.alerts.iterkeys()):
alert = self.alerts[id]
if (alert.end_time and alert.end_time < datetime.now(pytz.UTC).timestamp()) or (
not alert.end_time and alert.start_time < (datetime.now(
pytz.UTC) - timedelta(days=1)).timestamp()):
self.alerts.evict(id)
self.status = "OK"
self.last_cleanup_time = datetime.now(pytz.UTC)

15
data/alert.py
View File

@@ -1,7 +1,9 @@
import hashlib
import json
from dataclasses import dataclass
from datetime import datetime
import copy
import pytz
from core.constants import DXCC_FLAGS
@@ -14,7 +16,7 @@ from core.utils import infer_continent_from_callsign, \
@dataclass
class Alert:
# Unique identifier for the alert
id: int = None
id: str = None
# Callsign of the operator that has been alertted
dx_call: str = None
# Name of the operator that has been alertted
@@ -103,8 +105,15 @@ class Alert:
if self.dx_call and not self.dx_name:
self.dx_name = infer_name_from_callsign(self.dx_call)
# Always create an ID based on a hashcode
self.id = hash(str(self))
# Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
# to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
# apart from that they were retrieved from the API at different times. Note that the simple Python hash()
# function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
# use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
self_copy = copy.deepcopy(self)
self_copy.received_time = 0
self_copy.received_time_iso = ""
self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
# JSON serialise
def to_json(self):

15
data/spot.py
View File

@@ -1,3 +1,5 @@
import copy
import hashlib
import json
from dataclasses import dataclass
from datetime import datetime
@@ -16,7 +18,7 @@ from core.utils import infer_mode_type_from_mode, infer_band_from_freq, infer_co
@dataclass
class Spot:
# Unique identifier for the spot
id: int = None
id: str = None
# Callsign of the operator that has been spotted
dx_call: str = None
# Callsign of the operator that has spotted them
@@ -207,8 +209,15 @@ class Spot:
# is likely at home.
self.location_good = self.location_source == "SPOT" or (self.location_source == "QRZ" and not "/" in self.dx_call)
# Always create an ID based on a hashcode
self.id = hash(str(self))
# Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
# to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
# apart from that they were retrieved from the API at different times. Note that the simple Python hash()
# function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
# use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
self_copy = copy.deepcopy(self)
self_copy.received_time = 0
self_copy.received_time_iso = ""
self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
# JSON serialise
def to_json(self):

29
server/webserver.py
View File

@@ -70,7 +70,8 @@ class WebServer:
if not ALLOW_SPOTTING:
response.content_type = 'application/json'
response.status = 401
return json.dumps("Error - this server does not allow new spots to be added via the API.", default=serialize_everything)
return json.dumps("Error - this server does not allow new spots to be added via the API.",
default=serialize_everything)
# Reject if no spot
if not bottle.request.query.spot:
@@ -86,7 +87,8 @@ class WebServer:
if not spot.time or not spot.dx_call:
response.content_type = 'application/json'
response.status = 422
return json.dumps("Error - 'time' and 'dx_call' must be provided as a minimum.", default=serialize_everything)
return json.dumps("Error - 'time' and 'dx_call' must be provided as a minimum.",
default=serialize_everything)
# infer missing data, and add it to our database.
spot.source = "API"
@@ -169,7 +171,6 @@ class WebServer:
spots = spots[:int(query.get("limit"))]
return spots
# Utility method to apply filters to the overall alert list and return only a subset. Enables query parameters in
# the main "alerts" GET call.
def get_alert_list_with_filters(self):
@@ -185,26 +186,26 @@ class WebServer:
alert_ids = list(self.alerts.iterkeys())
alerts = []
for k in alert_ids:
# While we persist old spots in the system for a while to produce a useful list, any alert that has already
# passed its end time can be explicitly removed from the list to return.
# TODO deal with there being no end time
if self.alerts.get(k).end_time > datetime.now(pytz.UTC).timestamp():
alerts.append(self.alerts.get(k))
alerts = sorted(alerts, key=lambda alert: alert.start_time)
for k in query.keys():
match k:
case "received_since":
since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC)
alerts = [s for s in alerts if s.received_time > since]
alerts = [a for a in alerts if a.received_time > since]
case "max_duration":
max_duration = int(query.get(k))
alerts = [a for a in alerts if (a.end_time and a.end_time - a.start_time <= max_duration) or (
not a.end_time and datetime.now(pytz.UTC).timestamp() - a.start_time <= max_duration)]
case "source":
sources = query.get(k).split(",")
alerts = [s for s in alerts if s.source in sources]
alerts = [a for a in alerts if a.source in sources]
case "sig":
sigs = query.get(k).split(",")
alerts = [s for s in alerts if s.sig in sigs]
alerts = [a for a in alerts if a.sig in sigs]
case "dx_continent":
dxconts = query.get(k).split(",")
alerts = [s for s in alerts if s.dx_continent in dxconts]
alerts = [a for a in alerts if a.dx_continent in dxconts]
# If we have a "limit" parameter, we apply that last, regardless of where it appeared in the list of keys.
if "limit" in query.keys():
alerts = alerts[:int(query.get("limit"))]
@@ -219,7 +220,9 @@ class WebServer:
"mode_types": MODE_TYPES,
"sigs": SIGS,
# Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle things that aren't even available.
"spot_sources": list(map(lambda p: p["name"], filter(lambda p: p["enabled"], self.status_data["spot_providers"]))),
"alert_sources": list(map(lambda p: p["name"], filter(lambda p: p["enabled"], self.status_data["alert_providers"]))),
"spot_sources": list(
map(lambda p: p["name"], filter(lambda p: p["enabled"], self.status_data["spot_providers"]))),
"alert_sources": list(
map(lambda p: p["name"], filter(lambda p: p["enabled"], self.status_data["alert_providers"]))),
"continents": CONTINENTS,
"max_spot_age": MAX_SPOT_AGE}

2
views/webpage_about.tpl
View File

@@ -10,3 +10,5 @@
<p>The software was written by <a href="https://ianrenton.com">Ian Renton, MØTRT</a> and other contributors. Full details are available in the README.</p>
<p><a href="/">&laquo; Back home</a></p>
</div>
<script>$(document).ready(function() { $("#nav-link-about").addClass("active"); }); <!-- highlight active page in nav --></script>

3
views/webpage_alerts.tpl
View File

@@ -25,7 +25,7 @@
</div>
<div class="card-body">
<div id="settings-container" class="row row-cols-1 g-4 mb-4"></div>
<div id="settings-container" class="row row-cols-1 row-cols-md-4 g-4"></div>
</div>
</div>
@@ -42,3 +42,4 @@
</div>
<script src="/js/alerts.js"></script>
<script>$(document).ready(function() { $("#nav-link-alerts").addClass("active"); }); <!-- highlight active page in nav --></script>

1
views/webpage_apidocs.tpl
View File

@@ -2,3 +2,4 @@
<redoc spec-url="/apidocs/openapi.yml"></redoc>
<script src="https://cdn.redoc.ly/redoc/latest/bundles/redoc.standalone.js"> </script>
<script>$(document).ready(function() { $("#nav-link-api").addClass("active"); }); <!-- highlight active page in nav --></script>

8
views/webpage_base.tpl
View File

@@ -57,10 +57,10 @@
</button>
<div class="collapse navbar-collapse" id="navbarTogglerDemo02">
<ul class="navbar-nav me-auto mb-2 mb-lg-0">
<li class="nav-item ms-4"><a href="/" class="nav-link">Spots</a></li>
<li class="nav-item ms-4"><a href="/alerts" class="nav-link">Alerts</a></li>
<li class="nav-item ms-4"><a href="/about" class="nav-link">About</a></li>
<li class="nav-item ms-4"><a href="/apidocs" class="nav-link">API</a></li>
<li class="nav-item ms-4"><a href="/" class="nav-link" id="nav-link-spots">Spots</a></li>
<li class="nav-item ms-4"><a href="/alerts" class="nav-link" id="nav-link-alerts">Alerts</a></li>
<li class="nav-item ms-4"><a href="/about" class="nav-link" id="nav-link-about">About</a></li>
<li class="nav-item ms-4"><a href="/apidocs" class="nav-link" id="nav-link-api">API</a></li>
<li class="nav-item ms-4"><a href="https://git.ianrenton.com/ian/spothole" class="nav-link">Source Code</a></li>
</ul>
</div>

1
views/webpage_spots.tpl
View File

@@ -69,3 +69,4 @@
</div>
<script src="/js/spots.js"></script>
<script>$(document).ready(function() { $("#nav-link-spots").addClass("active"); }); <!-- highlight active page in nav --></script>

14
webassets/apidocs/openapi.yml
View File

@@ -199,6 +199,12 @@ paths:
required: false
schema:
type: integer
- name: max_duration
in: query
description: Limit the spots to only ones with a duration of this many seconds or less. Duration is end time minus start time, if end time is set, otherwise "now" minus start time. This is useful to filter out people who alert POTA activations lasting months or even years.
required: false
schema:
type: integer
- name: source
in: query
description: "Limit the spots to only ones from one or more sources. To select more than one source, supply a comma-separated list."
@@ -424,9 +430,9 @@ components:
type: object
properties:
id:
type: integer
type: string
description: Unique identifier based on a hash of the spot to distinguish this one from any others.
example: 123987609816349182
example: 442c5d56ac467341f1943e8596685073b38f5a5d4c3802ca1e16ecf98967956c
dx_call:
type: string
description: Callsign of the operator that has been spotted
@@ -675,9 +681,9 @@ components:
type: object
properties:
id:
type: integer
type: string
description: Unique identifier based on a hash of the alert to distinguish this one from any others.
example: 123987609816349182
example: 442c5d56ac467341f1943e8596685073b38f5a5d4c3802ca1e16ecf98967956c
dx_call:
type: string
description: Callsign of the operator that is going to be activating

4
webassets/css/style.css
View File

@@ -1,3 +1,7 @@
.navbar-nav .nav-link.active {
font-weight: bold;
}
#info-container{
width: 100%;
}

105
webassets/js/alerts.js
View File

@@ -1,3 +1,6 @@
// How often to query the server?
const REFRESH_INTERVAL_SEC = 60 * 30;
// Storage for the alert data that the server gives us.
var alerts = []
// Storage for the options that the server gives us. This will define our filters.
@@ -8,8 +11,9 @@ var lastUpdateTime;
// Load alerts and populate the table.
function loadAlerts() {
$.getJSON('/api/alerts' + buildQueryString(), function(jsonData) {
// Present loaded time
$("#timing-container").text("Data loaded at " + moment.utc().format('HH:mm') + " UTC.");
// Store last updated time
lastUpdateTime = moment.utc();
updateRefreshDisplay();
// Store data
alerts = jsonData;
// Update table
@@ -26,6 +30,7 @@ function buildQueryString() {
}
});
str = str + "limit=" + $("#alerts-to-fetch option:selected").val();
str = str + "&max_duration=604800";
return str;
}
@@ -64,71 +69,94 @@ function updateTable() {
table.find('thead tr').append(`<th class='hideonmobile'>Source</th>`);
table.find('thead tr').append(`<th class='hideonmobile'>Ref.</th>`);
if (alerts.length == 0) {
// Split alerts into three types, each of which will get its own table header: On now, next 24h, and later. "On now"
// is considered to be events with an end_time where start<now<end, or events with no end time that started in the
// last hour.
onNow = alerts.filter(a => (a["end_time"] != null && moment.unix(a["end_time"]).utc().isSameOrAfter() && moment.unix(a["start_time"]).utc().isBefore())
|| (a["end_time"] == null && moment.unix(a["start_time"]).utc().add(1, 'hours').isSameOrAfter() && moment.unix(a["start_time"]).utc().isBefore()));
next24h = alerts.filter(a => moment.unix(a["start_time"]).utc().isSameOrAfter() && moment.unix(a["start_time"]).utc().subtract(24, 'hours').isBefore());
later = alerts.filter(a => moment.unix(a["start_time"]).utc().subtract(24, 'hours').isSameOrAfter());
if (onNow.length > 0) {
table.find('tbody').append('<tr class="table-primary"><td colspan="100" style="text-align:center;">On Now</td></tr>');
addAlertRowsToTable(table.find('tbody'), onNow);
}
if (next24h.length > 0) {
table.find('tbody').append('<tr class="table-primary"><td colspan="100" style="text-align:center;">Starting within 24 hours</td></tr>');
addAlertRowsToTable(table.find('tbody'), next24h);
}
if (later.length > 0) {
table.find('tbody').append('<tr class="table-primary"><td colspan="100" style="text-align:center;">Starting later </td></tr>');
addAlertRowsToTable(table.find('tbody'), later);
}
if (onNow.length == 0 && next24h.length == 0 && later.length == 0) {
table.find('tbody').append('<tr class="table-danger"><td colspan="100" style="text-align:center;">No alerts match your filters.</td></tr>');
}
alerts.forEach(s => {
// Update DOM
$('#table-container').html(table);
}
// Add a row to tbody for each alert in the provided list
function addAlertRowsToTable(tbody, alerts) {
alerts.forEach(a => {
// Create row
let $tr = $('<tr>');
// Format UTC times for display
var start_time = moment.unix(s["start_time"]).utc();
var start_time = moment.unix(a["start_time"]).utc();
var start_time_formatted = start_time.format("YYYY-MM-DD HH:mm");
var end_time = moment.unix(s["start_time"]).utc();
var end_time = moment.unix(a["end_time"]).utc();
var end_time_formatted = (end_time != null) ? end_time.format("YYYY-MM-DD HH:mm") : "Not specified";
// Format dx country
var dx_country = s["dx_country"]
var dx_country = a["dx_country"]
if (dx_country == null) {
dx_country = "Unknown or not a country"
}
// Format freqs & modes
var freqsModesText = "";
if (s["freqs_modes"] != null) {
freqsModesText = escapeHtml(s["freqs_modes"]);
if (a["freqs_modes"] != null) {
freqsModesText = escapeHtml(a["freqs_modes"]);
}
// Format comment
var commentText = "";
if (s["comment"] != null) {
commentText = escapeHtml(s["comment"]);
if (a["comment"] != null) {
commentText = escapeHtml(a["comment"]);
}
// Sig or fallback to source
var sigSourceText = s["source"];
if (s["sig"]) {
sigSourceText = s["sig"];
var sigSourceText = a["source"];
if (a["sig"]) {
sigSourceText = a["sig"];
}
// Format sig_refs
var sig_refs = ""
if (s["sig_refs"]) {
sig_refs = s["sig_refs"].join(", ")
if (a["sig_refs"]) {
sig_refs = a["sig_refs"].join(", ")
}
// Populate the row
$tr.append(`<td class='nowrap'>${start_time_formatted}</td>`);
$tr.append(`<td class='nowrap'>${end_time_formatted}</td>`);
$tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${s["dx_flag"]}</span><a class='dx-link' href='https://qrz.com/db/${s["dx_call"]}' target='_new'>${s["dx_call"]}</a></td>`);
$tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${a["dx_flag"]}</span><a class='dx-link' href='https://qrz.com/db/${a["dx_call"]}' target='_new'>${a["dx_call"]}</a></td>`);
$tr.append(`<td class='hideonmobile'>${freqsModesText}</td>`);
$tr.append(`<td class='hideonmobile'>${commentText}</td>`);
$tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${sigSourceText}</td>`);
$tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid fa-${a["icon"]}'></i></span> ${sigSourceText}</td>`);
$tr.append(`<td class='hideonmobile'>${sig_refs}</td>`);
table.find('tbody').append($tr);
tbody.append($tr);
// Second row for mobile view only, containing source, ref, freqs/modes & comment
$tr2 = $("<tr class='hidenotonmobile'>");
if (s["qrt"] == true) {
$tr2.addClass("table-faded");
}
$tr2.append(`<td colspan="100"><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${sig_refs} ${freqsModesText}<br/>${commentText}</td>`);
table.find('tbody').append($tr2);
$tr2.append(`<td colspan="100"><span class='icon-wrapper'><i class='fa-solid fa-${a["icon"]}'></i></span> ${sig_refs} ${freqsModesText}<br/>${commentText}</td>`);
tbody.append($tr2);
});
// Update DOM
$('#table-container').html(table);
}
// Load server options. Once a successful callback is made from this, we then query alerts.
@@ -144,8 +172,9 @@ function loadOptions() {
// Load settings from settings storage
loadSettings();
// Load alerts
// Load alerts and set up the timer
loadAlerts();
setInterval(loadAlerts, REFRESH_INTERVAL_SEC * 1000);
});
}
@@ -183,6 +212,24 @@ function filtersUpdated() {
saveSettings();
}
// Update the refresh timing display
function updateRefreshDisplay() {
if (lastUpdateTime != null) {
let count = REFRESH_INTERVAL_SEC;
let secSinceUpdate = moment.duration(moment().diff(lastUpdateTime)).asSeconds();
updatingString = "Updating..."
if (secSinceUpdate < REFRESH_INTERVAL_SEC) {
count = REFRESH_INTERVAL_SEC - secSinceUpdate;
if (count <= 60) {
updatingString = "Updating in " + count.toFixed(0) + " seconds...";
} else {
updatingString = "Updating in " + Math.floor(count / 60.0).toFixed(0) + " minutes.";
}
}
$("#timing-container").text("Last updated at " + lastUpdateTime.format('HH:mm') + " UTC. " + updatingString);
}
}
// Utility function to escape HTML characters from a string.
function escapeHtml(str) {
if (typeof str !== 'string') {
@@ -244,6 +291,8 @@ function setUpEventListeners() {
$(document).ready(function() {
// Call loadOptions(), this will then trigger loading alerts and setting up timers.
loadOptions();
// Update the refresh timing display every second
setInterval(updateRefreshDisplay, 1000);
// Set up event listeners
setUpEventListeners();
});

4
webassets/js/spots.js
View File

@@ -314,7 +314,11 @@ function updateRefreshDisplay() {
updatingString = "Updating..."
if (secSinceUpdate < REFRESH_INTERVAL_SEC) {
count = REFRESH_INTERVAL_SEC - secSinceUpdate;
if (count <= 60) {
updatingString = "Updating in " + count.toFixed(0) + " seconds...";
} else {
updatingString = "Updating in " + Math.floor(count / 60.0).toFixed(0) + " minutes.";
}
}
$("#timing-container").text("Last updated at " + lastUpdateTime.format('HH:mm') + " UTC. " + updatingString);
}