Further alert implementation #17

alertproviders/alert_provider.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import pytz
 
@@ -35,7 +35,11 @@ class AlertProvider:
         for alert in alerts:
             # Fill in any blanks
             alert.infer_missing()
-            # Add to the list
+            # Add to the list, provided it meets the semsible date test. If alerts have an end time, it must be in the
+            # future, or if not, then the start date must be at least in the last 24 hours.
+            if (alert.end_time and alert.end_time > datetime.now(pytz.UTC).timestamp()) or (
+                    not alert.end_time and alert.start_time > (datetime.now(
+                    pytz.UTC) - timedelta(days=1)).timestamp()):
                 self.alerts.add(alert.id, alert, expire=MAX_ALERT_AGE)
 
     # Stop any threads and prepare for application shutdown

core/cleanup.py

@@ -1,5 +1,5 @@
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 from threading import Timer
 from time import sleep
 
@@ -32,6 +32,17 @@ class CleanupTimer:
             # Perform cleanup
             self.spots.expire()
             self.alerts.expire()
+
+            # Alerts can persist in the system for a while, so we want to explicitly clean up any alerts that have
+            # definitively ended, or if they have no definite end time, then if the start time was more than 24 hours
+            # ago.
+            for id in list(self.alerts.iterkeys()):
+                alert = self.alerts[id]
+                if (alert.end_time and alert.end_time < datetime.now(pytz.UTC).timestamp()) or (
+                        not alert.end_time and alert.start_time < (datetime.now(
+                    pytz.UTC) - timedelta(days=1)).timestamp()):
+                    self.alerts.evict(id)
+
             self.status = "OK"
             self.last_cleanup_time = datetime.now(pytz.UTC)
 

data/alert.py

@@ -1,7 +1,9 @@
+import hashlib
 import json
 from dataclasses import dataclass
 from datetime import datetime
 
+import copy
 import pytz
 
 from core.constants import DXCC_FLAGS
@@ -14,7 +16,7 @@ from core.utils import infer_continent_from_callsign, \
 @dataclass
 class Alert:
     # Unique identifier for the alert
-    id: int = None
+    id: str = None
     # Callsign of the operator that has been alertted
     dx_call: str = None
     # Name of the operator that has been alertted
@@ -103,8 +105,15 @@ class Alert:
         if self.dx_call and not self.dx_name:
             self.dx_name = infer_name_from_callsign(self.dx_call)
 
-        # Always create an ID based on a hashcode
+        # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
-        self.id = hash(str(self))
+        # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
+        # apart from that they were retrieved from the API at different times. Note that the simple Python hash()
+        # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
+        # use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
+        self_copy = copy.deepcopy(self)
+        self_copy.received_time = 0
+        self_copy.received_time_iso = ""
+        self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
 
     # JSON serialise
     def to_json(self):

data/spot.py

@@ -1,3 +1,5 @@
+import copy
+import hashlib
 import json
 from dataclasses import dataclass
 from datetime import datetime
@@ -16,7 +18,7 @@ from core.utils import infer_mode_type_from_mode, infer_band_from_freq, infer_co
 @dataclass
 class Spot:
     # Unique identifier for the spot
-    id: int = None
+    id: str = None
     # Callsign of the operator that has been spotted
     dx_call: str = None
     # Callsign of the operator that has spotted them
@@ -207,8 +209,15 @@ class Spot:
         # is likely at home.
         self.location_good = self.location_source == "SPOT" or (self.location_source == "QRZ" and not "/" in self.dx_call)
 
-        # Always create an ID based on a hashcode
+        # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
-        self.id = hash(str(self))
+        # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
+        # apart from that they were retrieved from the API at different times. Note that the simple Python hash()
+        # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
+        # use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
+        self_copy = copy.deepcopy(self)
+        self_copy.received_time = 0
+        self_copy.received_time_iso = ""
+        self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
 
     # JSON serialise
     def to_json(self):

server/webserver.py

@@ -70,7 +70,8 @@ class WebServer:
             if not ALLOW_SPOTTING:
                 response.content_type = 'application/json'
                 response.status = 401
-                return json.dumps("Error - this server does not allow new spots to be added via the API.", default=serialize_everything)
+                return json.dumps("Error - this server does not allow new spots to be added via the API.",
+                                  default=serialize_everything)
 
             # Reject if no spot
             if not bottle.request.query.spot:
@@ -86,7 +87,8 @@ class WebServer:
             if not spot.time or not spot.dx_call:
                 response.content_type = 'application/json'
                 response.status = 422
-                return json.dumps("Error - 'time' and 'dx_call' must be provided as a minimum.", default=serialize_everything)
+                return json.dumps("Error - 'time' and 'dx_call' must be provided as a minimum.",
+                                  default=serialize_everything)
 
             # infer missing data, and add it to our database.
             spot.source = "API"
@@ -169,7 +171,6 @@ class WebServer:
             spots = spots[:int(query.get("limit"))]
         return spots
 
-
     # Utility method to apply filters to the overall alert list and return only a subset. Enables query parameters in
     # the main "alerts" GET call.
     def get_alert_list_with_filters(self):
@@ -185,26 +186,26 @@ class WebServer:
         alert_ids = list(self.alerts.iterkeys())
         alerts = []
         for k in alert_ids:
-            # While we persist old spots in the system for a while to produce a useful list, any alert that has already
-            # passed its end time can be explicitly removed from the list to return.
-            # TODO deal with there being no end time
-            if self.alerts.get(k).end_time > datetime.now(pytz.UTC).timestamp():
             alerts.append(self.alerts.get(k))
         alerts = sorted(alerts, key=lambda alert: alert.start_time)
         for k in query.keys():
             match k:
                 case "received_since":
                     since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC)
-                    alerts = [s for s in alerts if s.received_time > since]
+                    alerts = [a for a in alerts if a.received_time > since]
+                case "max_duration":
+                    max_duration = int(query.get(k))
+                    alerts = [a for a in alerts if (a.end_time and a.end_time - a.start_time <= max_duration) or (
+                                not a.end_time and datetime.now(pytz.UTC).timestamp() - a.start_time <= max_duration)]
                 case "source":
                     sources = query.get(k).split(",")
-                    alerts = [s for s in alerts if s.source in sources]
+                    alerts = [a for a in alerts if a.source in sources]
                 case "sig":
                     sigs = query.get(k).split(",")
-                    alerts = [s for s in alerts if s.sig in sigs]
+                    alerts = [a for a in alerts if a.sig in sigs]
                 case "dx_continent":
                     dxconts = query.get(k).split(",")
-                    alerts = [s for s in alerts if s.dx_continent in dxconts]
+                    alerts = [a for a in alerts if a.dx_continent in dxconts]
         # If we have a "limit" parameter, we apply that last, regardless of where it appeared in the list of keys.
         if "limit" in query.keys():
             alerts = alerts[:int(query.get("limit"))]
@@ -219,7 +220,9 @@ class WebServer:
                 "mode_types": MODE_TYPES,
                 "sigs": SIGS,
                 # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle things that aren't even available.
-                "spot_sources": list(map(lambda p: p["name"], filter(lambda p: p["enabled"], self.status_data["spot_providers"]))),
+                "spot_sources": list(
-                "alert_sources": list(map(lambda p: p["name"], filter(lambda p: p["enabled"], self.status_data["alert_providers"]))),
+                    map(lambda p: p["name"], filter(lambda p: p["enabled"], self.status_data["spot_providers"]))),
+                "alert_sources": list(
+                    map(lambda p: p["name"], filter(lambda p: p["enabled"], self.status_data["alert_providers"]))),
                 "continents": CONTINENTS,
                 "max_spot_age": MAX_SPOT_AGE}

views/webpage_about.tpl

@@ -10,3 +10,5 @@
     <p>The software was written by <a href="https://ianrenton.com">Ian Renton, MØTRT</a> and other contributors. Full details are available in the README.</p>
     <p><a href="/">&laquo; Back home</a></p>
 </div>
+
+<script>$(document).ready(function() { $("#nav-link-about").addClass("active"); }); <!-- highlight active page in nav --></script>

views/webpage_alerts.tpl

@@ -25,7 +25,7 @@
 
         </div>
         <div class="card-body">
-            <div id="settings-container" class="row row-cols-1 g-4 mb-4"></div>
+            <div id="settings-container" class="row row-cols-1 row-cols-md-4 g-4"></div>
         </div>
     </div>
 
@@ -42,3 +42,4 @@
 </div>
 
 <script src="/js/alerts.js"></script>
+<script>$(document).ready(function() { $("#nav-link-alerts").addClass("active"); }); <!-- highlight active page in nav --></script>

views/webpage_apidocs.tpl

@@ -2,3 +2,4 @@
 
 <redoc spec-url="/apidocs/openapi.yml"></redoc>
 <script src="https://cdn.redoc.ly/redoc/latest/bundles/redoc.standalone.js"> </script>
+<script>$(document).ready(function() { $("#nav-link-api").addClass("active"); }); <!-- highlight active page in nav --></script>

views/webpage_base.tpl

@@ -57,10 +57,10 @@
             </button>
             <div class="collapse navbar-collapse" id="navbarTogglerDemo02">
                 <ul class="navbar-nav me-auto mb-2 mb-lg-0">
-                    <li class="nav-item ms-4"><a href="/" class="nav-link">Spots</a></li>
+                    <li class="nav-item ms-4"><a href="/" class="nav-link" id="nav-link-spots">Spots</a></li>
-                    <li class="nav-item ms-4"><a href="/alerts" class="nav-link">Alerts</a></li>
+                    <li class="nav-item ms-4"><a href="/alerts" class="nav-link" id="nav-link-alerts">Alerts</a></li>
-                    <li class="nav-item ms-4"><a href="/about" class="nav-link">About</a></li>
+                    <li class="nav-item ms-4"><a href="/about" class="nav-link" id="nav-link-about">About</a></li>
-                    <li class="nav-item ms-4"><a href="/apidocs" class="nav-link">API</a></li>
+                    <li class="nav-item ms-4"><a href="/apidocs" class="nav-link" id="nav-link-api">API</a></li>
                     <li class="nav-item ms-4"><a href="https://git.ianrenton.com/ian/spothole" class="nav-link">Source Code</a></li>
                 </ul>
             </div>

views/webpage_spots.tpl

@@ -69,3 +69,4 @@
 </div>
 
 <script src="/js/spots.js"></script>
+<script>$(document).ready(function() { $("#nav-link-spots").addClass("active"); }); <!-- highlight active page in nav --></script>

webassets/apidocs/openapi.yml

@@ -199,6 +199,12 @@ paths:
           required: false
           schema:
             type: integer
+        - name: max_duration
+          in: query
+          description: Limit the spots to only ones with a duration of this many seconds or less. Duration is end time minus start time, if end time is set, otherwise "now" minus start time. This is useful to filter out people who alert POTA activations lasting months or even years.
+          required: false
+          schema:
+            type: integer
         - name: source
           in: query
           description: "Limit the spots to only ones from one or more sources. To select more than one source, supply a comma-separated list."
@@ -424,9 +430,9 @@ components:
       type: object
       properties:
         id:
-          type: integer
+          type: string
           description: Unique identifier based on a hash of the spot to distinguish this one from any others.
-          example: 123987609816349182
+          example: 442c5d56ac467341f1943e8596685073b38f5a5d4c3802ca1e16ecf98967956c
         dx_call:
           type: string
           description: Callsign of the operator that has been spotted
@@ -675,9 +681,9 @@ components:
       type: object
       properties:
         id:
-          type: integer
+          type: string
           description: Unique identifier based on a hash of the alert to distinguish this one from any others.
-          example: 123987609816349182
+          example: 442c5d56ac467341f1943e8596685073b38f5a5d4c3802ca1e16ecf98967956c
         dx_call:
           type: string
           description: Callsign of the operator that is going to be activating

webassets/css/style.css

@@ -1,3 +1,7 @@
+.navbar-nav .nav-link.active {
+  font-weight: bold;
+}
+
 #info-container{
     width: 100%;
 }

webassets/js/alerts.js

@@ -1,3 +1,6 @@
+// How often to query the server?
+const REFRESH_INTERVAL_SEC = 60 * 30;
+
 // Storage for the alert data that the server gives us.
 var alerts = []
 // Storage for the options that the server gives us. This will define our filters.
@@ -8,8 +11,9 @@ var lastUpdateTime;
 // Load alerts and populate the table.
 function loadAlerts() {
     $.getJSON('/api/alerts' + buildQueryString(), function(jsonData) {
-        // Present loaded time
+        // Store last updated time
-        $("#timing-container").text("Data loaded at " + moment.utc().format('HH:mm') + " UTC.");
+        lastUpdateTime = moment.utc();
+        updateRefreshDisplay();
         // Store data
         alerts = jsonData;
         // Update table
@@ -26,6 +30,7 @@ function buildQueryString() {
         }
     });
     str = str + "limit=" + $("#alerts-to-fetch option:selected").val();
+    str = str + "&max_duration=604800";
     return str;
 }
 
@@ -64,71 +69,94 @@ function updateTable() {
     table.find('thead tr').append(`<th class='hideonmobile'>Source</th>`);
     table.find('thead tr').append(`<th class='hideonmobile'>Ref.</th>`);
 
-    if (alerts.length == 0) {
+    // Split alerts into three types, each of which will get its own table header: On now, next 24h, and later. "On now"
+    // is considered to be events with an end_time where start<now<end, or events with no end time that started in the
+    // last hour.
+    onNow = alerts.filter(a => (a["end_time"] != null && moment.unix(a["end_time"]).utc().isSameOrAfter() && moment.unix(a["start_time"]).utc().isBefore())
+        || (a["end_time"] == null && moment.unix(a["start_time"]).utc().add(1, 'hours').isSameOrAfter() && moment.unix(a["start_time"]).utc().isBefore()));
+    next24h = alerts.filter(a => moment.unix(a["start_time"]).utc().isSameOrAfter() && moment.unix(a["start_time"]).utc().subtract(24, 'hours').isBefore());
+    later = alerts.filter(a => moment.unix(a["start_time"]).utc().subtract(24, 'hours').isSameOrAfter());
+
+    if (onNow.length > 0) {
+        table.find('tbody').append('<tr class="table-primary"><td colspan="100" style="text-align:center;">On Now</td></tr>');
+        addAlertRowsToTable(table.find('tbody'), onNow);
+    }
+
+    if (next24h.length > 0) {
+        table.find('tbody').append('<tr class="table-primary"><td colspan="100" style="text-align:center;">Starting within 24 hours</td></tr>');
+        addAlertRowsToTable(table.find('tbody'), next24h);
+    }
+
+    if (later.length > 0) {
+        table.find('tbody').append('<tr class="table-primary"><td colspan="100" style="text-align:center;">Starting later </td></tr>');
+        addAlertRowsToTable(table.find('tbody'), later);
+    }
+
+    if (onNow.length == 0 && next24h.length == 0 && later.length == 0) {
         table.find('tbody').append('<tr class="table-danger"><td colspan="100" style="text-align:center;">No alerts match your filters.</td></tr>');
     }
 
-    alerts.forEach(s => {
+    // Update DOM
+    $('#table-container').html(table);
+}
+
+// Add a row to tbody for each alert in the provided list
+function addAlertRowsToTable(tbody, alerts) {
+    alerts.forEach(a => {
         // Create row
         let $tr = $('<tr>');
 
         // Format UTC times for display
-        var start_time = moment.unix(s["start_time"]).utc();
+        var start_time = moment.unix(a["start_time"]).utc();
         var start_time_formatted = start_time.format("YYYY-MM-DD HH:mm");
-        var end_time = moment.unix(s["start_time"]).utc();
+        var end_time = moment.unix(a["end_time"]).utc();
         var end_time_formatted = (end_time != null) ? end_time.format("YYYY-MM-DD HH:mm") : "Not specified";
 
         // Format dx country
-        var dx_country = s["dx_country"]
+        var dx_country = a["dx_country"]
         if (dx_country == null) {
             dx_country = "Unknown or not a country"
         }
 
         // Format freqs & modes
         var freqsModesText = "";
-        if (s["freqs_modes"] != null) {
+        if (a["freqs_modes"] != null) {
-            freqsModesText = escapeHtml(s["freqs_modes"]);
+            freqsModesText = escapeHtml(a["freqs_modes"]);
         }
 
         // Format comment
         var commentText = "";
-        if (s["comment"] != null) {
+        if (a["comment"] != null) {
-            commentText = escapeHtml(s["comment"]);
+            commentText = escapeHtml(a["comment"]);
         }
 
         // Sig or fallback to source
-        var sigSourceText = s["source"];
+        var sigSourceText = a["source"];
-        if (s["sig"]) {
+        if (a["sig"]) {
-            sigSourceText = s["sig"];
+            sigSourceText = a["sig"];
         }
 
         // Format sig_refs
         var sig_refs = ""
-        if (s["sig_refs"]) {
+        if (a["sig_refs"]) {
-            sig_refs = s["sig_refs"].join(", ")
+            sig_refs = a["sig_refs"].join(", ")
         }
 
         // Populate the row
         $tr.append(`<td class='nowrap'>${start_time_formatted}</td>`);
         $tr.append(`<td class='nowrap'>${end_time_formatted}</td>`);
-        $tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${s["dx_flag"]}</span><a class='dx-link' href='https://qrz.com/db/${s["dx_call"]}' target='_new'>${s["dx_call"]}</a></td>`);
+        $tr.append(`<td class='nowrap'><span class='flag-wrapper hideonmobile' title='${dx_country}'>${a["dx_flag"]}</span><a class='dx-link' href='https://qrz.com/db/${a["dx_call"]}' target='_new'>${a["dx_call"]}</a></td>`);
         $tr.append(`<td class='hideonmobile'>${freqsModesText}</td>`);
         $tr.append(`<td class='hideonmobile'>${commentText}</td>`);
-        $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${sigSourceText}</td>`);
+        $tr.append(`<td class='nowrap hideonmobile'><span class='icon-wrapper'><i class='fa-solid fa-${a["icon"]}'></i></span> ${sigSourceText}</td>`);
         $tr.append(`<td class='hideonmobile'>${sig_refs}</td>`);
-        table.find('tbody').append($tr);
+        tbody.append($tr);
 
         // Second row for mobile view only, containing source, ref, freqs/modes & comment
         $tr2 = $("<tr class='hidenotonmobile'>");
-        if (s["qrt"] == true) {
+        $tr2.append(`<td colspan="100"><span class='icon-wrapper'><i class='fa-solid fa-${a["icon"]}'></i></span> ${sig_refs} ${freqsModesText}<br/>${commentText}</td>`);
-          $tr2.addClass("table-faded");
+        tbody.append($tr2);
-        }
-        $tr2.append(`<td colspan="100"><span class='icon-wrapper'><i class='fa-solid fa-${s["icon"]}'></i></span> ${sig_refs} ${freqsModesText}<br/>${commentText}</td>`);
-        table.find('tbody').append($tr2);
     });
-
-    // Update DOM
-    $('#table-container').html(table);
 }
 
 // Load server options. Once a successful callback is made from this, we then query alerts.
@@ -144,8 +172,9 @@ function loadOptions() {
         // Load settings from settings storage
         loadSettings();
 
-        // Load alerts
+        // Load alerts and set up the timer
         loadAlerts();
+        setInterval(loadAlerts, REFRESH_INTERVAL_SEC * 1000);
     });
 }
 
@@ -183,6 +212,24 @@ function filtersUpdated() {
     saveSettings();
 }
 
+// Update the refresh timing display
+function updateRefreshDisplay() {
+    if (lastUpdateTime != null) {
+        let count = REFRESH_INTERVAL_SEC;
+        let secSinceUpdate = moment.duration(moment().diff(lastUpdateTime)).asSeconds();
+        updatingString = "Updating..."
+        if (secSinceUpdate < REFRESH_INTERVAL_SEC) {
+            count = REFRESH_INTERVAL_SEC - secSinceUpdate;
+            if (count <= 60) {
+                updatingString = "Updating in " + count.toFixed(0) + " seconds...";
+            } else {
+                updatingString = "Updating in " + Math.floor(count / 60.0).toFixed(0) + " minutes.";
+            }
+        }
+        $("#timing-container").text("Last updated at " + lastUpdateTime.format('HH:mm') + " UTC. " + updatingString);
+    }
+}
+
 // Utility function to escape HTML characters from a string.
 function escapeHtml(str) {
     if (typeof str !== 'string') {
@@ -244,6 +291,8 @@ function setUpEventListeners() {
 $(document).ready(function() {
     // Call loadOptions(), this will then trigger loading alerts and setting up timers.
     loadOptions();
+    // Update the refresh timing display every second
+    setInterval(updateRefreshDisplay, 1000);
     // Set up event listeners
     setUpEventListeners();
 });

webassets/js/spots.js

@@ -314,7 +314,11 @@ function updateRefreshDisplay() {
         updatingString = "Updating..."
         if (secSinceUpdate < REFRESH_INTERVAL_SEC) {
             count = REFRESH_INTERVAL_SEC - secSinceUpdate;
+            if (count <= 60) {
                 updatingString = "Updating in " + count.toFixed(0) + " seconds...";
+            } else {
+                updatingString = "Updating in " + Math.floor(count / 60.0).toFixed(0) + " minutes.";
+            }
         }
         $("#timing-container").text("Last updated at " + lastUpdateTime.format('HH:mm') + " UTC. " + updatingString);
     }