Improve expired spot handling and efficiency of handling expired spots during web requests.

core/cleanup.py

@@ -29,12 +29,15 @@ class CleanupTimer:
     # Perform cleanup and reschedule next timer
     def cleanup(self):
         try:
-            # Perform cleanup
+            # Perform cleanup via letting the data expire
             self.spots.expire()
             self.alerts.expire()
 
-            # Alerts can persist in the system for a while, so we want to explicitly clean up any alerts that have
+            # Explicitly clean up any spots and alerts that have expired
-            # expired
+            for id in list(self.spots.iterkeys()):
+                spot = self.spots[id]
+                if spot.expired():
+                    self.spots.delete(id)
             for id in list(self.alerts.iterkeys()):
                 alert = self.alerts[id]
                 if alert.expired():

data/alert.py

@@ -137,7 +137,7 @@ class Alert:
         return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True)
 
     # Decide if this alert has expired (in which case it should not be added to the system in the first place, and not
-    # returned by the web server if later requested, and removed by the cleanup functions. "Expired" is defined as
+    # returned by the web server if later requested, and removed by the cleanup functions). "Expired" is defined as
     # either having an end_time in the past, or if it only has a start_time, then that start time was more than 3 hours
     # ago. If it somehow doesn't have a start_time either, it is considered to be expired.
     def expired(self):

data/spot.py

@@ -4,11 +4,12 @@ import json
 import logging
 import re
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import pytz
 from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 
+from core.config import MAX_SPOT_AGE
 from core.lookup_helper import lookup_helper
 from core.sig_utils import get_icon_for_sig, get_sig_ref_info, ANY_SIG_REGEX, get_ref_regex_for_sig
 from data.sig_ref import SIGRef
@@ -403,3 +404,10 @@ class Spot:
             if sig_ref.id == new_sig_ref.id and sig_ref.sig == new_sig_ref.sig:
                 return
         self.sig_refs.append(new_sig_ref)
+
+    # Decide if this spot has expired (in which case it should not be added to the system in the first place, and not
+    # returned by the web server if later requested, and removed by the cleanup functions). "Expired" is defined as
+    # either having a time further ago than the server's MAX_SPOT_AGE. If it somehow doesn't have a time either, it is
+    # considered to be expired.
+    def expired(self):
+        return not self.time or self.time < (datetime.now(pytz.UTC) - timedelta(seconds=MAX_SPOT_AGE)).timestamp()

server/webserver.py

@@ -422,7 +422,6 @@ class WebServer:
             if a is not None:
                 alerts.append(a)
         # We never want alerts that seem to be in the past
-        alerts = list(filter(lambda alert: not alert.expired(), alerts))
         alerts = sorted(alerts, key=lambda alert: (alert.start_time if alert and alert.start_time else 0))
         for k in query.keys():
             match k:

spotproviders/gma.py

@@ -42,6 +42,7 @@ class GMA(HTTPSpotProvider):
 
             # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
             if "REF" in source_spot:
+                try:
                     ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
                                                                   headers=HTTP_HEADERS)
                     # Sometimes this is blank, so handle that
@@ -81,4 +82,6 @@ class GMA(HTTPSpotProvider):
                             # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
                             # that for us.
                             new_spots.append(spot)
+                except:
+                    logging.warn("Exception when looking up " + self.REF_INFO_URL_ROOT + source_spot["REF"] + ", ignoring this spot for now")
         return new_spots

spotproviders/spot_provider.py

@@ -34,7 +34,8 @@ class SpotProvider:
             if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time:
                 # Fill in any blanks
                 spot.infer_missing()
-                # Add to the list
+                # Add to the list, provided it heas not already expired.
+                if not spot.expired():
                     self.spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
         self.last_spot_time = datetime.fromtimestamp(max(map(lambda s: s.time, spots)), pytz.UTC)
 
@@ -44,7 +45,8 @@ class SpotProvider:
     def submit(self, spot):
         # Fill in any blanks
         spot.infer_missing()
-        # Add to the list
+        # Add to the list, provided it heas not already expired.
+        if not spot.expired():
             self.spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
             self.last_spot_time = datetime.fromtimestamp(spot.time, pytz.UTC)