From 5f16abf7093c09bb08d30e653e2ccd9f6d4c913c Mon Sep 17 00:00:00 2001 From: Ian Renton Date: Sat, 4 Oct 2025 13:28:22 +0100 Subject: [PATCH] Allow adding spots. Convert timestamps in the API to UNIX seconds. #2 --- core/utils.py | 6 ++---- data/spot.py | 31 +++++++++++++++++++++++-------- providers/aprsis.py | 2 +- providers/dxcluster.py | 2 +- providers/gma.py | 2 +- providers/hema.py | 2 +- providers/parksnpeaks.py | 2 +- providers/pota.py | 2 +- providers/provider.py | 6 +++--- providers/rbn.py | 2 +- providers/sota.py | 2 +- providers/wwbota.py | 2 +- providers/wwff.py | 2 +- server/webserver.py | 9 +++++---- webassets/apidocs/openapi.yml | 14 +++++++------- webassets/js/code.js | 2 +- 16 files changed, 51 insertions(+), 37 deletions(-) diff --git a/core/utils.py b/core/utils.py index a047e96..d23ceb3 100644 --- a/core/utils.py +++ b/core/utils.py @@ -150,9 +150,7 @@ def infer_mode_from_frequency(freq): # Convert objects to serialisable things. Used by JSON serialiser as a default when it encounters unserializable things. -# Converts datetimes to ISO. -# Anything else it tries to convert to a dict. +# Just converts objects to dict. Try to avoid doing anything clever here when serialising spots, because we also need +# to receive spots without complex handling. def serialize_everything(obj): - if isinstance(obj, datetime): - return obj.isoformat() return obj.__dict__ \ No newline at end of file diff --git a/data/spot.py b/data/spot.py index 9b05453..9a6a3f0 100644 --- a/data/spot.py +++ b/data/spot.py @@ -57,12 +57,16 @@ class Spot: freq: float = None # Band, defined by the frequency, e.g. "40m" or "70cm" band: str = None - # Time of the spot - time: datetime = None - # Time that this software received the spot. This is used with the "since_received" call to our API to receive all - # data that is new to us, even if by a quirk of the API it might be older than the list time the client polled the - # API. - received_time: datetime = datetime.now(pytz.UTC) + # Time of the spot, UTC seconds since UNIX epoch + time: float = None + # Time of the spot, ISO 8601 + time_iso: str = None + # Time that this software received the spot, UTC seconds since UNIX epoch. This is used with the "since_received" + # call to our API to receive all data that is new to us, even if by a quirk of the API it might be older than the + # list time the client polled the API. + received_time: float = None + # Time that this software received the spot, ISO 8601 + received_time_iso: str = None # Comment left by the spotter, if any comment: str = None # Special Interest Group (SIG), e.g. outdoor activity programme such as POTA @@ -96,9 +100,20 @@ class Spot: # Always create a GUID self.guid = str(uuid.uuid4()) - # If we somehow don't have a time, set it to some far past value so it sorts at the bottom of the list + # If we somehow don't have a spot time, set it to zero so it sorts off the bottom of any list but + # clients can still reliably parse it as a number. if not self.time: - self.time = datetime.min + self.time = 0 + + # If we don't have a received time, this has just been received so set that to "now" + if not self.received_time: + self.received_time = datetime.now(pytz.UTC).timestamp() + + # Fill in ISO versions of times, in case the client prefers that + if self.time and not self.time_iso: + self.time_iso = datetime.fromtimestamp(self.time, pytz.UTC).isoformat() + if self.received_time and not self.received_time_iso: + self.received_time_iso = datetime.fromtimestamp(self.received_time, pytz.UTC).isoformat() # Clean up DX call if it has an SSID or -# from RBN if self.dx_call and "-" in self.dx_call: diff --git a/providers/aprsis.py b/providers/aprsis.py index 4915e70..43628ec 100644 --- a/providers/aprsis.py +++ b/providers/aprsis.py @@ -48,7 +48,7 @@ class APRSIS(Provider): latitude=data["latitude"] if "latitude" in data else None, longitude=data["longitude"] if "longitude" in data else None, icon="tower-cell", - time=datetime.now(pytz.UTC)) # APRS-IS spots are live so we can assume spot time is "now" + time=datetime.now(pytz.UTC).timestamp()) # APRS-IS spots are live so we can assume spot time is "now" # Add to our list self.submit(spot) diff --git a/providers/dxcluster.py b/providers/dxcluster.py index a2a4b42..ad71932 100644 --- a/providers/dxcluster.py +++ b/providers/dxcluster.py @@ -70,7 +70,7 @@ class DXCluster(Provider): freq=float(match.group(2)) * 1000, comment=match.group(4).strip(), icon="desktop", - time=spot_datetime) + time=spot_datetime.timestamp()) # Add to our list self.submit(spot) diff --git a/providers/gma.py b/providers/gma.py index 5c40340..13d9fff 100644 --- a/providers/gma.py +++ b/providers/gma.py @@ -36,7 +36,7 @@ class GMA(HTTPProvider): sig_refs=[source_spot["REF"]], sig_refs_names=[source_spot["NAME"]], time=datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace( - tzinfo=pytz.UTC), + tzinfo=pytz.UTC).timestamp(), latitude=float(source_spot["LAT"]) if (source_spot["LAT"] != "") else None, # Seen GMA spots with no lat/lon longitude=float(source_spot["LON"]) if (source_spot["LON"] != "") else None) diff --git a/providers/hema.py b/providers/hema.py index 2563ce1..414a401 100644 --- a/providers/hema.py +++ b/providers/hema.py @@ -55,7 +55,7 @@ class HEMA(HTTPProvider): sig_refs=[spot_items[3].upper()], sig_refs_names=[spot_items[4]], icon="mound", - time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(tzinfo=pytz.UTC), + time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(tzinfo=pytz.UTC).timestamp(), latitude=float(spot_items[7]), longitude=float(spot_items[8])) diff --git a/providers/parksnpeaks.py b/providers/parksnpeaks.py index 669ab5f..64f7438 100644 --- a/providers/parksnpeaks.py +++ b/providers/parksnpeaks.py @@ -30,7 +30,7 @@ class ParksNPeaks(HTTPProvider): sig=source_spot["actClass"], sig_refs=[source_spot["actSiteID"]], icon="question", # todo determine from actClass - time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC)) + time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp()) # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. if spot.sig not in ["POTA", "SOTA", "WWFF"]: diff --git a/providers/pota.py b/providers/pota.py index 0c2cdd7..cd51748 100644 --- a/providers/pota.py +++ b/providers/pota.py @@ -30,7 +30,7 @@ class POTA(HTTPProvider): sig_refs=[source_spot["reference"]], sig_refs_names=[source_spot["name"]], icon="tree", - time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=pytz.UTC), + time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(), grid=source_spot["grid6"], latitude=source_spot["latitude"], longitude=source_spot["longitude"]) diff --git a/providers/provider.py b/providers/provider.py index a71c630..3ecb8f6 100644 --- a/providers/provider.py +++ b/providers/provider.py @@ -35,12 +35,12 @@ class Provider: # subclasses on receiving spots. def submit_batch(self, spots): for spot in spots: - if spot.time > self.last_spot_time: + if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time: # Fill in any blanks spot.infer_missing() # Add to the list self.spots.add(spot.guid, spot, expire=MAX_SPOT_AGE) - self.last_spot_time = max(map(lambda s: s.time, spots)) + self.last_spot_time = datetime.fromtimestamp(max(map(lambda s: s.time, spots)), pytz.UTC) # Submit a single spot retrieved from the provider. This will be added to the list regardless of its age. Spots # passing the check will also have their infer_missing() method called to complete their data set. This is called by @@ -50,7 +50,7 @@ class Provider: spot.infer_missing() # Add to the list self.spots.add(spot.guid, spot, expire=MAX_SPOT_AGE) - self.last_spot_time = spot.time + self.last_spot_time = datetime.fromtimestamp(spot.time, pytz.UTC) # Stop any threads and prepare for application shutdown def stop(self): diff --git a/providers/rbn.py b/providers/rbn.py index 1222dc9..25f1406 100644 --- a/providers/rbn.py +++ b/providers/rbn.py @@ -71,7 +71,7 @@ class RBN(Provider): freq=float(match.group(2)) * 1000, comment=match.group(4).strip(), icon="tower-cell", - time=spot_datetime) + time=spot_datetime.timestamp()) # Add to our list self.submit(spot) diff --git a/providers/sota.py b/providers/sota.py index 298ddc2..f301095 100644 --- a/providers/sota.py +++ b/providers/sota.py @@ -49,7 +49,7 @@ class SOTA(HTTPProvider): sig_refs=[source_spot["summitCode"]], sig_refs_names=[source_spot["summitName"]], icon="mountain-sun", - time=datetime.fromisoformat(source_spot["timeStamp"]), + time=datetime.fromisoformat(source_spot["timeStamp"]).timestamp(), activation_score=source_spot["points"]) # SOTA doesn't give summit lat/lon/grid in the main call, so we need another separate call for this diff --git a/providers/wwbota.py b/providers/wwbota.py index f4a7693..d52c151 100644 --- a/providers/wwbota.py +++ b/providers/wwbota.py @@ -33,7 +33,7 @@ class WWBOTA(HTTPProvider): sig_refs=refs, sig_refs_names=ref_names, icon="radiation", - time=datetime.fromisoformat(source_spot["time"]), + time=datetime.fromisoformat(source_spot["time"]).timestamp(), # WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For # now, we will just pick the first one to use as our grid, latitude and longitude. grid=source_spot["references"][0]["locator"], diff --git a/providers/wwff.py b/providers/wwff.py index d005fb9..acaa01f 100644 --- a/providers/wwff.py +++ b/providers/wwff.py @@ -30,7 +30,7 @@ class WWFF(HTTPProvider): sig_refs=[source_spot["reference"]], sig_refs_names=[source_spot["reference_name"]], icon="seedling", - time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC), + time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(), latitude=source_spot["latitude"], longitude=source_spot["longitude"]) diff --git a/server/webserver.py b/server/webserver.py index 46d315c..02bddaf 100644 --- a/server/webserver.py +++ b/server/webserver.py @@ -87,10 +87,11 @@ class WebServer: response.content_type = 'application/json' response.set_header('Cache-Control', 'no-store') return json.dumps("OK", default=serialize_everything) - except Exception: - response.content_type = 'application/json' - response.status = 422 - return json.dumps("An error occurred parsing your spot. Check it is compliant with the API.", default=serialize_everything) + except Exception as e: + logging.error(e) + response.content_type = 'application/json' + response.status = 422 + return json.dumps("An error occurred parsing your spot: " + str(e), default=serialize_everything) # Serve a templated page def serve_template(self, template_name): diff --git a/webassets/apidocs/openapi.yml b/webassets/apidocs/openapi.yml index fce5c9c..cb43efe 100644 --- a/webassets/apidocs/openapi.yml +++ b/webassets/apidocs/openapi.yml @@ -312,7 +312,7 @@ paths: operationId: spot parameters: - name: spot - description: The spot data to post + description: The spot data to post. The structure must contain at least "time" and "dx_call" to be accepted. required: true schema: type: @@ -487,13 +487,13 @@ components: - Unknown example: 40m time: - type: string - description: Time of the spot, ISO 8601 format - example: 2025-09-28T19:12:41Z + type: number + description: Time of the spot, UTC seconds since UNIX epoch + example: 1759579508 received_time: - type: string - description: Time that this software received the spot, ISO 8601 format. This is used with the "since_received" call to our API to receive all data that is new to us, even if by a quirk of the API it might be older than the list time the client polled the API. - example: 2025-09-28T19:12:41Z + type: number + description: Time that this software received the spot, UTC seconds since UNIX epoch. This is used with the "since_received" call to our API to receive all data that is new to us, even if by a quirk of the API it might be older than the list time the client polled the API. + example: 1759579508 comment: type: string description: Comment left by the spotter, if any diff --git a/webassets/js/code.js b/webassets/js/code.js index bbdc914..45edfef 100644 --- a/webassets/js/code.js +++ b/webassets/js/code.js @@ -83,7 +83,7 @@ function updateTable() { } // Format a UTC time for display - var time = moment.utc(s["time"], moment.ISO_8601); + var time = moment.unix(s["time"]).utc(); var time_formatted = time.format("HH:mm") // Format dx country