Fix extra duplicates showing for spots and alerts

data/alert.py

@@ -113,18 +113,15 @@ class Alert:
             if self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0] and not self.sig:
                 self.sig = self.sig_refs[0].sig
 
-            # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
+            # Create an ID based on the source and source ID if possible, as these guaranee uniqueness. If there is no
-            # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
+            # source ID, use a combination of callsign and start time. Excluding things like the comment here allows for
-            # apart from that they were retrieved from the API at different times. Note that the simple Python hash()
+            # user updates of their alert comments without duplicating in the system.
-            # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
-            # use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
-            # The ID is computed before the online lookups below so that it is stable regardless of whether credentials
-            # are provided, allowing the enriched API response to be matched to the stored alert by ID.
             if not self.id:
-                self_copy = copy.deepcopy(self)
+                if self.source and self.source_id:
-                self_copy.received_time = 0
+                    self.id = hashlib.sha256(str({"s": self.source, "sid": self.source_id}).encode("utf-8")).hexdigest()
-                self_copy.received_time_iso = ""
+                else:
-                self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
+                    self.id = hashlib.sha256(
+                        str({"s": self.source, "c": self.dx_calls, "t": self.start_time}).encode("utf-8")).hexdigest()
 
             # DX operator name lookup, using QRZ.com/HamQTH.
             if self.dx_calls and not self.dx_names:

data/spot.py

@@ -279,7 +279,8 @@ class Spot:
                     # If so, add that to the sig_refs list for this spot.
                     ref_regex = get_ref_regex_for_sig(found_sig)
                     if ref_regex:
-                        ref_matches = re.finditer(r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)", self.comment,
+                        ref_matches = re.finditer(r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)",
+                                                  self.comment,
                                                   re.IGNORECASE)
                         for ref_match in ref_matches:
                             self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig))
@@ -361,18 +362,15 @@ class Spot:
             if self.comment and not self.qrt:
                 self.qrt = "QRT" in self.comment.upper()
 
-            # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
+            # Create an ID based on the source and source ID if possible, as these guaranee uniqueness. If there is no
-            # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
+            # source ID, use a combination of callsign and spot time. Spot time is down to the second or even the
-            # apart from that they were retrieved from the API at different times. Note that the simple Python hash()
+            # millisecond, so we can be reasonably sure two spots that match are the same spot.
-            # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
-            # use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
-            # The ID is computed before the online lookups below so that it is stable regardless of whether credentials
-            # are provided, allowing the enriched API response to be matched to the stored spot by ID.
             if not self.id:
-                self_copy = copy.deepcopy(self)
+                if self.source and self.source_id:
-                self_copy.received_time = 0
+                    self.id = hashlib.sha256(str({"s": self.source, "sid": self.source_id}).encode("utf-8")).hexdigest()
-                self_copy.received_time_iso = ""
+                else:
-                self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
+                    self.id = hashlib.sha256(
+                        str({"s": self.source, "c": self.dx_call, "t": self.time}).encode("utf-8")).hexdigest()
 
             # DX operator details lookup. This should be the last resort compared to taking the data from the actual
             # spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon or DXCC lat/lon

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1786556946"></script>
+<script src="/static/js/add-spot.js?v=1786599927"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -82,7 +82,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1786556946"></script>
+<script src="/static/js/alerts.js?v=1786599927"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -79,8 +79,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1786556946"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1786599927"></script>
-<script src="/static/js/bands.js?v=1786556946"></script>
+<script src="/static/js/bands.js?v=1786599927"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1786556946" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1786599927" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
 <script src="/static/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
 <script src="/static/vendor/js/tinycolor2-1.6.0.min.js"></script>
 
-<script src="/static/js/utils.js?v=1786556946"></script>
+<script src="/static/js/utils.js?v=1786599927"></script>
-<script src="/static/js/ui-ham.js?v=1786556946"></script>
+<script src="/static/js/ui-ham.js?v=1786599927"></script>
-<script src="/static/js/geo.js?v=1786556946"></script>
+<script src="/static/js/geo.js?v=1786599927"></script>
-<script src="/static/js/common.js?v=1786556946"></script>
+<script src="/static/js/common.js?v=1786599927"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1786556946"></script>
+<script src="/static/js/conditions.js?v=1786599927"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -112,8 +112,8 @@
 <script src="/static/vendor/js/leaflet-cqzones.js"></script>
 <script src="/static/vendor/js/leaflet-workedallbritainireland.js" type="module"></script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1786556946"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1786599927"></script>
-<script src="/static/js/map.js?v=1786556946"></script>
+<script src="/static/js/map.js?v=1786599927"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -118,8 +118,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1786556946"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1786599927"></script>
-<script src="/static/js/spots.js?v=1786556946"></script>
+<script src="/static/js/spots.js?v=1786599927"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -81,7 +81,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1786556946"></script>
+<script src="/static/js/status.js?v=1786599927"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");