Refactor of caching & data storage part 16 #118

core/data_providers.py

@@ -29,15 +29,7 @@ class DataProviders:
 
 
     def start(self):
-        for p in self.spot_providers:
-            if p.enabled:
-                p.start()
-        for p in self.alert_providers:
-            if p.enabled:
-                p.start()
-        for p in self.solar_condition_providers:
-            if p.enabled:
-                p.start()
+        # Start data providers before spot/alert providers so the lookup data is there already for incoming spots
         for p in self.static_data_providers:
             if p.enabled:
                 p.start()
@@ -47,6 +39,15 @@ class DataProviders:
         for p in self.callsign_data_providers:
             if p.enabled:
                 p.start()
+        for p in self.spot_providers:
+            if p.enabled:
+                p.start()
+        for p in self.alert_providers:
+            if p.enabled:
+                p.start()
+        for p in self.solar_condition_providers:
+            if p.enabled:
+                p.start()
 
     def stop(self):
         for sp in self.spot_providers:

core/status_reporter.py

@@ -84,7 +84,8 @@ class StatusReporter:
         DATA_STORE.status_data["callsign_data_providers"] = list(
             map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
+                           "lookup_count": p.lookup_count},
                 DATA_PROVIDERS.callsign_data_providers))
         DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"],
                                                "last_api_access": WEB_SERVER.web_server_metrics[

core/utils.py

@@ -2,6 +2,7 @@ import logging
 
 import simplejson
 from pyhamtools.frequency import freq_to_band
+from pyhamtools.locator import latlong_to_locator
 
 from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES
 from core.data_store import DATA_STORE
@@ -95,6 +96,9 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
         itu_zone = data["ituz"] if "ituz" in data else None
         lat = float(data["latitude"]) if "latitude" in data else None
         lon = float(data["longitude"]) if "longitude" in data else None
+        grid = None
+        if lat and lon:
+            grid = latlong_to_locator(lat, lon)
 
         return Callsign(call=callsign,
                         home_call=home_call,
@@ -105,8 +109,9 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
                         itu_zone=itu_zone,
                         latitude=lat,
                         longitude=lon,
+                        grid=grid,
                         location_source="DXCC")
 
-    except ValueError:
+    except (KeyError, ValueError):
         # Unknown callsign, can't look anything up, return a Callsign object with basic data so that gets cached
         return Callsign(call=callsign)

providers/callsigndata/callsign_data_provider.py

@@ -19,6 +19,7 @@ class CallsignDataProvider:
         self.priority = int(provider_config["priority"])
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
+        self.lookup_count = 0
         self._storage = storage
 
     def start(self):

providers/callsigndata/clublogapi.py

@@ -37,6 +37,7 @@ class ClublogAPI(APIQueryCallsignDataProvider):
                 callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
+                self.lookup_count += 1
             else:
                 return None
 

providers/callsigndata/clublogxml.py

@@ -1,10 +1,13 @@
 import gzip
 import logging
+from datetime import datetime
 
+import pytz
 from pyhamtools import LookupLib, Callinfo
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -49,7 +52,18 @@ class ClublogXML(FileDownloadCallsignDataProvider):
             return False
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        if self._callinfo:
-            return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
-        else:
-            return None
+        callsign_data = Callsign(call=callsign)
+
+        try:
+            if self._callinfo:
+                callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+                self.status = "OK"
+                self.lookup_count += 1
+            else:
+                return None
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from Clublog XML data", e, exc_info=True)
+
+        return callsign_data

providers/callsigndata/countryfiles.py

@@ -4,6 +4,7 @@ from pyhamtools import LookupLib, Callinfo
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -30,7 +31,18 @@ class CountryFiles(FileDownloadCallsignDataProvider):
             return False
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        if self._callinfo:
-            return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
-        else:
-            return None
+        callsign_data = Callsign(call=callsign)
+
+        try:
+            if self._callinfo:
+                callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+                self.status = "OK"
+                self.lookup_count += 1
+            else:
+                return None
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from Country file", e, exc_info=True)
+
+        return callsign_data

providers/callsigndata/hamqth.py

@@ -1,7 +1,8 @@
 import logging
 import urllib.parse
-from datetime import timedelta
+from datetime import timedelta, datetime
 
+import pytz
 import xmltodict
 from pyhamtools import callinfo
 from requests import ConnectTimeout, ReadTimeout
@@ -32,14 +33,14 @@ class HamQTH(APIQueryCallsignDataProvider):
         # If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because
         #         # someone might provide credentials next time around.
         if not lookup_credentials or not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
-                or lookup_credentials.hamqth_session_key):
+                or lookup_credentials.hamqth_session_id):
             return None
 
         try:
             # Obtain session key from credentials, by looking it up from username & password if necessary.
-            session_key = None
-            if lookup_credentials.hamqth_session_key:
-                session_key = lookup_credentials.hamqth_session_key
+            session_id = None
+            if lookup_credentials.hamqth_session_id:
+                session_id = lookup_credentials.hamqth_session_id
             elif lookup_credentials.hamqth_username and lookup_credentials.hamqth_password:
                 try:
                     session_data = self._CREDENTIALS_CACHE.get(
@@ -48,7 +49,7 @@ class HamQTH(APIQueryCallsignDataProvider):
                         headers=HTTP_HEADERS).content
                     dict_data = xmltodict.parse(session_data)
                     if "session_id" in dict_data["HamQTH"]["session"]:
-                        session_key = str(dict_data["HamQTH"]["session"]["session_id"])
+                        session_id = str(dict_data["HamQTH"]["session"]["session_id"])
                     else:
                         # Log this failure at debug level only, not our problem if user entered the wrong password.
                         logging.debug("HamQTH login details incorrect, failed to look up with HamQTH.")
@@ -57,7 +58,7 @@ class HamQTH(APIQueryCallsignDataProvider):
                     logging.error("Exception when getting HamQTH session key")
                     return None
 
-            if not session_key:
+            if not session_id:
                 return None
 
             # Try the call as given, then fall back to the base call (strips /P, /M etc.)
@@ -73,13 +74,14 @@ class HamQTH(APIQueryCallsignDataProvider):
             for lookup_call in calls_to_try:
                 try:
                     response = self._URL_DATA_CACHE.get(
-                        self._HAMQTH_BASE_URL + "?id=" + session_key + "&callsign=" + urllib.parse.quote_plus(
+                        self._HAMQTH_BASE_URL + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                             lookup_call) + "&prg=" + self._PRG, headers=HTTP_HEADERS, timeout=10)
                     if response.ok:
                         self.status = "OK"
                         self.last_update_time = datetime.now(pytz.UTC)
-                        data = xmltodict.parse(response.content)["HamQTH"]["search"]
+                        self.lookup_count += 1
                         # Found data, convert it to our object and return it
+                        data = xmltodict.parse(response.content)["HamQTH"]["search"]
                         return self.hamqth_response_to_callsign(callsign, data)
 
                     elif not response.from_cache:

providers/callsigndata/qrz.py

@@ -1,7 +1,8 @@
 import logging
 import urllib.parse
-from datetime import timedelta
+from datetime import timedelta, datetime
 
+import pytz
 import xmltodict
 from pyhamtools import callinfo
 from requests import ConnectTimeout, ReadTimeout
@@ -80,6 +81,7 @@ class QRZ(APIQueryCallsignDataProvider):
                             if "Callsign" in qrz_response:
                                 self.status = "OK"
                                 self.last_update_time = datetime.now(pytz.UTC)
+                                self.lookup_count += 1
                                 # Found data, convert it to our object and return it
                                 return self.qrz_response_to_callsign(callsign, qrz_response.get("Callsign"))
 

server/handlers/api/lookups.py

@@ -8,15 +8,15 @@ import tornado
 from tornado import httputil
 from tornado.web import Application
 
+from core.call_lookup_helper import get_call_info
 from core.constants import SIGS
 from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.prometheus_metrics_handler import api_requests_counter
-from core.sig_utils import get_ref_regex_for_sig
 from core.sig_lookup_helper import populate_missing_sig_ref_info
+from core.sig_utils import get_ref_regex_for_sig
 from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 from data.sig_ref import SIGRef
-from data.spot import Spot
 
 
 class APILookupCallHandler(tornado.web.RequestHandler):
@@ -45,27 +45,9 @@ class APILookupCallHandler(tornado.web.RequestHandler):
             if "call" in query_params.keys():
                 call = str(query_params.get("call")).upper()
                 if re.match(r"^[A-Z0-9/\-]*$", call):
-                    # Take the callsign, make a "fake spot" so we can run infer_missing() on it, then repack the
-                    # resulting data in the correct way for the API response.
                     credentials = extract_credentials(query_params)
-                    fake_spot = Spot(dx_call=call)
-                    fake_spot.infer_missing(credentials)
-                    data = {
-                        "call": call,
-                        "name": fake_spot.dx_name,
-                        "qth": fake_spot.dx_qth,
-                        "country": fake_spot.dx_country,
-                        "flag": fake_spot.dx_flag,
-                        "continent": fake_spot.dx_continent,
-                        "dxcc_id": fake_spot.dx_dxcc_id,
-                        "cq_zone": fake_spot.dx_cq_zone,
-                        "itu_zone": fake_spot.dx_itu_zone,
-                        "grid": fake_spot.dx_grid,
-                        "latitude": fake_spot.dx_latitude,
-                        "longitude": fake_spot.dx_longitude,
-                        "location_source": fake_spot.dx_location_source
-                    }
-                    self.write(safe_json_dumps(data))
+                    callsign_data = get_call_info(call, credentials)
+                    self.write(safe_json_dumps(callsign_data))
 
                 else:
                     self.write(safe_json_dumps("Error - '" + call + "' does not look like a valid callsign."))

server/sse_broadcaster.py

@@ -11,6 +11,9 @@ class SSEBroadcaster:
     def __init__(self):
         self._handlers = set()
         self._lock = threading.Lock()
+        self._loop = None
+
+    def bind_to_web_server_loop(self):
         self._loop = IOLoop.current()
 
     def register(self, handler):
@@ -22,9 +25,9 @@ class SSEBroadcaster:
             self._handlers.discard(handler)
 
     def publish(self, value):
-        self._loop.add_callback(self._fan_out, value)
+         self._loop.add_callback(self._broadcast, value)
 
-    def _fan_out(self, value):
+    def _broadcast(self, value):
         with self._lock:
             handlers = list(self._handlers)
         for handler in handlers:

server/webserver.py

@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import os
+import threading
 
 import tornado
 from tornado.web import StaticFileHandler
@@ -61,6 +62,10 @@ class WebServer:
     async def _start_inner(self):
         """Start method (async). Sets up the Tornado application."""
 
+        # Bind the SSE broadcasters to the web server's loop, so they fire correctly
+        self._spot_broadcaster.bind_to_web_server_loop()
+        self._alert_broadcaster.bind_to_web_server_loop()
+
         # Prepare a list of common arguments that are passed in to every API & page handler. This is just a basic thing
         # to avoid copy-pasting the same thing to every route declaration below.
         handler_opts = {"web_server_metrics": self.web_server_metrics}

static/apidocs/openapi.yml

@@ -325,7 +325,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/CallLookup'
+                $ref: '#/components/schemas/CallsignData'
         '422':
           description: Validation error e.g. callsign missing or format incorrect
           content:
@@ -1791,6 +1791,10 @@ components:
             The last time at which this provider received data, UTC seconds since UNIX epoch. If this
             is zero, the provider has never updated.
           example: 1759579508
+        lookup_count:
+          type: number
+          description: The number of callsign lookups performed using this provider since the server was started.
+          example: 1234
 
     SpotList:
       type: array
@@ -1962,12 +1966,16 @@ components:
             on this server.
           example: true
 
-    CallLookup:
+    CallsignData:
       type: object
       properties:
         call:
           type: string
           description: Callsign, as provided to the API
+          example: DL/M0TRT/P
+        home_call:
+          type: string
+          description: The "home" call, without prefixes or suffixes
           example: M0TRT
         name:
           type: string

static/js/status.js

@@ -65,6 +65,7 @@ function loadStatus() {
                     <div class="col"><strong>${p["name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
+                    <div class="col">Lookups: ${(p["enabled"] && p["lookup_count"] > 0) ? p["lookup_count"] : "N/A"}</div>
                 </div>`);
         });
     });