Refactor of caching & data storage part 16 #118

providers/callsigndata/callsign_data_provider.py

@@ -19,6 +19,7 @@ class CallsignDataProvider:
         self.priority = int(provider_config["priority"])
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
+        self.lookup_count = 0
         self._storage = storage
 
     def start(self):

providers/callsigndata/clublogapi.py

@@ -37,6 +37,7 @@ class ClublogAPI(APIQueryCallsignDataProvider):
                 callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
+                self.lookup_count += 1
             else:
                 return None
 

providers/callsigndata/clublogxml.py

@@ -1,10 +1,13 @@
 import gzip
 import logging
+from datetime import datetime
 
+import pytz
 from pyhamtools import LookupLib, Callinfo
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -49,7 +52,18 @@ class ClublogXML(FileDownloadCallsignDataProvider):
             return False
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        if self._callinfo:
-            return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
-        else:
-            return None
+        callsign_data = Callsign(call=callsign)
+
+        try:
+            if self._callinfo:
+                callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+                self.status = "OK"
+                self.lookup_count += 1
+            else:
+                return None
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from Clublog XML data", e, exc_info=True)
+
+        return callsign_data

providers/callsigndata/countryfiles.py

@@ -4,6 +4,7 @@ from pyhamtools import LookupLib, Callinfo
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
+from data.callsign import Callsign
 from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
 
 
@@ -30,7 +31,18 @@ class CountryFiles(FileDownloadCallsignDataProvider):
             return False
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
-        if self._callinfo:
-            return get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
-        else:
-            return None
+        callsign_data = Callsign(call=callsign)
+
+        try:
+            if self._callinfo:
+                callsign_data = get_callsign_object_from_pyhamtools_callinfo(callsign, self._callinfo)
+                self.status = "OK"
+                self.lookup_count += 1
+            else:
+                return None
+
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception when looking up data from Country file", e, exc_info=True)
+
+        return callsign_data

providers/callsigndata/hamqth.py

@@ -1,7 +1,8 @@
 import logging
 import urllib.parse
-from datetime import timedelta
+from datetime import timedelta, datetime
 
+import pytz
 import xmltodict
 from pyhamtools import callinfo
 from requests import ConnectTimeout, ReadTimeout
@@ -32,14 +33,14 @@ class HamQTH(APIQueryCallsignDataProvider):
         # If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because
         #         # someone might provide credentials next time around.
         if not lookup_credentials or not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
-                or lookup_credentials.hamqth_session_key):
+                or lookup_credentials.hamqth_session_id):
             return None
 
         try:
             # Obtain session key from credentials, by looking it up from username & password if necessary.
-            session_key = None
-            if lookup_credentials.hamqth_session_key:
-                session_key = lookup_credentials.hamqth_session_key
+            session_id = None
+            if lookup_credentials.hamqth_session_id:
+                session_id = lookup_credentials.hamqth_session_id
             elif lookup_credentials.hamqth_username and lookup_credentials.hamqth_password:
                 try:
                     session_data = self._CREDENTIALS_CACHE.get(
@@ -48,7 +49,7 @@ class HamQTH(APIQueryCallsignDataProvider):
                         headers=HTTP_HEADERS).content
                     dict_data = xmltodict.parse(session_data)
                     if "session_id" in dict_data["HamQTH"]["session"]:
-                        session_key = str(dict_data["HamQTH"]["session"]["session_id"])
+                        session_id = str(dict_data["HamQTH"]["session"]["session_id"])
                     else:
                         # Log this failure at debug level only, not our problem if user entered the wrong password.
                         logging.debug("HamQTH login details incorrect, failed to look up with HamQTH.")
@@ -57,7 +58,7 @@ class HamQTH(APIQueryCallsignDataProvider):
                     logging.error("Exception when getting HamQTH session key")
                     return None
 
-            if not session_key:
+            if not session_id:
                 return None
 
             # Try the call as given, then fall back to the base call (strips /P, /M etc.)
@@ -73,13 +74,14 @@ class HamQTH(APIQueryCallsignDataProvider):
             for lookup_call in calls_to_try:
                 try:
                     response = self._URL_DATA_CACHE.get(
-                        self._HAMQTH_BASE_URL + "?id=" + session_key + "&callsign=" + urllib.parse.quote_plus(
+                        self._HAMQTH_BASE_URL + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                             lookup_call) + "&prg=" + self._PRG, headers=HTTP_HEADERS, timeout=10)
                     if response.ok:
                         self.status = "OK"
                         self.last_update_time = datetime.now(pytz.UTC)
-                        data = xmltodict.parse(response.content)["HamQTH"]["search"]
+                        self.lookup_count += 1
                         # Found data, convert it to our object and return it
+                        data = xmltodict.parse(response.content)["HamQTH"]["search"]
                         return self.hamqth_response_to_callsign(callsign, data)
 
                     elif not response.from_cache:

providers/callsigndata/qrz.py

@@ -1,7 +1,8 @@
 import logging
 import urllib.parse
-from datetime import timedelta
+from datetime import timedelta, datetime
 
+import pytz
 import xmltodict
 from pyhamtools import callinfo
 from requests import ConnectTimeout, ReadTimeout
@@ -80,6 +81,7 @@ class QRZ(APIQueryCallsignDataProvider):
                             if "Callsign" in qrz_response:
                                 self.status = "OK"
                                 self.last_update_time = datetime.now(pytz.UTC)
+                                self.lookup_count += 1
                                 # Found data, convert it to our object and return it
                                 return self.qrz_response_to_callsign(callsign, qrz_response.get("Callsign"))