WWTOTA cluster support #97

alertproviders/bota.py

@@ -20,29 +20,31 @@ class BOTA(HTTPAlertProvider):
         new_alerts = []
         # Find the table of upcoming alerts
         bs = BeautifulSoup(http_response.content.decode(), features="lxml")
-        forthcoming_activations_div = bs.body.find('div', attrs={'class': 'view-activations-public'})
-        if forthcoming_activations_div:
-            tbody = forthcoming_activations_div.find('table', attrs={'class': 'views-table'}).find('tbody')
-            for row in tbody.find_all('tr'):
-                cells = row.find_all('td')
-                first_cell_text = str(cells[0].find('a').contents[0]).strip()
-                ref_name = first_cell_text.split(" by ")[0]
-                dx_call = str(cells[1].find('a').contents[0]).strip().upper()
+        div = bs.body.find('div', attrs={'class': 'view-activations-public'})
+        if div:
+            table = div.find('table', attrs={'class': 'views-table'})
+            if table:
+                tbody = table.find('tbody')
+                for row in tbody.find_all('tr'):
+                    cells = row.find_all('td')
+                    first_cell_text = str(cells[0].find('a').contents[0]).strip()
+                    ref_name = first_cell_text.split(" by ")[0]
+                    dx_call = str(cells[1].find('a').contents[0]).strip().upper()
 
-                # Get the date, dealing with the fact we get no year so have to figure out if it's last year or next year
-                date_text = str(cells[2].find('span').contents[0]).strip()
-                date_time = datetime.strptime(date_text,"%d %b - %H:%M UTC").replace(tzinfo=pytz.UTC)
-                date_time = date_time.replace(year=datetime.now(pytz.UTC).year)
-                # If this was more than a day ago, activation is actually next year
-                if date_time < datetime.now(pytz.UTC) - timedelta(days=1):
-                    date_time = date_time.replace(year=datetime.now(pytz.UTC).year + 1)
+                    # Get the date, dealing with the fact we get no year so have to figure out if it's last year or next year
+                    date_text = str(cells[2].find('span').contents[0]).strip()
+                    date_time = datetime.strptime(date_text,"%d %b - %H:%M UTC").replace(tzinfo=pytz.UTC)
+                    date_time = date_time.replace(year=datetime.now(pytz.UTC).year)
+                    # If this was more than a day ago, activation is actually next year
+                    if date_time < datetime.now(pytz.UTC) - timedelta(days=1):
+                        date_time = date_time.replace(year=datetime.now(pytz.UTC).year + 1)
 
-                # Convert to our alert format
-                alert = Alert(source=self.name,
-                              dx_calls=[dx_call],
-                              sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
-                              start_time=date_time.timestamp(),
-                              is_dxpedition=False)
+                    # Convert to our alert format
+                    alert = Alert(source=self.name,
+                                  dx_calls=[dx_call],
+                                  sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
+                                  start_time=date_time.timestamp(),
+                                  is_dxpedition=False)
 
-                new_alerts.append(alert)
+                    new_alerts.append(alert)
         return new_alerts

core/constants.py

@@ -4,7 +4,7 @@ from data.sig import SIG
 
 # General software
 SOFTWARE_NAME = "Spothole by M0TRT"
-SOFTWARE_VERSION = "1.2-pre"
+SOFTWARE_VERSION = "1.3-pre"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": SOFTWARE_NAME + ", v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}

core/lookup_helper.py

@@ -140,12 +140,14 @@ class LookupHelper:
     # database live if possible.
     def download_clublog_ctyxml(self):
         try:
-            logging.info("Downloading Clublog cty.xml...")
+            logging.info("Downloading Clublog cty.xml.gz...")
             response = self.CLUBLOG_CTY_XML_CACHE.get("https://cdn.clublog.org/cty.php?api=" + self.CLUBLOG_API_KEY,
                                                       headers=HTTP_HEADERS)
+            logging.info("Caching Clublog cty.xml.gz...")
             open(self.CLUBLOG_XML_DOWNLOAD_LOCATION + ".gz", 'wb').write(response.content)
             with gzip.open(self.CLUBLOG_XML_DOWNLOAD_LOCATION + ".gz", "rb") as uncompressed:
                 file_content = uncompressed.read()
+            logging.info("Caching Clublog cty.xml...")
             with open(self.CLUBLOG_XML_DOWNLOAD_LOCATION, "wb") as f:
                 f.write(file_content)
                 f.flush()

core/sig_utils.py

@@ -135,6 +135,10 @@ def populate_sig_ref_info(sig_ref):
                         sig_ref.latitude = ll[0]
                         sig_ref.longitude = ll[1]
                         break
+        elif sig.upper() == "WWTOTA":
+            if not sig_ref.name:
+                sig_ref.name = sig_ref.id
+            sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + sig_ref.name
         elif sig.upper() == "WAB" or sig.upper() == "WAI":
             ll = wab_wai_square_to_lat_lon(ref_id)
             if ll:

spotproviders/wwtota.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+import json
 import pytz
 
 from data.sig_ref import SIGRef
@@ -17,23 +18,24 @@ class WWTOTA(HTTPSpotProvider):
 
     def http_response_to_spots(self, http_response):
         new_spots = []
+        response_fixed = http_response.text.replace("\\/", "/")
+        response_json = json.loads(response_fixed)
+
         # Iterate through source data
-        for source_spot in http_response.json()["spots"]:
-            print(source_spot) # todo
+        for source_spot in response_json["spots"]:
             # Convert to our spot format
-            # spot = Spot(source=self.name,
-            #             source_id=source_spot["spotId"],
-            #             dx_call=source_spot["activator"].upper(),
-            #             de_call=source_spot["spotter"].upper(),
-            #             freq=float(source_spot["frequency"]) * 1000,
-            #             mode=source_spot["mode"].upper(),
-            #             comment=source_spot["comments"],
-            #             sig="WWTOTA",
-            #             sig_refs=[SIGRef(id=source_spot["reference"], sig="POTA", name=source_spot["name"])],
-            #             time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(
-            #                 tzinfo=pytz.UTC).timestamp())
-            #
-            # # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
-            # # that for us.
-            # new_spots.append(spot)
+            likely_freq = float(source_spot["freq"]) * 1000
+            if likely_freq < 1000000:
+                likely_freq = likely_freq * 1000
+            spot = Spot(source=self.name,
+                        dx_call=source_spot["call"].upper(),
+                        freq=likely_freq,
+                        comment=source_spot["comment"],
+                        sig="WWTOTA",
+                        sig_refs=[SIGRef(id=source_spot["ref"], sig="WWTOTA")],
+                        time=datetime.strptime(response_json["updated"][:10] + source_spot["time"], "%Y-%m-%d%H:%M").timestamp())
+
+            # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
+            # that for us.
+            new_spots.append(spot)
         return new_spots