diff --git a/alertproviders/http_alert_provider.py b/alertproviders/http_alert_provider.py index a4dd99a..a59a003 100644 --- a/alertproviders/http_alert_provider.py +++ b/alertproviders/http_alert_provider.py @@ -41,15 +41,20 @@ class HTTPAlertProvider(AlertProvider): # Request data from API logging.debug("Polling " + self.name + " alert API...") http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30)) - # Pass off to the subclass for processing - new_alerts = self._http_response_to_alerts(http_response) - # Submit the new alerts for processing. There might not be any alerts for the less popular programs. - if new_alerts: - self._submit_batch(new_alerts) + # Check response code was good + if http_response.status_code == 200: + # Pass off to the subclass for processing + new_alerts = self._http_response_to_alerts(http_response) + # Submit the new alerts for processing. There might not be any alerts for the less popular programs. + if new_alerts: + self._submit_batch(new_alerts) - self.status = "OK" - self.last_update_time = datetime.now(pytz.UTC) - logging.debug("Received data from " + self.name + " alert API.") + self.status = "OK" + self.last_update_time = datetime.now(pytz.UTC) + logging.debug("Received data from " + self.name + " alert API.") + else: + self.status = "Error" + logging.warning(f"{self.name} alert API returned HTTP {http_response.status_code}.") except Exception: self.status = "Error" diff --git a/config-example.yml b/config-example.yml index 2c48dc7..2b545dc 100644 --- a/config-example.yml +++ b/config-example.yml @@ -215,6 +215,10 @@ allow-spotting: true # your log quickly on a popular server. log-web-requests: false +# Minimum severity of log statements to print to the system log. Defaults to INFO, change to DEBUG if you need more +# information. +log-level: INFO + # Options for the web UI. web-ui-options: spot-count: [ 10, 25, 50, 100 ] diff --git a/core/config.py b/core/config.py index ff0f235..4cb43c8 100644 --- a/core/config.py +++ b/core/config.py @@ -22,6 +22,7 @@ WEB_SERVER_PORT = config["web-server-port"] ALLOW_SPOTTING = config["allow-spotting"] WEB_UI_OPTIONS = config["web-ui-options"] API_ONLY_MODE = config.get("api-only-mode", False) +LOG_LEVEL = config.get("log-level", "INFO") LOG_WEB_REQUESTS = config.get("log-web-requests", False) # For ease of config, each spot provider owns its own config about whether it should be enabled by default in the web UI diff --git a/server/handlers/api/addspot.py b/server/handlers/api/addspot.py index 60c8762..20c64c1 100644 --- a/server/handlers/api/addspot.py +++ b/server/handlers/api/addspot.py @@ -127,7 +127,7 @@ class APISpotHandler(tornado.web.RequestHandler): self.set_header("Content-Type", "application/json") except Exception as e: - logging.error(e) + logging.error("Exception when handling client request to add spot API: %s", e, exc_info=True) self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) self.set_header("Cache-Control", "no-store") diff --git a/server/handlers/api/alerts.py b/server/handlers/api/alerts.py index d2f140a..b556fbc 100644 --- a/server/handlers/api/alerts.py +++ b/server/handlers/api/alerts.py @@ -59,11 +59,10 @@ class APIAlertsHandler(tornado.web.RequestHandler): self.write(safe_json_dumps(data)) self.set_status(200) except ValueError as e: - logging.error(e) self.write(safe_json_dumps("Bad request - " + str(e))) self.set_status(400) except Exception as e: - logging.error(e) + logging.error("Exception when handling client request to alerts API: %s", e, exc_info=True) self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) self.set_header("Cache-Control", "no-store") diff --git a/server/handlers/api/dxstats.py b/server/handlers/api/dxstats.py index e6f3b03..7a683a5 100644 --- a/server/handlers/api/dxstats.py +++ b/server/handlers/api/dxstats.py @@ -1,4 +1,5 @@ import json +import logging from collections import Counter from datetime import datetime, timedelta from typing import Any @@ -9,6 +10,7 @@ from tornado import httputil from tornado.web import Application from core.prometheus_metrics_handler import api_requests_counter +from core.utils import safe_json_dumps CONTINENTS = ["EU", "NA", "SA", "AS", "AF", "OC", "AN"] BANDS = ["160m", "80m", "60m", "40m", "30m", "20m", "17m", "15m", "12m", "10m", "6m"] @@ -29,29 +31,35 @@ class APIDxStatsHandler(tornado.web.RequestHandler): self._web_server_metrics = web_server_metrics def get(self): - self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) - self._web_server_metrics["api_access_counter"] += 1 - self._web_server_metrics["status"] = "OK" - api_requests_counter.inc() + try: + self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) + self._web_server_metrics["api_access_counter"] += 1 + self._web_server_metrics["status"] = "OK" + api_requests_counter.inc() - one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp() - counts = Counter() + one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp() + counts = Counter() - for key in self._spots.iterkeys(): - spot = self._spots.get(key) - if spot is None: - continue - if not spot.time or spot.time < one_hour_ago: - continue - if spot.de_continent in CONTINENTS_SET and spot.dx_continent in CONTINENTS_SET and spot.band in BANDS_SET: - counts[spot.de_continent, spot.dx_continent, spot.band] += 1 + for key in self._spots.iterkeys(): + spot = self._spots.get(key) + if spot is None: + continue + if not spot.time or spot.time < one_hour_ago: + continue + if spot.de_continent in CONTINENTS_SET and spot.dx_continent in CONTINENTS_SET and spot.band in BANDS_SET: + counts[spot.de_continent, spot.dx_continent, spot.band] += 1 - result = { - de: {dx: {band: counts[de, dx, band] for band in BANDS} for dx in CONTINENTS} - for de in CONTINENTS - } + result = { + de: {dx: {band: counts[de, dx, band] for band in BANDS} for dx in CONTINENTS} + for de in CONTINENTS + } - self.write(json.dumps(result)) - self.set_status(200) - self.set_header("Cache-Control", "no-store") - self.set_header("Content-Type", "application/json") + self.write(json.dumps(result)) + self.set_status(200) + self.set_header("Cache-Control", "no-store") + self.set_header("Content-Type", "application/json") + + except Exception as e: + logging.error("Exception when handling client request to dx stats API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) + self.set_status(500) diff --git a/server/handlers/api/lookups.py b/server/handlers/api/lookups.py index 4ab34cd..4475bdc 100644 --- a/server/handlers/api/lookups.py +++ b/server/handlers/api/lookups.py @@ -74,7 +74,7 @@ class APILookupCallHandler(tornado.web.RequestHandler): self.set_status(422) except Exception as e: - logging.error(e) + logging.error("Exception when handling client request to call lookup API: %s", e, exc_info=True) self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) @@ -126,7 +126,7 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler): self.set_status(422) except Exception as e: - logging.error(e) + logging.error("Exception when handling client request to sig ref lookup API: %s", e, exc_info=True) self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) @@ -188,7 +188,7 @@ class APILookupGridHandler(tornado.web.RequestHandler): self.set_status(422) except Exception as e: - logging.error(e) + logging.error("Exception when handling client request to grid ref lookup API: %s", e, exc_info=True) self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) diff --git a/server/handlers/api/options.py b/server/handlers/api/options.py index cc545f2..1ec6c00 100644 --- a/server/handlers/api/options.py +++ b/server/handlers/api/options.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime from typing import Any @@ -25,31 +26,37 @@ class APIOptionsHandler(tornado.web.RequestHandler): self._web_server_metrics = web_server_metrics def get(self): - # Metrics - self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) - self._web_server_metrics["api_access_counter"] += 1 - self._web_server_metrics["status"] = "OK" - api_requests_counter.inc() + try: + # Metrics + self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) + self._web_server_metrics["api_access_counter"] += 1 + self._web_server_metrics["status"] = "OK" + api_requests_counter.inc() - options = {"bands": BANDS, - "modes": ALL_MODES, - "mode_types": MODE_TYPES, - "sigs": SIGS, - # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle things that aren't even available. - "spot_sources": list( - map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["spot_providers"]))), - "alert_sources": list( - map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"]))), - "continents": CONTINENTS, - "propagation_modes": list(PROPAGATION_MODES.values()), - "max_spot_age": MAX_SPOT_AGE, - "spot_allowed": ALLOW_SPOTTING} - # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from - # one of our proviers. - if ALLOW_SPOTTING: - options["spot_sources"].append("API") + options = {"bands": BANDS, + "modes": ALL_MODES, + "mode_types": MODE_TYPES, + "sigs": SIGS, + # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle things that aren't even available. + "spot_sources": list( + map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["spot_providers"]))), + "alert_sources": list( + map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"]))), + "continents": CONTINENTS, + "propagation_modes": list(PROPAGATION_MODES.values()), + "max_spot_age": MAX_SPOT_AGE, + "spot_allowed": ALLOW_SPOTTING} + # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from + # one of our proviers. + if ALLOW_SPOTTING: + options["spot_sources"].append("API") - self.write(safe_json_dumps(options)) - self.set_status(200) - self.set_header("Cache-Control", "no-store") - self.set_header("Content-Type", "application/json") + self.write(safe_json_dumps(options)) + self.set_status(200) + self.set_header("Cache-Control", "no-store") + self.set_header("Content-Type", "application/json") + + except Exception as e: + logging.error("Exception when handling client request to options API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) + self.set_status(500) diff --git a/server/handlers/api/solar_conditions.py b/server/handlers/api/solar_conditions.py index 5fdcc46..7193b30 100644 --- a/server/handlers/api/solar_conditions.py +++ b/server/handlers/api/solar_conditions.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime from typing import Any @@ -7,6 +8,7 @@ from tornado import httputil from tornado.web import Application from core.prometheus_metrics_handler import api_requests_counter +from core.utils import safe_json_dumps class APISolarConditionsHandler(tornado.web.RequestHandler): @@ -22,13 +24,19 @@ class APISolarConditionsHandler(tornado.web.RequestHandler): self._web_server_metrics = web_server_metrics def get(self): - # Metrics - self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) - self._web_server_metrics["api_access_counter"] += 1 - self._web_server_metrics["status"] = "OK" - api_requests_counter.inc() + try: + # Metrics + self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) + self._web_server_metrics["api_access_counter"] += 1 + self._web_server_metrics["status"] = "OK" + api_requests_counter.inc() - self.write(self._solar_conditions.to_json()) - self.set_status(200) - self.set_header("Cache-Control", "no-store") - self.set_header("Content-Type", "application/json") + self.write(self._solar_conditions.to_json()) + self.set_status(200) + self.set_header("Cache-Control", "no-store") + self.set_header("Content-Type", "application/json") + + except Exception as e: + logging.error("Exception when handling client request to solar conditions API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) + self.set_status(500) diff --git a/server/handlers/api/spots.py b/server/handlers/api/spots.py index 0eba452..675076f 100644 --- a/server/handlers/api/spots.py +++ b/server/handlers/api/spots.py @@ -59,11 +59,10 @@ class APISpotsHandler(tornado.web.RequestHandler): self.write(safe_json_dumps(data)) self.set_status(200) except ValueError as e: - logging.error(e) self.write(safe_json_dumps("Bad request - " + str(e))) self.set_status(400) except Exception as e: - logging.error(e) + logging.error("Excedption when handling client request to spots API: %s", e, exc_info=True) self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) self.set_header("Cache-Control", "no-store") diff --git a/server/handlers/api/status.py b/server/handlers/api/status.py index 1a125cd..589831c 100644 --- a/server/handlers/api/status.py +++ b/server/handlers/api/status.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime from typing import Any @@ -23,13 +24,19 @@ class APIStatusHandler(tornado.web.RequestHandler): self._web_server_metrics = web_server_metrics def get(self): - # Metrics - self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) - self._web_server_metrics["api_access_counter"] += 1 - self._web_server_metrics["status"] = "OK" - api_requests_counter.inc() + try: + # Metrics + self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) + self._web_server_metrics["api_access_counter"] += 1 + self._web_server_metrics["status"] = "OK" + api_requests_counter.inc() - self.write(safe_json_dumps(self._status_data)) - self.set_status(200) - self.set_header("Cache-Control", "no-store") - self.set_header("Content-Type", "application/json") + self.write(safe_json_dumps(self._status_data)) + self.set_status(200) + self.set_header("Cache-Control", "no-store") + self.set_header("Content-Type", "application/json") + + except Exception as e: + logging.error("Exception when handling client request to status API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) + self.set_status(500) diff --git a/solarconditionsproviders/giroionosonde.py b/solarconditionsproviders/giroionosonde.py index 8a9e398..86d2042 100644 --- a/solarconditionsproviders/giroionosonde.py +++ b/solarconditionsproviders/giroionosonde.py @@ -127,10 +127,11 @@ class GIROIonosonde(SolarConditionsProvider): from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S") to_str = to_time.strftime("%Y.%m.%d+%H:%M:%S") url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUFD,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}" - response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15)) - if response.status_code != 200: + http_response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15)) + if http_response.status_code != 200: + logging.warning(f"Giro ionosonde API returned HTTP {http_response.status_code}.") return None, None, None - return self._parse_all(response.text) + return self._parse_all(http_response.text) @staticmethod def _parse_all(text): diff --git a/solarconditionsproviders/hamqsl.py b/solarconditionsproviders/hamqsl.py index 62cfd70..92c5306 100644 --- a/solarconditionsproviders/hamqsl.py +++ b/solarconditionsproviders/hamqsl.py @@ -18,10 +18,6 @@ class HamQSL(HTTPSolarConditionsProvider): super().__init__(provider_config, URL, POLL_INTERVAL) def _http_response_to_solar_conditions(self, http_response): - if http_response.status_code != 200: - logging.warning("HamQSL solar conditions API returned HTTP " + str(http_response.status_code)) - return None - root = ElementTree.fromstring(http_response.text) sd = root.find("solardata") if sd is None: diff --git a/solarconditionsproviders/http_solar_conditions_provider.py b/solarconditionsproviders/http_solar_conditions_provider.py index 0a8dfcd..4a639fd 100644 --- a/solarconditionsproviders/http_solar_conditions_provider.py +++ b/solarconditionsproviders/http_solar_conditions_provider.py @@ -39,12 +39,17 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider): try: logging.debug("Polling " + self.name + " solar conditions API...") http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30)) - new_data = self._http_response_to_solar_conditions(http_response) - self.update_data(new_data) + # Check response code was good + if http_response.status_code == 200: + new_data = self._http_response_to_solar_conditions(http_response) + self.update_data(new_data) - self.status = "OK" - self.last_update_time = datetime.now(pytz.UTC) - logging.debug("Received data from " + self.name + " solar conditions API.") + self.status = "OK" + self.last_update_time = datetime.now(pytz.UTC) + logging.debug("Received data from " + self.name + " solar conditions API.") + else: + self.status = "Error" + logging.warning(f"{self.name} solar conditions API returned HTTP {http_response.status_code}.") except Exception: self.status = "Error" diff --git a/solarconditionsproviders/kc2gprop.py b/solarconditionsproviders/kc2gprop.py index 41342b1..99660ad 100644 --- a/solarconditionsproviders/kc2gprop.py +++ b/solarconditionsproviders/kc2gprop.py @@ -44,9 +44,9 @@ class KC2GProp(SolarConditionsProvider): def _poll(self): try: logging.debug("Polling KC2G ionosonde data...") - response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30)) - if response.status_code != 200: - logging.warning(f"KC2G ionosonde API returned HTTP {response.status_code}") + http_response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30)) + if http_response.status_code != 200: + logging.warning(f"KC2G ionosonde API returned HTTP {http_response.status_code}") return now = datetime.now(timezone.utc) @@ -57,7 +57,7 @@ class KC2GProp(SolarConditionsProvider): ionosonde_data = dict(self._solar_conditions.ionosonde_data or {}) updated_count = 0 - for reading in response.json(): + for reading in http_response.json(): station = reading.get("station", {}) ursi = station.get("code") name = station.get("name") diff --git a/solarconditionsproviders/noaa3dayforecast.py b/solarconditionsproviders/noaa3dayforecast.py index d677e66..de5bf1a 100644 --- a/solarconditionsproviders/noaa3dayforecast.py +++ b/solarconditionsproviders/noaa3dayforecast.py @@ -80,10 +80,6 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider): return result if result else None def _http_response_to_solar_conditions(self, http_response): - if http_response.status_code != 200: - logging.warning("NOAA K-index forecast API returned HTTP " + str(http_response.status_code)) - return None - lines = http_response.text.splitlines() # Find the "NOAA Kp index breakdown" section header diff --git a/spothole.py b/spothole.py index 3657487..d2ed8be 100644 --- a/spothole.py +++ b/spothole.py @@ -8,7 +8,7 @@ import sys from diskcache import Cache from core.cleanup import CleanupTimer -from core.config import config, SERVER_OWNER_CALLSIGN +from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL from core.constants import SOFTWARE_VERSION from core.lookup_helper import lookup_helper from core.status_reporter import StatusReporter @@ -84,9 +84,9 @@ def get_solar_conditions_provider_from_config(config_providers_entry): if __name__ == '__main__': # Set up logging root = logging.getLogger() - root.setLevel(logging.INFO) + root.setLevel(LOG_LEVEL) handler = logging.StreamHandler(sys.stdout) - handler.setLevel(logging.INFO) + handler.setLevel(LOG_LEVEL) formatter = logging.Formatter("%(levelname)s : %(message)s") handler.setFormatter(formatter) root.handlers.clear() diff --git a/spotproviders/http_spot_provider.py b/spotproviders/http_spot_provider.py index 39b052d..57bd046 100644 --- a/spotproviders/http_spot_provider.py +++ b/spotproviders/http_spot_provider.py @@ -41,15 +41,20 @@ class HTTPSpotProvider(SpotProvider): # Request data from API logging.debug("Polling " + self.name + " spot API...") http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30)) - # Pass off to the subclass for processing - new_spots = self._http_response_to_spots(http_response) - # Submit the new spots for processing. There might not be any spots for the less popular programs. - if new_spots: - self._submit_batch(new_spots) + # Check response code was good + if http_response.status_code == 200: + # Pass off to the subclass for processing + new_spots = self._http_response_to_spots(http_response) + # Submit the new spots for processing. There might not be any spots for the less popular programs. + if new_spots: + self._submit_batch(new_spots) - self.status = "OK" - self.last_update_time = datetime.now(pytz.UTC) - logging.debug("Received data from " + self.name + " spot API.") + self.status = "OK" + self.last_update_time = datetime.now(pytz.UTC) + logging.debug("Received data from " + self.name + " spot API.") + else: + self.status = "Error" + logging.warning(f"{self.name} spot API returned HTTP {http_response.status_code}.") except Exception: self.status = "Error" diff --git a/templates/add_spot.html b/templates/add_spot.html index 660b17b..587c910 100644 --- a/templates/add_spot.html +++ b/templates/add_spot.html @@ -76,7 +76,7 @@ - + diff --git a/templates/alerts.html b/templates/alerts.html index 0e90d50..26dff8c 100644 --- a/templates/alerts.html +++ b/templates/alerts.html @@ -75,7 +75,7 @@ - + diff --git a/templates/bands.html b/templates/bands.html index 8467f6a..5c0203a 100644 --- a/templates/bands.html +++ b/templates/bands.html @@ -75,8 +75,8 @@ - - + + diff --git a/templates/base.html b/templates/base.html index 8639554..a801bba 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,6 +1,6 @@ {% extends "skeleton.html" %} {% block head_extra %} - + @@ -10,10 +10,10 @@ - - - - + + + + {% end %} {% block body %}
diff --git a/templates/conditions.html b/templates/conditions.html index d71800c..8124c77 100644 --- a/templates/conditions.html +++ b/templates/conditions.html @@ -284,7 +284,7 @@
- + diff --git a/templates/map.html b/templates/map.html index 296628d..ec5c18d 100644 --- a/templates/map.html +++ b/templates/map.html @@ -95,8 +95,8 @@ - - + + diff --git a/templates/spots.html b/templates/spots.html index 41462e3..701fce4 100644 --- a/templates/spots.html +++ b/templates/spots.html @@ -116,8 +116,8 @@ - - + + diff --git a/templates/status.html b/templates/status.html index c4cb91b..0454209 100644 --- a/templates/status.html +++ b/templates/status.html @@ -59,7 +59,7 @@ - +