diff --git a/.idea/spothole.iml b/.idea/spothole.iml index 803561c..f16c839 100644 --- a/.idea/spothole.iml +++ b/.idea/spothole.iml @@ -2,8 +2,9 @@ + - + diff --git a/README.md b/README.md index 2224f3f..9127be0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ![Spothole](/webassets/img/logo.png) +# ![Spothole](/static/img/logo.png) Spothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open JSON API as well as a website to browse the data. @@ -89,15 +89,15 @@ callsign and park data. Various approaches exist to writing your own client, but in general: -* Refer to the API docs. These are built on an OpenAPI definition file (`/webassets/apidocs/openapi.yml`), which you can +* Refer to the API docs. These are built on an OpenAPI definition file (`/static/apidocs/openapi.yml`), which you can automatically use to generate a client skeleton using various software. * Call the main "spots" or "alerts" API endpoints to get the data you want. For example, your app could call `https://spothole.app/api/v2/spots` once every few minutes. Apply filters if necessary. * Call the "options" API to get an idea of which bands, modes etc. the server knows about. You might want to do that first before calling the spots/alerts APIs, to allow you to populate your filters correctly. -* Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "map" and "bands" - pages simply query the main spot API on a timer, whereas the main/spots page combines this approach with using the - Server-Sent Events (SSE) endpoint to update live. +* Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "alerts"/"upcoming" + page simply query the main spot API on a timer, whereas the spots, map and bands pages combine this approach with + using the Server-Sent Events (SSE) endpoint to update live. * Let me know if you get stuck, I'm happy to help. Please don't hammer the API with an unnecessarily high request rate. For example, Spothole only queries the POTA API @@ -318,93 +318,78 @@ To set up nginx as a reverse proxy that sits in front of Spothole, first ensure Create a file at `/etc/nginx/sites-available/` called `spothole`. Give it the following contents, replacing `spothole.app` with the domain name on which you want to run Spothole. If you changed the port on which Spothole runs, -update that on the "proxy_pass" line too. +update that on the "proxy_pass" line, and if you installed Spothole somewhere other than `/home/spothole/spothole`, +adjust the alias location for serving static files. + +(The latter section, configuring the nginx server to serve static files directly, improves efficiency because it saves +Spothole itself from serving JS, CSS etc. files. If you can't do this for some reason, e.g. your nginx and spothole are +on different computers, you can omit the `location /static/ {}` block.) ```nginx server { server_name spothole.app; + # Global proxy settings + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_connect_timeout 10s; + proxy_buffering on; + + # Pass on IP address and host information to Spothole, in case logging this information is required + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + # Wellknown area for Lets Encrypt location /.well-known/ { alias /var/www/html/.well-known/; } + + # Load static assets directly from the Spothole static directory + location /static/ { + alias /home/spothole/spothole/static/; + expires 1h; + add_header Cache-Control "public, max-age=3600, must-revalidate"; + } # SSE endpoints - location ~ ^/api/v2/(spots|alerts)/stream { + location ~ ^/api/v2/(spots|alerts)/stream/? { proxy_pass http://127.0.0.1:8080; - # Allow keep-alive - proxy_http_version 1.1; - proxy_set_header Connection ""; - - # Set correct content type for SSE API calls - add_header Content-Type text/event-stream always; - - # Set remove buffering, remove caching, add suitable timeouts for SSE API calls + # Remove buffering, remove caching, add suitable timeouts for SSE API calls proxy_buffering off; proxy_cache off; proxy_read_timeout 24h; - proxy_connect_timeout 10s; proxy_send_timeout 24h; proxy_set_header X-Accel-Buffering no; add_header Cache-Control no-store always; # Allow cross-origin requests to API proxy_hide_header Access-Control-Allow-Origin; - add_header Access-Control-Allow-Origin * always; - - # Pass on IP address and host information to Spothole, in case logging this information is required - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-Proto $scheme; + add_header Access-Control-Allow-Origin * always; } # Other API endpoints location /api/ { proxy_pass http://127.0.0.1:8080; - # Allow keep-alive - proxy_http_version 1.1; - proxy_set_header Connection ""; - - # Set up buffering, remove caching, add suitable timeouts for API calls - proxy_buffering on; + # Remove buffering, remove caching, add suitable timeouts for API calls + proxy_buffering off; proxy_cache off; proxy_read_timeout 30s; - proxy_connect_timeout 10s; add_header Cache-Control no-store always; # Allow cross-origin requests to API proxy_hide_header Access-Control-Allow-Origin; add_header Access-Control-Allow-Origin * always; - - # Pass on IP address and host information to Spothole, in case logging this information is required - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-Proto $scheme; } - # Static assets + # Templated pages location / { proxy_pass http://127.0.0.1:8080; - - # Allow keep-alive - proxy_http_version 1.1; - proxy_set_header Connection ""; - - # Set up buffering and caching, add suitable timeouts for static asset requests - proxy_buffering on; proxy_read_timeout 30s; - proxy_connect_timeout 10s; - add_header Cache-Control "public, max-age=3600, must-revalidate" always; - - # Pass on IP address and host information to Spothole, in case logging this information is required - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-Proto $scheme; + add_header Cache-Control "no-cache, must-revalidate" always; } } ``` @@ -467,12 +452,12 @@ To navigate your way around the source code, this list may help. *HTML/JS/CSS front-end code* -* `/webassets` - Root for static files served by the web server -* `/webassets/apidocs` - Contains the OpenAPI spec (`openapi.yml`) -* `/webassets/css` - CSS files used by the web front-end -* `/webassets/img` - image files used by the web front-end -* `/webassets/js` - JavaScript used by the web front-end -* `/webassets/vendor` - Third-party libraries (CSS, JS, fonts and images) +* `/static` - Root for static files served by the web server. These are all served from a path starting `/static/`. +* `/static/apidocs` - Contains the OpenAPI spec (`openapi.yml`) +* `/static/css` - CSS files used by the web front-end +* `/static/img` - image files used by the web front-end +* `/static/js` - JavaScript used by the web front-end +* `/static/vendor` - Third-party libraries (CSS, JS, fonts and images) *Miscellaneous* @@ -524,14 +509,14 @@ my knowledge, created by HA8TKS for his CQ and ITU zone layers for Leaflet. `/da maps of conference centres. The project contains a set of flag icons generated using the "Noto Color Emoji" font on a Debian system, in the -`/webassets/img/flags/` directory. +`/static/img/flags/` directory. The software uses a number of Python libraries as listed in `requirements.txt`, and a number of JavaScript libraries. This project would not have been possible without these libraries, so many thanks to their developers. ### Third Party Libraries -A number of third-party libraries are self-hosted in the `/webassets/vendor/` directory. These files are subject to +A number of third-party libraries are self-hosted in the `/static/vendor/` directory. These files are subject to their own licences and are not covered by the overall licence declared in the `LICENSE` file. A number of third-party libraries are self-hosted in the `/webassets/vendor/` directory. These files are subject to diff --git a/alertproviders/http_alert_provider.py b/alertproviders/http_alert_provider.py index a4dd99a..9bef504 100644 --- a/alertproviders/http_alert_provider.py +++ b/alertproviders/http_alert_provider.py @@ -4,6 +4,7 @@ from threading import Thread, Event import pytz import requests +from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from alertproviders.alert_provider import AlertProvider from core.constants import HTTP_HEADERS @@ -41,16 +42,25 @@ class HTTPAlertProvider(AlertProvider): # Request data from API logging.debug("Polling " + self.name + " alert API...") http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30)) - # Pass off to the subclass for processing - new_alerts = self._http_response_to_alerts(http_response) - # Submit the new alerts for processing. There might not be any alerts for the less popular programs. - if new_alerts: - self._submit_batch(new_alerts) + # Check response code was good + if http_response.ok: + # Pass off to the subclass for processing + new_alerts = self._http_response_to_alerts(http_response) + # Submit the new alerts for processing. There might not be any alerts for the less popular programs. + if new_alerts: + self._submit_batch(new_alerts) - self.status = "OK" - self.last_update_time = datetime.now(pytz.UTC) - logging.debug("Received data from " + self.name + " alert API.") + self.status = "OK" + self.last_update_time = datetime.now(pytz.UTC) + logging.debug("Received data from " + self.name + " alert API.") + else: + self.status = "Error" + logging.warning(f"HTTP {http_response.status_code} when calling {self.name} alerts API.") + except ConnectionError: + logging.warning(f"Connection error when accessing {self.name} alerts API.") + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when accessing {self.name} alerts API.") except Exception: self.status = "Error" logging.exception("Exception in HTTP JSON Alert Provider (" + self.name + ")") diff --git a/config-example.yml b/config-example.yml index f0c5a1b..8aa8374 100644 --- a/config-example.yml +++ b/config-example.yml @@ -225,6 +225,14 @@ allow-upstream-spotting: true recaptcha-site-key: "" recaptcha-secret-key: "" +# Log web requests? Useful to see what your users are requesting in terms of pages and API endpoints, but will fill up +# your log quickly on a popular server. +log-web-requests: false + +# Minimum severity of log statements to print to the system log. Defaults to INFO, change to DEBUG if you need more +# information. +log-level: INFO + # Options for the web UI. web-ui-options: spot-count: [ 10, 25, 50, 100 ] diff --git a/core/cache_utils.py b/core/cache_utils.py index 2ec492c..68e521f 100644 --- a/core/cache_utils.py +++ b/core/cache_utils.py @@ -8,8 +8,11 @@ from requests_cache import CachedSession # of time has passed. This is used throughout Spothole to cache data that does not change # rapidly. The ThreadSafeSession construct here protects it against some multithreading # contention weirdness we sometimes used to see on startup where the cache was hammered -# pretty hard. -_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30)) +# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type +# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote +# servers for data they've told us they can't provide. +_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30), + allowable_codes=(200, 400, 401, 403, 404)) _lock = threading.Lock() diff --git a/core/config.py b/core/config.py index 7aac959..9865e12 100644 --- a/core/config.py +++ b/core/config.py @@ -25,6 +25,8 @@ WEB_UI_OPTIONS = config.get("web-ui-options", {}) API_ONLY_MODE = config.get("api-only-mode", False) RECAPTCHA_SECRET_KEY = config.get("recaptcha-secret-key", "") RECAPTCHA_SITE_KEY = config.get("recaptcha-site-key", "") +LOG_LEVEL = config.get("log-level", "INFO") +LOG_WEB_REQUESTS = config.get("log-web-requests", False) # For ease of config, each spot provider owns its own config about whether it should be enabled by default in the web UI # but for consistency we provide this to the front-end in web-ui-options because it has no impact outside of the web UI. diff --git a/core/constants.py b/core/constants.py index 4127d65..4e3f5fa 100644 --- a/core/constants.py +++ b/core/constants.py @@ -27,8 +27,8 @@ SIGS = [ SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"), SIG(name="BOTA", comment_names=[], description="Beaches on the Air"), SIG(name="KRMNPA", comment_names=["KRMNPA"], description="Keith Roget Memorial National Parks Award"), - SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"[A-Z]{2}\-\d{4}"), - SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2}R\-\d{4}"), + SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"LL[A-Z]{2}\-\d{4}"), + SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2,3}R\-\d{4}"), SIG(name="Tiles", comment_names=[], description="Tiles on the Air", ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}"), SIG(name="WAB", comment_names=["WAB"], description="Worked All Britain", ref_regex=r"[A-Z]{1,2}[0-9]{2}"), SIG(name="WAI", comment_names=["WAI"], description="Worked All Ireland", ref_regex=r"[A-Z][0-9]{2}"), @@ -104,5 +104,6 @@ PROPAGATION_MODES = { "AU": "Aurora", "MS": "Meteor scatter", "RS": "Rain scatter", - "AS": "Aircraft scatter" + "AS": "Aircraft scatter", + "SAT": "Satellite" } diff --git a/core/lookup_helper.py b/core/lookup_helper.py index 7fe2594..8d8dec4 100644 --- a/core/lookup_helper.py +++ b/core/lookup_helper.py @@ -5,13 +5,13 @@ import re import urllib.parse from datetime import timedelta -import requests import xmltodict from diskcache import Cache from pyhamtools import LookupLib, Callinfo, callinfo from pyhamtools.exceptions import APIKeyMissingError from pyhamtools.frequency import freq_to_band from pyhamtools.locator import latlong_to_locator +from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from requests_cache import CachedSession from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE @@ -143,13 +143,21 @@ class LookupHelper: try: logging.info("Downloading Country-files.com cty.plist...") response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist", - headers=HTTP_HEADERS).text + headers=HTTP_HEADERS) - with open(self._country_files_cty_plist_download_location, "w") as f: - f.write(response) - f.flush() - return True + if response.ok: + with open(self._country_files_cty_plist_download_location, "w") as f: + f.write(response.text) + f.flush() + return True + else: + logging.warning(f"HTTP {response.status_code} when downloading Country-files.com cty.plist.") + return False + except ConnectionError: + logging.warning(f"Connection error when downloading Clublog cty.xml.") + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when downloading Clublog cty.xml.") except Exception as e: logging.error("Exception when downloading Clublog cty.xml", e) return False @@ -161,13 +169,21 @@ class LookupHelper: logging.info("Downloading dxcc.json...") response = SEMI_STATIC_URL_DATA_CACHE.get( "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json", - headers=HTTP_HEADERS).text + headers=HTTP_HEADERS) - with open(self._dxcc_json_download_location, "w") as f: - f.write(response) - f.flush() - return True + if response.ok: + with open(self._dxcc_json_download_location, "w") as f: + f.write(response.text) + f.flush() + return True + else: + logging.warning(f"HTTP {response.status_code} when downloading dxcc.json.") + return False + except ConnectionError: + logging.warning(f"Connection error when downloading dxcc.json.") + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when downloading dxcc.json.") except Exception as e: logging.error("Exception when downloading dxcc.json", e) return False @@ -490,25 +506,47 @@ class LookupHelper: # Try the call as given, then fall back to the base call (strips /P, /M etc.) calls_to_try = [call] - home_call = callinfo.Callinfo.get_homecall(call) - if home_call != call: - calls_to_try.append(home_call) + try: + home_call = callinfo.Callinfo.get_homecall(call) + if home_call != call: + calls_to_try.append(home_call) + except ValueError: + logging.debug("Could not look up home call for callsign %s", call) for lookup_call in calls_to_try: try: - lookup_response = requests.get( + response = SEMI_STATIC_URL_DATA_CACHE.get( self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call), - headers=HTTP_HEADERS, timeout=10).content - raw = xmltodict.parse(lookup_response).get("QRZDatabase", {}).get("Callsign") - if raw: - data = _normalize_qrz_data(raw) - self._qrz_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds - return data + headers=HTTP_HEADERS, timeout=10) + if response.ok: + qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {}) + if qrz_response: + if "Callsign" in qrz_response: + data = _normalize_qrz_data(qrz_response.get("Callsign")) + self._qrz_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds + return data + elif "Session" in qrz_response and "Error" in qrz_response.get("Session"): + # Errors here are normally just "callsign not in database", no need to log that ourselves + # above debug level. + logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call, + qrz_response.get("Session").get("Error")) + + elif not response.from_cache: + logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call) + elif not response.from_cache: + logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call) + except (KeyError, ValueError): continue + except ConnectionError: + logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call) + continue + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call) + continue except Exception: - logging.error("Exception when looking up QRZ data") - return None + logging.error("Exception when looking up callsign %s using QRZ", lookup_call, exc_info=True) + continue # Not found in QRZ; cache None so we don't keep retrying self._qrz_callsign_data_cache.add(call, None, expire=604800) # 1 week in seconds @@ -546,23 +584,36 @@ class LookupHelper: # Try the call as given, then fall back to the base call (strips /P, /M etc.) calls_to_try = [call] - home_call = callinfo.Callinfo.get_homecall(call) - if home_call != call: - calls_to_try.append(home_call) + try: + home_call = callinfo.Callinfo.get_homecall(call) + if home_call != call: + calls_to_try.append(home_call) + except ValueError: + logging.debug("Could not look up home call for callsign %s", call) for lookup_call in calls_to_try: try: - lookup_data = SEMI_STATIC_URL_DATA_CACHE.get( + response = SEMI_STATIC_URL_DATA_CACHE.get( self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus( - lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS).content - data = xmltodict.parse(lookup_data)["HamQTH"]["search"] - self._hamqth_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds - return data + lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS) + if response.ok: + data = xmltodict.parse(response.content)["HamQTH"]["search"] + self._hamqth_callsign_data_cache.add(call, data, expire=604800) # 1 week in seconds + return data + elif not response.from_cache: + logging.warning("HTTP %d looking up callsign %s using HamQTH", response.status_code, lookup_call) + except (KeyError, ValueError): continue + except ConnectionError: + logging.warning(f"Connection error when looking up callsign %s using HamQTH", lookup_call) + continue + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when looking up callsign %s using HamQTH", lookup_call) + continue except Exception: - logging.error("Exception when looking up HamQTH data") - return None + logging.error("Exception when looking up callsign %s using HamQTH", lookup_call, exc_info=True) + continue # Not found in HamQTH; cache None so we don't keep retrying self._hamqth_callsign_data_cache.add(call, None, expire=604800) # 1 week in seconds diff --git a/core/sig_utils.py b/core/sig_utils.py index db180dc..258d5ac 100644 --- a/core/sig_utils.py +++ b/core/sig_utils.py @@ -2,6 +2,7 @@ import csv import logging from pyhamtools.locator import latlong_to_locator, locator_to_latlong +from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE from core.constants import SIGS, HTTP_HEADERS @@ -11,6 +12,14 @@ from core.geo_utils import wab_wai_square_to_lat_lon # file in Spothole and load it on startup. with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f: _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")} +# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single +# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came +# live from the internet, to avoid repopulating them every time we pull the same data from the cache. +_WWFF_INDEX_CACHE = {} +_SIOTA_INDEX_CACHE = {} +_WOTA_INDEX_CACHE = {} +_ZLOTA_INDEX_CACHE = {} +_LLOTA_INDEX_CACHE = {} def get_ref_regex_for_sig(sig): @@ -45,92 +54,136 @@ def populate_sig_ref_info(sig_ref): try: if sig.upper() == "POTA": response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS) - if not response.ok: + if response.ok: + data = response.json() + if data: + fullname = str(data["name"]) if "name" in data else None + if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "": + fullname = fullname + " " + data["parktypeDesc"] + sig_ref.name = fullname + sig_ref.url = "https://pota.app/#/park/" + ref_id + sig_ref.grid = data["grid6"] if "grid6" in data else None + sig_ref.latitude = data["latitude"] if "latitude" in data else None + sig_ref.longitude = data["longitude"] if "longitude" in data else None + elif not response.from_cache: + logging.warning("Malformed response looking up %s ref %s", sig, ref_id) + elif not response.from_cache: logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - data = response.json() if response.ok else None - if data: - fullname = str(data["name"]) if "name" in data else None - if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "": - fullname = fullname + " " + data["parktypeDesc"] - sig_ref.name = fullname - sig_ref.url = "https://pota.app/#/park/" + ref_id - sig_ref.grid = data["grid6"] if "grid6" in data else None - sig_ref.latitude = data["latitude"] if "latitude" in data else None - sig_ref.longitude = data["longitude"] if "longitude" in data else None + elif sig.upper() == "SOTA": response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id, headers=HTTP_HEADERS) - if not response.ok: + if response.ok: + data = response.json() + if data: + sig_ref.name = data["name"] if "name" in data else None + sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id + sig_ref.grid = data["locator"] if "locator" in data else None + sig_ref.latitude = data["latitude"] if "latitude" in data else None + sig_ref.longitude = data["longitude"] if "longitude" in data else None + sig_ref.activation_score = data["points"] if "points" in data else None + elif not response.from_cache: + logging.warning("Malformed response looking up %s ref %s", sig, ref_id) + elif not response.from_cache: logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - data = response.json() if response.ok else None - if data: - sig_ref.name = data["name"] if "name" in data else None - sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id - sig_ref.grid = data["locator"] if "locator" in data else None - sig_ref.latitude = data["latitude"] if "latitude" in data else None - sig_ref.longitude = data["longitude"] if "longitude" in data else None - sig_ref.activation_score = data["points"] if "points" in data else None + elif sig.upper() == "WWBOTA": response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id, headers=HTTP_HEADERS) - if not response.ok: + if response.ok: + data = response.json() + if data: + sig_ref.name = data["name"] if "name" in data else None + sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None + sig_ref.grid = data["locator"] if "locator" in data else None + sig_ref.latitude = data["lat"] if "lat" in data else None + sig_ref.longitude = data["long"] if "long" in data else None + elif not response.from_cache: + logging.warning("Malformed response looking up %s ref %s", sig, ref_id) + elif not response.from_cache: logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - data = response.json() if response.ok else None - if data: - sig_ref.name = data["name"] if "name" in data else None - sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None - sig_ref.grid = data["locator"] if "locator" in data else None - sig_ref.latitude = data["lat"] if "lat" in data else None - sig_ref.longitude = data["long"] if "long" in data else None + elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA": response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id, headers=HTTP_HEADERS) - if not response.ok: + if response.ok: + data = response.json() + if data: + sig_ref.name = data["name"] if "name" in data else None + sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id + sig_ref.grid = data["locator"] if "locator" in data else None + + # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are + # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this + # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters, + # just use the single point we have. + if data.get("latitude") is not None and data.get("longitude") is not None and data.get( + "lat2") is not None and data.get("lng2") is not None: + sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0 + sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0 + else: + sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None + sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None + elif not response.from_cache: + logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id) + elif not response.from_cache: logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - data = response.json() if response.ok else None - if data: - sig_ref.name = data["name"] if "name" in data else None - sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id - sig_ref.grid = data["locator"] if "locator" in data else None - sig_ref.latitude = data["latitude"] if "latitude" in data else None - sig_ref.longitude = data["longitude"] if "longitude" in data else None + elif sig.upper() == "WWFF": - wwff_response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv", - headers=HTTP_HEADERS) - if not wwff_response.ok: - logging.warning("HTTP %d looking up %s ref %s", wwff_response.status_code, sig, ref_id) - return sig_ref - wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_response.content.decode().splitlines())} - row = wwff_index.get(ref_id) - if row: - sig_ref.name = row["name"] if "name" in row else None - sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id - sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None - sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None - sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row["longitude"] != "-" else None + response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv", + headers=HTTP_HEADERS) + if response.ok: + global _WWFF_INDEX_CACHE + if not bool(_WWFF_INDEX_CACHE) or not response.from_cache: + # New data from WWFF, update our internal map + _WWFF_INDEX_CACHE = {row["reference"]: row for row in + csv.DictReader(response.content.decode().splitlines())} + row = _WWFF_INDEX_CACHE.get(ref_id) + if row: + sig_ref.name = row["name"] if "name" in row else None + sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id + sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None + sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None + sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[ + "longitude"] != "-" else None + elif not response.from_cache: + logging.warning("WWFF database did not contain data for ref %s", ref_id) + elif not response.from_cache: + logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + elif sig.upper() == "SIOTA": - siota_response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv", - headers=HTTP_HEADERS) - if not siota_response.ok: - logging.warning("HTTP %d looking up %s ref %s", siota_response.status_code, sig, ref_id) - return sig_ref - siota_index = {row["SILO_CODE"]: row for row in - csv.DictReader(siota_response.content.decode().splitlines())} - row = siota_index.get(ref_id) - if row: - sig_ref.name = row["NAME"] if "NAME" in row else None - sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None - sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None - sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None + response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv", + headers=HTTP_HEADERS) + if response.ok: + global _SIOTA_INDEX_CACHE + if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache: + # New data from SIOTA, update our internal map + _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in + csv.DictReader(response.content.decode().splitlines())} + row = _SIOTA_INDEX_CACHE.get(ref_id) + if row: + sig_ref.name = row["NAME"] if "NAME" in row else None + sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None + sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None + sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None + elif not response.from_cache: + logging.warning("SIOTA database did not contain data for ref %s", ref_id) + elif not response.from_cache: + logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + elif sig.upper() == "WOTA": response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json", headers=HTTP_HEADERS) - if not response.ok: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - data = response.json() if response.ok else None - if data: - for feature in data.get("features", []): - if feature["properties"]["wotaId"] == ref_id: + if response.ok: + data = response.json() + if data: + global _WOTA_INDEX_CACHE + if not bool(_WOTA_INDEX_CACHE) or not response.from_cache: + # New data from WOTA, update our internal map + _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in + data.get("features", [])} + feature = _WOTA_INDEX_CACHE.get(ref_id) + if feature: sig_ref.name = feature["properties"]["title"] # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be # added to them @@ -141,48 +194,68 @@ def populate_sig_ref_info(sig_ref): sig_ref.grid = feature["properties"]["qthLocator"] sig_ref.latitude = feature["geometry"]["coordinates"][1] sig_ref.longitude = feature["geometry"]["coordinates"][0] - break + elif not response.from_cache: + logging.warning("Malformed response looking up %s ref %s", sig, ref_id) + elif not response.from_cache: + logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + elif sig.upper() == "ZLOTA": response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS) - if not response.ok: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - data = response.json() if response.ok else None - if isinstance(data, list): - for asset in data: - if asset["code"] == ref_id: + if response.ok: + data = response.json() + if isinstance(data, list): + global _ZLOTA_INDEX_CACHE + if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache: + # New data from ZLOTA, update our internal map + _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data} + asset = _ZLOTA_INDEX_CACHE.get(ref_id) + if asset: sig_ref.name = asset["name"] - sig_ref.url = "https://ontheair.nz/assets/ZLI_OT-030" + ref_id.replace("/", "_") + sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_") try: sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6) except: logging.debug("Invalid lat/lon received for reference") sig_ref.latitude = asset["y"] sig_ref.longitude = asset["x"] - break + elif not response.from_cache: + logging.warning("Malformed response looking up %s ref %s", sig, ref_id) + elif not response.from_cache: + logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + elif sig.upper() == "BOTA": if not sig_ref.name: sig_ref.name = sig_ref.id sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-") + elif sig.upper() == "LLOTA": response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references", headers=HTTP_HEADERS) - if not response.ok: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - data = response.json() if response.ok else None - if isinstance(data, list): - for ref in data: - if ref["reference_code"] == ref_id: + if response.ok: + data = response.json() + if isinstance(data, list): + global _LLOTA_INDEX_CACHE + if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache: + # New data from LLOTA, update our internal map + _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data} + ref = _LLOTA_INDEX_CACHE.get(ref_id) + if ref: sig_ref.name = str(ref["name"]) sig_ref.url = "https://llota.app/list/ref/" + ref_id sig_ref.grid = str(ref["grid_locator"]) ll = locator_to_latlong(sig_ref.grid) sig_ref.latitude = ll[0] sig_ref.longitude = ll[1] - break + elif not response.from_cache: + logging.warning("Malformed response looking up %s ref %s", sig, ref_id) + elif not response.from_cache: + logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) + elif sig.upper() == "WWTOTA": if not sig_ref.name: sig_ref.name = sig_ref.id sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + str(sig_ref.name) + elif sig.upper() == "TILES": # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same if not sig_ref.name: @@ -193,6 +266,7 @@ def populate_sig_ref_info(sig_ref): ll = locator_to_latlong(str(sig_ref.grid)) sig_ref.latitude = ll[0] sig_ref.longitude = ll[1] + elif sig.upper() == "WAB" or sig.upper() == "WAI": ll = wab_wai_square_to_lat_lon(ref_id) if ll: @@ -202,21 +276,31 @@ def populate_sig_ref_info(sig_ref): sig_ref.latitude = ll[0] sig_ref.longitude = ll[1] except: - logging.debug("Invalid lat/lon received for reference") + logging.warning("Invalid lat/lon received for WAB/WAI reference") + elif sig.upper() == "DME": # Zero-pad to 5 digits to match our source data row = _DME_INDEX.get(ref_id.zfill(5)) if row: sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"] - sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get("LATITUD_ETRS89_REGCAN95") else None - sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get("LONGITUD_ETRS89_REGCAN95") else None + sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get( + "LATITUD_ETRS89_REGCAN95") else None + sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get( + "LONGITUD_ETRS89_REGCAN95") else None if sig_ref.latitude and sig_ref.longitude: try: sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6) except Exception: - logging.debug("Invalid lat/lon received for reference") + logging.warning("Invalid lat/lon received for DME reference") + else: + logging.warning("DME database did not contain data for ref %s", ref_id) + + except ConnectionError: + logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id) + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when looking up sig_ref info for " + sig + " ref " + ref_id) except Exception: - logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id, exc_info=True) + logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True) return sig_ref diff --git a/core/utils.py b/core/utils.py index d8c6c6d..9c99210 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1,8 +1,11 @@ -def serialize_everything(obj): - """Convert objects to serialisable things. Used by JSON serialiser as a default when it encounters unserializable things. - Just converts objects to dict. Try to avoid doing anything clever here when serialising spots, because we also need - to receive spots without complex handling.""" - return obj.__dict__ +import simplejson + + +def safe_json_dumps(obj): + """Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats + which are invalid in JSON.""" + + return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__) def empty_queue(q): diff --git a/data/spot.py b/data/spot.py index 92c3cf0..56f2e78 100644 --- a/data/spot.py +++ b/data/spot.py @@ -133,6 +133,17 @@ class Spot: # The ID the source gave it, if any. source_id: str | None = None + def __post_init__(self): + """Normalise fields that don't survive a plain dict to Spot conversion. This is used in the "add spot" API + endpoint where the client is submitting JSON, and we want to recreate a full Spot object, including nested + objects such as the sig_refs list..""" + + if self.sig_refs: + self.sig_refs = [ + sig_ref if isinstance(sig_ref, SIGRef) else SIGRef(**sig_ref) + for sig_ref in self.sig_refs + ] + def infer_missing(self, credentials=None): """Infer missing parameters where possible""" @@ -312,7 +323,7 @@ class Spot: self.propagation_mode = PROPAGATION_MODES[mode_tag] else: self.propagation_mode = mode_tag - logging.info("Seen a new propagation mode tag not yet in the system: {}", mode_tag) + logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag) # DX Grid to lat/lon and vice versa in case one is missing if self.dx_grid and not self.dx_latitude: diff --git a/requirements.txt b/requirements.txt index 293f6ec..163a941 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,5 @@ beautifulsoup4~=4.14.2 websocket-client~=1.8.0 tornado~=6.4.2 tornado_eventsource~=3.0.0 -geopandas~=0.13.2 \ No newline at end of file +geopandas~=0.13.2 +simplejson~=4.1.1 \ No newline at end of file diff --git a/server/handlers/api/addspot.py b/server/handlers/api/addspot.py index 79ec8d5..adedbf0 100644 --- a/server/handlers/api/addspot.py +++ b/server/handlers/api/addspot.py @@ -1,4 +1,3 @@ -import json import logging import re import threading @@ -16,8 +15,7 @@ from core.constants import UNKNOWN_BAND from core.lookup_helper import infer_band_from_freq from core.prometheus_metrics_handler import api_requests_counter from core.sig_utils import get_ref_regex_for_sig -from core.utils import serialize_everything -from data.sig_ref import SIGRef +from core.utils import safe_json_dumps from data.spot import Spot from spotproviders.spot_provider import SpotProvider @@ -49,8 +47,7 @@ class APISpotHandler(tornado.web.RequestHandler): # Reject if not allowed if not ALLOW_SPOTTING: self.set_status(401) - self.write(json.dumps("Error - this server does not allow new spots to be added via the API.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - this server does not allow new spots to be added via the API.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -58,8 +55,7 @@ class APISpotHandler(tornado.web.RequestHandler): # Reject if format not json if not self.request.headers.get('Content-Type', '').startswith("application/json"): self.set_status(415) - self.write( - json.dumps("Error - request Content-Type must be application/json", default=serialize_everything)) + self.write(safe_json_dumps("Error - request Content-Type must be application/json")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -68,7 +64,7 @@ class APISpotHandler(tornado.web.RequestHandler): post_data = self.request.body if not post_data: self.set_status(422) - self.write(json.dumps("Error - request body is empty", default=serialize_everything)) + self.write(safe_json_dumps("Error - request body is empty")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -106,20 +102,10 @@ class APISpotHandler(tornado.web.RequestHandler): # Convert spot field to a Spot object spot = Spot(**spot_data) - # Converting to a spot object this way won't have coped with sig_ref objects, so fix that. (Would be nice to - # redo this in a functional style) - if spot.sig and spot.sig_refs: - real_sig_refs = [] - for dict_obj in spot.sig_refs: - dict_obj = {**dict_obj, "sig": spot.sig} - real_sig_refs.append(json.loads(json.dumps(dict_obj), object_hook=lambda d: SIGRef(**d))) - spot.sig_refs = real_sig_refs - # Reject if no timestamp, frequency, dx_call or de_call if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call: self.set_status(422) - self.write(json.dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -127,15 +113,13 @@ class APISpotHandler(tornado.web.RequestHandler): # Reject invalid-looking callsigns if not re.match(r"^[A-Za-z0-9/\-]*$", spot.dx_call): self.set_status(422) - self.write(json.dumps("Error - '" + spot.dx_call + "' does not look like a valid callsign.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - '" + spot.dx_call + "' does not look like a valid callsign.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return if not re.match(r"^[A-Za-z0-9/\-]*$", spot.de_call): self.set_status(422) - self.write(json.dumps("Error - '" + spot.de_call + "' does not look like a valid callsign.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - '" + spot.de_call + "' does not look like a valid callsign.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -143,8 +127,7 @@ class APISpotHandler(tornado.web.RequestHandler): # Reject if frequency not in a known band if infer_band_from_freq(spot.freq) == UNKNOWN_BAND: self.set_status(422) - self.write(json.dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -154,8 +137,7 @@ class APISpotHandler(tornado.web.RequestHandler): r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$", spot.dx_grid.upper()): self.set_status(422) - self.write(json.dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -164,9 +146,8 @@ class APISpotHandler(tornado.web.RequestHandler): if spot.sig and spot.sig_refs and len(spot.sig_refs) > 0 and spot.sig_refs[0].id and get_ref_regex_for_sig( spot.sig) and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id): self.set_status(422) - self.write(json.dumps( - "Error - '" + spot.sig_refs[0].id + "' does not look like a valid reference for " + spot.sig + ".", - default=serialize_everything)) + self.write(safe_json_dumps( + "Error - '" + spot.sig_refs[0].id + "' does not look like a valid reference for " + spot.sig + ".")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -241,14 +222,14 @@ class APISpotHandler(tornado.web.RequestHandler): self.write(json.dumps("Warning - " + upstream_warning, default=serialize_everything)) self.set_status(201) else: - self.write(json.dumps("OK", default=serialize_everything)) + self.write(safe_json_dumps("OK")) self.set_status(201) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") except Exception as e: - logging.error(e) - self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything)) + logging.error("Exception when handling client request to add spot API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") diff --git a/server/handlers/api/alerts.py b/server/handlers/api/alerts.py index 10083e9..d2b14e8 100644 --- a/server/handlers/api/alerts.py +++ b/server/handlers/api/alerts.py @@ -1,5 +1,4 @@ import copy -import json import logging from datetime import datetime from queue import Queue @@ -12,7 +11,7 @@ from tornado import httputil from tornado.web import Application from core.prometheus_metrics_handler import api_requests_counter -from core.utils import serialize_everything, empty_queue +from core.utils import safe_json_dumps, empty_queue from data.lookup_credentials import extract_credentials SSE_HANDLER_MAX_QUEUE_SIZE = 100 @@ -57,15 +56,14 @@ class APIAlertsHandler(tornado.web.RequestHandler): data = get_alert_list_with_filters(self._alerts, query_params) if credentials: data = self._enrich(data, credentials) - self.write(json.dumps(data, default=serialize_everything)) + self.write(safe_json_dumps(data)) self.set_status(200) except ValueError as e: - logging.error(e) - self.write(json.dumps("Bad request - " + str(e), default=serialize_everything)) + self.write(safe_json_dumps("Bad request - " + str(e))) self.set_status(400) except Exception as e: - logging.error(e) - self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything)) + logging.error("Exception when handling client request to alerts API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") @@ -150,7 +148,7 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler): if self._credentials: alert = copy.deepcopy(alert) alert.infer_missing(self._credentials) - self.write_message(msg=json.dumps(alert, default=serialize_everything)) + self.write_message(msg=safe_json_dumps(alert)) else: # Send a keepalive comment if the queue was empty diff --git a/server/handlers/api/dxstats.py b/server/handlers/api/dxstats.py index be43eda..26bd96d 100644 --- a/server/handlers/api/dxstats.py +++ b/server/handlers/api/dxstats.py @@ -1,4 +1,5 @@ import json +import logging from collections import Counter from datetime import datetime, timedelta from typing import Any @@ -9,6 +10,7 @@ from tornado import httputil from tornado.web import Application from core.prometheus_metrics_handler import api_requests_counter +from core.utils import safe_json_dumps CONTINENTS = ["EU", "NA", "SA", "AS", "AF", "OC", "AN"] BANDS = ["160m", "80m", "60m", "40m", "30m", "20m", "17m", "15m", "12m", "10m", "6m"] @@ -29,29 +31,35 @@ class APIDxStatsHandler(tornado.web.RequestHandler): self._web_server_metrics = web_server_metrics def get(self): - self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) - self._web_server_metrics["api_access_counter"] += 1 - self._web_server_metrics["status"] = "OK" - api_requests_counter.inc() + try: + self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) + self._web_server_metrics["api_access_counter"] += 1 + self._web_server_metrics["status"] = "OK" + api_requests_counter.inc() - one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp() - counts = Counter() + one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp() + counts = Counter() - for key in self._spots.iterkeys(): - spot = self._spots.get(key) - if spot is None: - continue - if not spot.time or spot.time < one_hour_ago: - continue - if spot.de_continent in CONTINENTS_SET and spot.dx_continent in CONTINENTS_SET and spot.band in BANDS_SET: - counts[spot.de_continent, spot.dx_continent, spot.band] += 1 + for key in self._spots.iterkeys(): + spot = self._spots.get(key) + if spot is None: + continue + if not spot.time or spot.time < one_hour_ago: + continue + if spot.de_continent in CONTINENTS_SET and spot.dx_continent in CONTINENTS_SET and spot.band in BANDS_SET: + counts[spot.de_continent, spot.dx_continent, spot.band] += 1 - result = { - de: {dx: {band: counts[de, dx, band] for band in BANDS} for dx in CONTINENTS} - for de in CONTINENTS - } + result = { + de: {dx: {band: counts[de, dx, band] for band in BANDS} for dx in CONTINENTS} + for de in CONTINENTS + } - self.write(json.dumps(result)) - self.set_status(200) - self.set_header("Cache-Control", "no-store") - self.set_header("Content-Type", "application/json") + self.write(json.dumps(result)) + self.set_status(200) + self.set_header("Cache-Control", "no-store") + self.set_header("Content-Type", "application/json") + + except Exception as e: + logging.error("Exception when handling client request to dx stats API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) + self.set_status(500) diff --git a/server/handlers/api/lookups.py b/server/handlers/api/lookups.py index fcaadd9..ee75950 100644 --- a/server/handlers/api/lookups.py +++ b/server/handlers/api/lookups.py @@ -1,4 +1,3 @@ -import json import logging import re from datetime import datetime @@ -13,7 +12,7 @@ from core.constants import SIGS from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone from core.prometheus_metrics_handler import api_requests_counter from core.sig_utils import get_ref_regex_for_sig, populate_sig_ref_info -from core.utils import serialize_everything +from core.utils import safe_json_dumps from data.lookup_credentials import extract_credentials from data.sig_ref import SIGRef from data.spot import Spot @@ -65,19 +64,18 @@ class APILookupCallHandler(tornado.web.RequestHandler): "longitude": fake_spot.dx_longitude, "location_source": fake_spot.dx_location_source } - self.write(json.dumps(data, default=serialize_everything)) + self.write(safe_json_dumps(data)) else: - self.write(json.dumps("Error - '" + call + "' does not look like a valid callsign.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - '" + call + "' does not look like a valid callsign.")) self.set_status(422) else: - self.write(json.dumps("Error - call must be provided", default=serialize_everything)) + self.write(safe_json_dumps("Error - call must be provided")) self.set_status(422) except Exception as e: - logging.error(e) - self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything)) + logging.error("Exception when handling client request to call lookup API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) self.set_header("Cache-Control", "no-store") @@ -114,24 +112,22 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler): if sig in list(map(lambda p: p.name.upper(), SIGS)): if not get_ref_regex_for_sig(sig) or re.match(get_ref_regex_for_sig(sig), ref_id): data = populate_sig_ref_info(SIGRef(id=ref_id, sig=sig)) - self.write(json.dumps(data, default=serialize_everything)) + self.write(safe_json_dumps(data)) else: - self.write( - json.dumps( - "Error - '" + ref_id + "' does not look like a valid reference ID for " + sig + ".", - default=serialize_everything)) + self.write(safe_json_dumps( + "Error - '" + ref_id + "' does not look like a valid reference ID for " + sig + ".")) self.set_status(422) else: - self.write(json.dumps("Error - sig '" + sig + "' is not known.", default=serialize_everything)) + self.write(safe_json_dumps("Error - sig '" + sig + "' is not known.")) self.set_status(422) else: - self.write(json.dumps("Error - sig and id must be provided", default=serialize_everything)) + self.write(safe_json_dumps("Error - sig and id must be provided")) self.set_status(422) except Exception as e: - logging.error(e) - self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything)) + logging.error("Exception when handling client request to sig ref lookup API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) self.set_header("Cache-Control", "no-store") @@ -185,15 +181,15 @@ class APILookupGridHandler(tornado.web.RequestHandler): "latitude": lat + lat_cell_size, "longitude": lon + lon_cell_size, }} - self.write(json.dumps(response, default=serialize_everything)) + self.write(safe_json_dumps(response)) else: - self.write(json.dumps("Error - grid must be provided", default=serialize_everything)) + self.write(safe_json_dumps("Error - grid must be provided")) self.set_status(422) except Exception as e: - logging.error(e) - self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything)) + logging.error("Exception when handling client request to grid ref lookup API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) self.set_header("Cache-Control", "no-store") diff --git a/server/handlers/api/options.py b/server/handlers/api/options.py index 3bdbd02..93b5838 100644 --- a/server/handlers/api/options.py +++ b/server/handlers/api/options.py @@ -1,4 +1,4 @@ -import json +import logging from datetime import datetime from typing import Any @@ -10,7 +10,7 @@ from tornado.web import Application from core.config import MAX_SPOT_AGE, ALLOW_SPOTTING from core.constants import BANDS, ALL_MODES, MODE_TYPES, SIGS, CONTINENTS, PROPAGATION_MODES from core.prometheus_metrics_handler import api_requests_counter -from core.utils import serialize_everything +from core.utils import safe_json_dumps class APIOptionsHandler(tornado.web.RequestHandler): @@ -28,19 +28,20 @@ class APIOptionsHandler(tornado.web.RequestHandler): self._spot_providers = spot_providers or [] def get(self): - # Metrics - self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) - self._web_server_metrics["api_access_counter"] += 1 - self._web_server_metrics["status"] = "OK" - api_requests_counter.inc() + try: + # Metrics + self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) + self._web_server_metrics["api_access_counter"] += 1 + self._web_server_metrics["status"] = "OK" + api_requests_counter.inc() - # Build a map of SIG name -> list of provider names that can submit spots for that SIG + # Build a map of SIG name -> list of provider names that can submit spots for that SIG spot_submit_providers = {} for provider in self._spot_providers: if not provider.enabled: continue for sig in SIGS: - if provider.can_submit_spot(sig.name): + if provider.can_submit_spot(sig.name): spot_submit_providers.setdefault(sig.name, []).append(provider.name) # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle @@ -51,8 +52,8 @@ class APIOptionsHandler(tornado.web.RequestHandler): map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"]))) # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from # one of our providers. - if ALLOW_SPOTTING: - spot_sources.append("API") + if ALLOW_SPOTTING: + spot_sources.append("API") options = {"bands": BANDS, "modes": ALL_MODES, @@ -66,7 +67,12 @@ class APIOptionsHandler(tornado.web.RequestHandler): "spot_allowed": ALLOW_SPOTTING, "spot_submit_providers": spot_submit_providers} - self.write(json.dumps(options, default=serialize_everything)) - self.set_status(200) - self.set_header("Cache-Control", "no-store") - self.set_header("Content-Type", "application/json") + self.write(safe_json_dumps(options)) + self.set_status(200) + self.set_header("Cache-Control", "no-store") + self.set_header("Content-Type", "application/json") + + except Exception as e: + logging.error("Exception when handling client request to options API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) + self.set_status(500) diff --git a/server/handlers/api/solar_conditions.py b/server/handlers/api/solar_conditions.py index 5b2f49d..1efe105 100644 --- a/server/handlers/api/solar_conditions.py +++ b/server/handlers/api/solar_conditions.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime from typing import Any @@ -7,6 +8,7 @@ from tornado import httputil from tornado.web import Application from core.prometheus_metrics_handler import api_requests_counter +from core.utils import safe_json_dumps class APISolarConditionsHandler(tornado.web.RequestHandler): @@ -22,13 +24,19 @@ class APISolarConditionsHandler(tornado.web.RequestHandler): self._web_server_metrics = web_server_metrics def get(self): - # Metrics - self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) - self._web_server_metrics["api_access_counter"] += 1 - self._web_server_metrics["status"] = "OK" - api_requests_counter.inc() + try: + # Metrics + self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) + self._web_server_metrics["api_access_counter"] += 1 + self._web_server_metrics["status"] = "OK" + api_requests_counter.inc() - self.write(self._solar_conditions.to_json()) - self.set_status(200) - self.set_header("Cache-Control", "no-store") - self.set_header("Content-Type", "application/json") + self.write(self._solar_conditions.to_json()) + self.set_status(200) + self.set_header("Cache-Control", "no-store") + self.set_header("Content-Type", "application/json") + + except Exception as e: + logging.error("Exception when handling client request to solar conditions API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) + self.set_status(500) diff --git a/server/handlers/api/spots.py b/server/handlers/api/spots.py index 6882265..f50f3fe 100644 --- a/server/handlers/api/spots.py +++ b/server/handlers/api/spots.py @@ -1,5 +1,4 @@ import copy -import json import logging from datetime import datetime, timedelta from queue import Queue @@ -12,7 +11,7 @@ from tornado import httputil from tornado.web import Application from core.prometheus_metrics_handler import api_requests_counter -from core.utils import serialize_everything, empty_queue +from core.utils import safe_json_dumps, empty_queue from data.lookup_credentials import extract_credentials SSE_HANDLER_MAX_QUEUE_SIZE = 1000 @@ -57,15 +56,14 @@ class APISpotsHandler(tornado.web.RequestHandler): data = get_spot_list_with_filters(self._spots, query_params) if credentials: data = self._enrich(data, credentials) - self.write(json.dumps(data, default=serialize_everything)) + self.write(safe_json_dumps(data)) self.set_status(200) except ValueError as e: - logging.error(e) - self.write(json.dumps("Bad request - " + str(e), default=serialize_everything)) + self.write(safe_json_dumps("Bad request - " + str(e))) self.set_status(400) except Exception as e: - logging.error(e) - self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything)) + logging.error("Excedption when handling client request to spots API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) self.set_status(500) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") @@ -152,7 +150,7 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler): if self._credentials: spot = copy.deepcopy(spot) spot.infer_missing(self._credentials) - self.write_message(msg=json.dumps(spot, default=serialize_everything)) + self.write_message(msg=safe_json_dumps(spot)) else: # Send a keepalive comment if the queue was empty diff --git a/server/handlers/api/status.py b/server/handlers/api/status.py index 3594cdf..5ee6f1d 100644 --- a/server/handlers/api/status.py +++ b/server/handlers/api/status.py @@ -1,4 +1,4 @@ -import json +import logging from datetime import datetime from typing import Any @@ -8,7 +8,7 @@ from tornado import httputil from tornado.web import Application from core.prometheus_metrics_handler import api_requests_counter -from core.utils import serialize_everything +from core.utils import safe_json_dumps class APIStatusHandler(tornado.web.RequestHandler): @@ -24,13 +24,19 @@ class APIStatusHandler(tornado.web.RequestHandler): self._web_server_metrics = web_server_metrics def get(self): - # Metrics - self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) - self._web_server_metrics["api_access_counter"] += 1 - self._web_server_metrics["status"] = "OK" - api_requests_counter.inc() + try: + # Metrics + self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC) + self._web_server_metrics["api_access_counter"] += 1 + self._web_server_metrics["status"] = "OK" + api_requests_counter.inc() - self.write(json.dumps(self._status_data, default=serialize_everything)) - self.set_status(200) - self.set_header("Cache-Control", "no-store") - self.set_header("Content-Type", "application/json") + self.write(safe_json_dumps(self._status_data)) + self.set_status(200) + self.set_header("Cache-Control", "no-store") + self.set_header("Content-Type", "application/json") + + except Exception as e: + logging.error("Exception when handling client request to status API: %s", e, exc_info=True) + self.write(safe_json_dumps("Error - an internal server error occurred.")) + self.set_status(500) diff --git a/server/handlers/manifesthandler.py b/server/handlers/manifesthandler.py new file mode 100644 index 0000000..978b090 --- /dev/null +++ b/server/handlers/manifesthandler.py @@ -0,0 +1,11 @@ +import tornado.web + +from core.config import BASE_URL + + +class ManifestHandler(tornado.web.RequestHandler): + """Handler for manifest.webmanifest, which needs BASE_URL inserted and a custom content-type""" + + def get(self): + self.set_header("Content-Type", "application/manifest+json; charset=UTF-8") + self.render("manifest.webmanifest", baseurl=BASE_URL) diff --git a/server/webserver.py b/server/webserver.py index eb64bc3..d56f404 100644 --- a/server/webserver.py +++ b/server/webserver.py @@ -5,7 +5,7 @@ import os import tornado from tornado.web import StaticFileHandler -from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE +from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL from core.utils import empty_queue from server.handlers.api.addspot import APISpotHandler from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler @@ -16,6 +16,7 @@ from server.handlers.api.options import APIOptionsHandler from server.handlers.api.solar_conditions import APISolarConditionsHandler from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler from server.handlers.api.status import APIStatusHandler +from server.handlers.manifesthandler import ManifestHandler from server.handlers.metrics import PrometheusMetricsHandler from server.handlers.pagetemplate import PageTemplateHandler @@ -110,11 +111,16 @@ class WebServer: if ALLOW_SPOTTING: ui_routes += [(r"/add-spot", PageTemplateHandler, {"template_name": "add_spot", **handler_opts})] - # API docs, Prometheus metrics, and finally static assets are always available regardless of API-only mode. + # API docs, Prometheus metrics, webapp manifest and static assets are always available regardless of API-only + # mode. misc_routes = [ (r"/apidocs", PageTemplateHandler, {"template_name": "apidocs", **handler_opts}), (r"/metrics", PrometheusMetricsHandler), - (r"/(.*)", StaticFileHandler, {"path": os.path.join(_HERE, "../webassets")}) + (r"/manifest.webmanifest", ManifestHandler), + # If e.g. nginx is configured as a reverse proxy with a hard-coded path to static files, as per the README, + # this will never have to handle anything, but having it here allows Spothole to work without nginx for + # testing. + (r"/static/(.*)", StaticFileHandler, {"path": os.path.join(_HERE, "../static")}) ] app = tornado.web.Application(api_routes + v1_compat_routes + ui_routes + misc_routes, @@ -123,6 +129,7 @@ class WebServer: debug=False) app.listen(self._port, xheaders=True) logging.info("Web server running on port " + str(WEB_SERVER_PORT)) + logging.info("You can access your copy of Spothole at " + BASE_URL) await self._shutdown_event.wait() def notify_new_spot(self, spot): @@ -176,20 +183,22 @@ class WebServer: pass def request_log(handler): - """Custom log function to provide more data about requests.""" + """Custom log function to provide more data about requests when enabled, and to provide the ability to turn off + web request logging altogetether.""" - if handler.get_status() < 500: - log_method = logging.info - else: - log_method = logging.warning + if LOG_WEB_REQUESTS: + if handler.get_status() < 500: + log_method = logging.info + else: + log_method = logging.warning - request = handler.request - client_ip = request.remote_ip - referrer = request.headers.get("Referer", "-") - user_agent = request.headers.get("User-Agent", "-") + request = handler.request + client_ip = request.remote_ip + referrer = request.headers.get("Referer", "-") + user_agent = request.headers.get("User-Agent", "-") - log_method( - f'{client_ip} - "{request.method} {request.uri}" ' - f'{handler.get_status()} {request.request_time():.2f}ms | ' - f'Ref: {referrer} | UA: {user_agent}' - ) + log_method( + f'{client_ip} - "{request.method} {request.uri}" ' + f'{handler.get_status()} {request.request_time():.2f}ms | ' + f'Ref: {referrer} | UA: {user_agent}' + ) diff --git a/solarconditionsproviders/giroionosonde.py b/solarconditionsproviders/giroionosonde.py index 8a9e398..1c5795f 100644 --- a/solarconditionsproviders/giroionosonde.py +++ b/solarconditionsproviders/giroionosonde.py @@ -5,6 +5,7 @@ from threading import Thread, Event import pytz import requests +from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from core.constants import HTTP_HEADERS from solarconditionsproviders.ionosonde_utils import compute_band_states @@ -127,10 +128,18 @@ class GIROIonosonde(SolarConditionsProvider): from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S") to_str = to_time.strftime("%Y.%m.%d+%H:%M:%S") url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUFD,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}" - response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15)) - if response.status_code != 200: + try: + http_response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15)) + if not http_response.ok: + logging.warning(f"HTTP {http_response.status_code} when calling Giro ionosonde API.") + return None, None, None + return self._parse_all(http_response.text) + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when accessing Giro ionosonde API.") + return None, None, None + except ConnectionError: + logging.warning("Connection error when accessing Giro ionosonde API.") return None, None, None - return self._parse_all(response.text) @staticmethod def _parse_all(text): diff --git a/solarconditionsproviders/hamqsl.py b/solarconditionsproviders/hamqsl.py index 62cfd70..92c5306 100644 --- a/solarconditionsproviders/hamqsl.py +++ b/solarconditionsproviders/hamqsl.py @@ -18,10 +18,6 @@ class HamQSL(HTTPSolarConditionsProvider): super().__init__(provider_config, URL, POLL_INTERVAL) def _http_response_to_solar_conditions(self, http_response): - if http_response.status_code != 200: - logging.warning("HamQSL solar conditions API returned HTTP " + str(http_response.status_code)) - return None - root = ElementTree.fromstring(http_response.text) sd = root.find("solardata") if sd is None: diff --git a/solarconditionsproviders/http_solar_conditions_provider.py b/solarconditionsproviders/http_solar_conditions_provider.py index 0a8dfcd..0f7903e 100644 --- a/solarconditionsproviders/http_solar_conditions_provider.py +++ b/solarconditionsproviders/http_solar_conditions_provider.py @@ -4,6 +4,7 @@ from threading import Thread, Event import pytz import requests +from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from core.constants import HTTP_HEADERS from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider @@ -39,13 +40,22 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider): try: logging.debug("Polling " + self.name + " solar conditions API...") http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30)) - new_data = self._http_response_to_solar_conditions(http_response) - self.update_data(new_data) + # Check response code was good + if http_response.ok: + new_data = self._http_response_to_solar_conditions(http_response) + self.update_data(new_data) - self.status = "OK" - self.last_update_time = datetime.now(pytz.UTC) - logging.debug("Received data from " + self.name + " solar conditions API.") + self.status = "OK" + self.last_update_time = datetime.now(pytz.UTC) + logging.debug("Received data from " + self.name + " solar conditions API.") + else: + self.status = "Error" + logging.warning(f"HTTP {http_response.status_code} when calling {self.name} solar conditions API.") + except ConnectionError: + logging.warning(f"Connection error when accessing {self.name} solar conditions API.") + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when accessing {self.name} solar conditions API.") except Exception: self.status = "Error" logging.exception("Exception in HTTP Solar Conditions Provider (" + self.name + ")") diff --git a/solarconditionsproviders/kc2gprop.py b/solarconditionsproviders/kc2gprop.py index 41342b1..16af204 100644 --- a/solarconditionsproviders/kc2gprop.py +++ b/solarconditionsproviders/kc2gprop.py @@ -4,6 +4,7 @@ from threading import Thread, Event import pytz import requests +from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from core.constants import HTTP_HEADERS from solarconditionsproviders.ionosonde_utils import compute_band_states @@ -44,9 +45,9 @@ class KC2GProp(SolarConditionsProvider): def _poll(self): try: logging.debug("Polling KC2G ionosonde data...") - response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30)) - if response.status_code != 200: - logging.warning(f"KC2G ionosonde API returned HTTP {response.status_code}") + http_response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30)) + if not http_response.ok: + logging.warning(f"HTTP {http_response.status_code} when calling KG2G ionosonde API.") return now = datetime.now(timezone.utc) @@ -57,7 +58,7 @@ class KC2GProp(SolarConditionsProvider): ionosonde_data = dict(self._solar_conditions.ionosonde_data or {}) updated_count = 0 - for reading in response.json(): + for reading in http_response.json(): station = reading.get("station", {}) ursi = station.get("code") name = station.get("name") @@ -115,6 +116,10 @@ class KC2GProp(SolarConditionsProvider): self.last_update_time = datetime.now(pytz.UTC) logging.debug(f"Updated KC2G ionosonde data for {updated_count} stations.") + except ConnectionError: + logging.warning("Connection error when accessing KC2G ionosonde API.") + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when accessing KC2G ionosonde API.") except Exception: self.status = "Error" logging.exception("Exception in KC2G ionosonde data provider") diff --git a/solarconditionsproviders/noaa3dayforecast.py b/solarconditionsproviders/noaa3dayforecast.py index d677e66..de5bf1a 100644 --- a/solarconditionsproviders/noaa3dayforecast.py +++ b/solarconditionsproviders/noaa3dayforecast.py @@ -80,10 +80,6 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider): return result if result else None def _http_response_to_solar_conditions(self, http_response): - if http_response.status_code != 200: - logging.warning("NOAA K-index forecast API returned HTTP " + str(http_response.status_code)) - return None - lines = http_response.text.splitlines() # Find the "NOAA Kp index breakdown" section header diff --git a/spothole.py b/spothole.py index 300cd8f..f14abde 100644 --- a/spothole.py +++ b/spothole.py @@ -8,7 +8,7 @@ import sys from diskcache import Cache from core.cleanup import CleanupTimer -from core.config import config, SERVER_OWNER_CALLSIGN +from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL from core.constants import SOFTWARE_VERSION from core.lookup_helper import lookup_helper from core.status_reporter import StatusReporter @@ -84,9 +84,9 @@ def get_solar_conditions_provider_from_config(config_providers_entry): if __name__ == '__main__': # Set up logging root = logging.getLogger() - root.setLevel(logging.INFO) + root.setLevel(LOG_LEVEL) handler = logging.StreamHandler(sys.stdout) - handler.setLevel(logging.INFO) + handler.setLevel(LOG_LEVEL) formatter = logging.Formatter("%(levelname)s : %(message)s") handler.setFormatter(formatter) root.handlers.clear() diff --git a/spotproviders/gma.py b/spotproviders/gma.py index a5d3221..daaf888 100644 --- a/spotproviders/gma.py +++ b/spotproviders/gma.py @@ -14,9 +14,9 @@ class GMA(HTTPSpotProvider): """Spot provider for General Mountain Activity""" POLL_INTERVAL_SEC = 120 - SPOTS_URL = "https://www.cqgma.org/api/spots/25/" + SPOTS_URL = "https://www.gma.rocks/api/spots/25/" # GMA spots don't contain the details of the programme they are for, we need a separate lookup for that - REF_INFO_URL_ROOT = "https://www.cqgma.org/api/ref/?" + REF_INFO_URL_ROOT = "https://www.gma.rocks/api/ref/?" def __init__(self, provider_config): # Ensure there is an API key in our config, and set up the query URL using it. If no key is provided, @@ -39,7 +39,7 @@ class GMA(HTTPSpotProvider): de_call=source_spot["SPOTTER"].upper(), # Seen GMA spots with no frequency or with "QRT" in this field freq=float(source_spot["QRG"]) * 1000 if ( - source_spot["QRG"] != "" and source_spot["QRG"] != "QRT") else None, + source_spot["QRG"] != "" and source_spot["QRG"] != "QRT") else None, # Filter out some weird mode strings mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None, comment=source_spot["TEXT"], @@ -58,8 +58,8 @@ class GMA(HTTPSpotProvider): try: ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"], headers=HTTP_HEADERS) - # Sometimes this is blank, so handle that - if ref_response.text is not None and ref_response.text != "": + # Sometimes this is blank even if it's a 200 response, so handle that + if ref_response.ok and ref_response.text is not None and ref_response.text != "": ref_info = ref_response.json() # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA @@ -94,9 +94,17 @@ class GMA(HTTPSpotProvider): spot.sig_refs[0].sig = ref_info["reftype"] spot.sig = ref_info["reftype"] - # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do - # that for us. + # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; + # other code will do that for us. new_spots.append(spot) + + elif not ref_response.from_cache: + if not ref_response.ok: + logging.warning( + f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot["REF"]}") + else: + logging.warning( + f"GMA API returned a malformed response when looking up ref {source_spot["REF"]}") except: logging.warning("Exception when looking up " + self.REF_INFO_URL_ROOT + source_spot[ "REF"] + ", ignoring this spot for now") diff --git a/spotproviders/hema.py b/spotproviders/hema.py index 8c795a4..a2a833f 100644 --- a/spotproviders/hema.py +++ b/spotproviders/hema.py @@ -1,8 +1,10 @@ +import logging import re from datetime import datetime import pytz import requests +from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from core.constants import HTTP_HEADERS from data.sig_ref import SIGRef @@ -35,36 +37,41 @@ class HEMA(HTTPSpotProvider): new_spots = [] # OK, if the spot seed actually changed, now we make the real request for data. if spot_seed_changed: - source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30)) - source_data_items = source_data.text.split("=") - # Iterate through source data items. - for source_spot in source_data_items: - spot_items = source_spot.split(";") - # Any line with less than 9 items is not a proper spot line - if len(spot_items) >= 9: - # Fiddle with some data to extract bits we need. Freq/mode and spotter/comment come in combined fields. - freq_mode_match = re.search(self.FREQ_MODE_PATTERN, spot_items[5]) - spotter_comment_match = re.search(self.SPOTTER_COMMENT_PATTERN, spot_items[6]) - if not freq_mode_match or not spotter_comment_match: - continue + try: + source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30)) + source_data_items = source_data.text.split("=") + # Iterate through source data items. + for source_spot in source_data_items: + spot_items = source_spot.split(";") + # Any line with less than 9 items is not a proper spot line + if len(spot_items) >= 9: + # Fiddle with some data to extract bits we need. Freq/mode and spotter/comment come in combined fields. + freq_mode_match = re.search(self.FREQ_MODE_PATTERN, spot_items[5]) + spotter_comment_match = re.search(self.SPOTTER_COMMENT_PATTERN, spot_items[6]) + if not freq_mode_match or not spotter_comment_match: + continue - # Convert to our spot format - spot = Spot(source=self.name, - dx_call=spot_items[2].upper(), - de_call=spotter_comment_match.group(1).upper(), - freq=float(freq_mode_match.group(1)) * 1000000, - mode=freq_mode_match.group(2).upper(), - comment=spotter_comment_match.group(2), - sig="HEMA", - sig_refs=[SIGRef(id=spot_items[3].upper(), sig="HEMA", name=spot_items[4])], - time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace( - tzinfo=pytz.UTC).timestamp(), - dx_latitude=float(spot_items[7]), - dx_longitude=float(spot_items[8])) + # Convert to our spot format + spot = Spot(source=self.name, + dx_call=spot_items[2].upper(), + de_call=spotter_comment_match.group(1).upper(), + freq=float(freq_mode_match.group(1)) * 1000000, + mode=freq_mode_match.group(2).upper(), + comment=spotter_comment_match.group(2), + sig="HEMA", + sig_refs=[SIGRef(id=spot_items[3].upper(), sig="HEMA", name=spot_items[4])], + time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace( + tzinfo=pytz.UTC).timestamp(), + dx_latitude=float(spot_items[7]), + dx_longitude=float(spot_items[8])) - # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do - # that for us. - new_spots.append(spot) + # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other + # code will do that for us. + new_spots.append(spot) + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when accessing HEMA spots API.") + except ConnectionError: + logging.warning("Connection error when accessing HEMA spots API.") return new_spots def can_submit_spot(self, sig): diff --git a/spotproviders/http_spot_provider.py b/spotproviders/http_spot_provider.py index 335cfde..912d8e4 100644 --- a/spotproviders/http_spot_provider.py +++ b/spotproviders/http_spot_provider.py @@ -4,6 +4,8 @@ from threading import Thread, Event import pytz import requests +from requests import ReadTimeout +from requests.exceptions import ConnectionError, ConnectTimeout from core.constants import HTTP_HEADERS from spotproviders.spot_provider import SpotProvider @@ -50,16 +52,25 @@ class HTTPSpotProvider(SpotProvider): # Request data from API logging.debug("Polling " + self.name + " spot API...") http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30)) - # Pass off to the subclass for processing - new_spots = self._http_response_to_spots(http_response) - # Submit the new spots for processing. There might not be any spots for the less popular programs. - if new_spots: - self._submit_batch(new_spots) + # Check response code was good + if http_response.ok: + # Pass off to the subclass for processing + new_spots = self._http_response_to_spots(http_response) + # Submit the new spots for processing. There might not be any spots for the less popular programs. + if new_spots: + self._submit_batch(new_spots) - self.status = "OK" - self.last_update_time = datetime.now(pytz.UTC) - logging.debug("Received data from " + self.name + " spot API.") + self.status = "OK" + self.last_update_time = datetime.now(pytz.UTC) + logging.debug("Received data from " + self.name + " spot API.") + else: + self.status = "Error" + logging.warning(f"HTTP {http_response.status_code} when calling {self.name} spot API.") + except ConnectionError: + logging.warning(f"Connection error when accessing {self.name} spots API.") + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when accessing {self.name} spots API.") except Exception: self.status = "Error" logging.exception("Exception in HTTP JSON Spot Provider (" + self.name + ")") diff --git a/spotproviders/parksnpeaks.py b/spotproviders/parksnpeaks.py index 79578d8..890d4d6 100644 --- a/spotproviders/parksnpeaks.py +++ b/spotproviders/parksnpeaks.py @@ -34,8 +34,8 @@ class ParksNPeaks(HTTPSpotProvider): dx_call=source_spot["actCallsign"].upper(), de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None, # typo exists in API - freq=float(source_spot["actFreq"].replace(",", "")) * 1000000 if ( - source_spot["actFreq"] != "") else None, + freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "") + .replace("+/-", "").strip()) * 1000000 if (source_spot["actFreq"] != "") else None, # Seen PNP spots with empty frequency, and with comma-separated thousands digits mode=source_spot["actMode"].upper(), comment=source_spot["actComments"], diff --git a/spotproviders/sota.py b/spotproviders/sota.py index e6c6aa4..cfb4b1c 100644 --- a/spotproviders/sota.py +++ b/spotproviders/sota.py @@ -1,6 +1,8 @@ +import logging from datetime import datetime import requests +from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout from core.constants import HTTP_HEADERS, SSB_SUB_MODES, DV_SUB_MODES from data.sig_ref import SIGRef @@ -36,28 +38,33 @@ class SOTA(HTTPSpotProvider): new_spots = [] # OK, if the epoch actually changed, now we make the real request for data. if epoch_changed: - source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30)).json() - # Iterate through source data - for source_spot in source_data: - # Convert to our spot format - spot = Spot(source=self.name, - source_id=source_spot["id"], - dx_call=source_spot["activatorCallsign"].upper(), - dx_name=source_spot["activatorName"], - de_call=source_spot["callsign"].upper(), - freq=(float(source_spot["frequency"]) * 1000000) if ( - source_spot["frequency"] is not None) else None, - # Seen SOTA spots with no frequency! - mode=source_spot["mode"].upper(), - comment=source_spot["comments"], - sig="SOTA", - sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA", name=source_spot["summitName"], - activation_score=source_spot["points"])], - time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp()) + try: + source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30)).json() + # Iterate through source data + for source_spot in source_data: + # Convert to our spot format + spot = Spot(source=self.name, + source_id=source_spot["id"], + dx_call=source_spot["activatorCallsign"].upper(), + dx_name=source_spot["activatorName"], + de_call=source_spot["callsign"].upper(), + freq=(float(source_spot["frequency"]) * 1000000) if ( + source_spot["frequency"] is not None) else None, + # Seen SOTA spots with no frequency! + mode=source_spot["mode"].upper(), + comment=source_spot["comments"], + sig="SOTA", + sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA", name=source_spot["summitName"], + activation_score=source_spot["points"])], + time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp()) - # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do - # that for us. - new_spots.append(spot) + # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do + # that for us. + new_spots.append(spot) + except ConnectionError: + logging.warning("Connection error when accessing SOTA spots API") + except (ConnectTimeout, ReadTimeout): + logging.warning(f"Timeout when accessing SOTA spots API.") return new_spots def can_submit_spot(self, sig): diff --git a/webassets/apidocs/openapi.yml b/static/apidocs/openapi.yml similarity index 100% rename from webassets/apidocs/openapi.yml rename to static/apidocs/openapi.yml diff --git a/webassets/audio/ping.mp3 b/static/audio/ping.mp3 similarity index 100% rename from webassets/audio/ping.mp3 rename to static/audio/ping.mp3 diff --git a/webassets/css/style.css b/static/css/style.css similarity index 100% rename from webassets/css/style.css rename to static/css/style.css diff --git a/webassets/favicon.ico b/static/favicon.ico similarity index 100% rename from webassets/favicon.ico rename to static/favicon.ico diff --git a/webassets/img/banner.png b/static/img/banner.png similarity index 100% rename from webassets/img/banner.png rename to static/img/banner.png diff --git a/webassets/img/flags/1.png b/static/img/flags/1.png similarity index 100% rename from webassets/img/flags/1.png rename to static/img/flags/1.png diff --git a/webassets/img/flags/10.png b/static/img/flags/10.png similarity index 100% rename from webassets/img/flags/10.png rename to static/img/flags/10.png diff --git a/webassets/img/flags/100.png b/static/img/flags/100.png similarity index 100% rename from webassets/img/flags/100.png rename to static/img/flags/100.png diff --git a/webassets/img/flags/101.png b/static/img/flags/101.png similarity index 100% rename from webassets/img/flags/101.png rename to static/img/flags/101.png diff --git a/webassets/img/flags/102.png b/static/img/flags/102.png similarity index 100% rename from webassets/img/flags/102.png rename to static/img/flags/102.png diff --git a/webassets/img/flags/103.png b/static/img/flags/103.png similarity index 100% rename from webassets/img/flags/103.png rename to static/img/flags/103.png diff --git a/webassets/img/flags/104.png b/static/img/flags/104.png similarity index 100% rename from webassets/img/flags/104.png rename to static/img/flags/104.png diff --git a/webassets/img/flags/105.png b/static/img/flags/105.png similarity index 100% rename from webassets/img/flags/105.png rename to static/img/flags/105.png diff --git a/webassets/img/flags/106.png b/static/img/flags/106.png similarity index 100% rename from webassets/img/flags/106.png rename to static/img/flags/106.png diff --git a/webassets/img/flags/107.png b/static/img/flags/107.png similarity index 100% rename from webassets/img/flags/107.png rename to static/img/flags/107.png diff --git a/webassets/img/flags/108.png b/static/img/flags/108.png similarity index 100% rename from webassets/img/flags/108.png rename to static/img/flags/108.png diff --git a/webassets/img/flags/109.png b/static/img/flags/109.png similarity index 100% rename from webassets/img/flags/109.png rename to static/img/flags/109.png diff --git a/webassets/img/flags/11.png b/static/img/flags/11.png similarity index 100% rename from webassets/img/flags/11.png rename to static/img/flags/11.png diff --git a/webassets/img/flags/110.png b/static/img/flags/110.png similarity index 100% rename from webassets/img/flags/110.png rename to static/img/flags/110.png diff --git a/webassets/img/flags/111.png b/static/img/flags/111.png similarity index 100% rename from webassets/img/flags/111.png rename to static/img/flags/111.png diff --git a/webassets/img/flags/112.png b/static/img/flags/112.png similarity index 100% rename from webassets/img/flags/112.png rename to static/img/flags/112.png diff --git a/webassets/img/flags/113.png b/static/img/flags/113.png similarity index 100% rename from webassets/img/flags/113.png rename to static/img/flags/113.png diff --git a/webassets/img/flags/114.png b/static/img/flags/114.png similarity index 100% rename from webassets/img/flags/114.png rename to static/img/flags/114.png diff --git a/webassets/img/flags/115.png b/static/img/flags/115.png similarity index 100% rename from webassets/img/flags/115.png rename to static/img/flags/115.png diff --git a/webassets/img/flags/116.png b/static/img/flags/116.png similarity index 100% rename from webassets/img/flags/116.png rename to static/img/flags/116.png diff --git a/webassets/img/flags/117.png b/static/img/flags/117.png similarity index 100% rename from webassets/img/flags/117.png rename to static/img/flags/117.png diff --git a/webassets/img/flags/118.png b/static/img/flags/118.png similarity index 100% rename from webassets/img/flags/118.png rename to static/img/flags/118.png diff --git a/webassets/img/flags/119.png b/static/img/flags/119.png similarity index 100% rename from webassets/img/flags/119.png rename to static/img/flags/119.png diff --git a/webassets/img/flags/12.png b/static/img/flags/12.png similarity index 100% rename from webassets/img/flags/12.png rename to static/img/flags/12.png diff --git a/webassets/img/flags/120.png b/static/img/flags/120.png similarity index 100% rename from webassets/img/flags/120.png rename to static/img/flags/120.png diff --git a/webassets/img/flags/122.png b/static/img/flags/122.png similarity index 100% rename from webassets/img/flags/122.png rename to static/img/flags/122.png diff --git a/webassets/img/flags/123.png b/static/img/flags/123.png similarity index 100% rename from webassets/img/flags/123.png rename to static/img/flags/123.png diff --git a/webassets/img/flags/124.png b/static/img/flags/124.png similarity index 100% rename from webassets/img/flags/124.png rename to static/img/flags/124.png diff --git a/webassets/img/flags/125.png b/static/img/flags/125.png similarity index 100% rename from webassets/img/flags/125.png rename to static/img/flags/125.png diff --git a/webassets/img/flags/126.png b/static/img/flags/126.png similarity index 100% rename from webassets/img/flags/126.png rename to static/img/flags/126.png diff --git a/webassets/img/flags/127.png b/static/img/flags/127.png similarity index 100% rename from webassets/img/flags/127.png rename to static/img/flags/127.png diff --git a/webassets/img/flags/128.png b/static/img/flags/128.png similarity index 100% rename from webassets/img/flags/128.png rename to static/img/flags/128.png diff --git a/webassets/img/flags/129.png b/static/img/flags/129.png similarity index 100% rename from webassets/img/flags/129.png rename to static/img/flags/129.png diff --git a/webassets/img/flags/13.png b/static/img/flags/13.png similarity index 100% rename from webassets/img/flags/13.png rename to static/img/flags/13.png diff --git a/webassets/img/flags/130.png b/static/img/flags/130.png similarity index 100% rename from webassets/img/flags/130.png rename to static/img/flags/130.png diff --git a/webassets/img/flags/131.png b/static/img/flags/131.png similarity index 100% rename from webassets/img/flags/131.png rename to static/img/flags/131.png diff --git a/webassets/img/flags/132.png b/static/img/flags/132.png similarity index 100% rename from webassets/img/flags/132.png rename to static/img/flags/132.png diff --git a/webassets/img/flags/133.png b/static/img/flags/133.png similarity index 100% rename from webassets/img/flags/133.png rename to static/img/flags/133.png diff --git a/webassets/img/flags/134.png b/static/img/flags/134.png similarity index 100% rename from webassets/img/flags/134.png rename to static/img/flags/134.png diff --git a/webassets/img/flags/135.png b/static/img/flags/135.png similarity index 100% rename from webassets/img/flags/135.png rename to static/img/flags/135.png diff --git a/webassets/img/flags/136.png b/static/img/flags/136.png similarity index 100% rename from webassets/img/flags/136.png rename to static/img/flags/136.png diff --git a/webassets/img/flags/137.png b/static/img/flags/137.png similarity index 100% rename from webassets/img/flags/137.png rename to static/img/flags/137.png diff --git a/webassets/img/flags/138.png b/static/img/flags/138.png similarity index 100% rename from webassets/img/flags/138.png rename to static/img/flags/138.png diff --git a/webassets/img/flags/139.png b/static/img/flags/139.png similarity index 100% rename from webassets/img/flags/139.png rename to static/img/flags/139.png diff --git a/webassets/img/flags/14.png b/static/img/flags/14.png similarity index 100% rename from webassets/img/flags/14.png rename to static/img/flags/14.png diff --git a/webassets/img/flags/140.png b/static/img/flags/140.png similarity index 100% rename from webassets/img/flags/140.png rename to static/img/flags/140.png diff --git a/webassets/img/flags/141.png b/static/img/flags/141.png similarity index 100% rename from webassets/img/flags/141.png rename to static/img/flags/141.png diff --git a/webassets/img/flags/142.png b/static/img/flags/142.png similarity index 100% rename from webassets/img/flags/142.png rename to static/img/flags/142.png diff --git a/webassets/img/flags/143.png b/static/img/flags/143.png similarity index 100% rename from webassets/img/flags/143.png rename to static/img/flags/143.png diff --git a/webassets/img/flags/144.png b/static/img/flags/144.png similarity index 100% rename from webassets/img/flags/144.png rename to static/img/flags/144.png diff --git a/webassets/img/flags/145.png b/static/img/flags/145.png similarity index 100% rename from webassets/img/flags/145.png rename to static/img/flags/145.png diff --git a/webassets/img/flags/146.png b/static/img/flags/146.png similarity index 100% rename from webassets/img/flags/146.png rename to static/img/flags/146.png diff --git a/webassets/img/flags/147.png b/static/img/flags/147.png similarity index 100% rename from webassets/img/flags/147.png rename to static/img/flags/147.png diff --git a/webassets/img/flags/148.png b/static/img/flags/148.png similarity index 100% rename from webassets/img/flags/148.png rename to static/img/flags/148.png diff --git a/webassets/img/flags/149.png b/static/img/flags/149.png similarity index 100% rename from webassets/img/flags/149.png rename to static/img/flags/149.png diff --git a/webassets/img/flags/15.png b/static/img/flags/15.png similarity index 100% rename from webassets/img/flags/15.png rename to static/img/flags/15.png diff --git a/webassets/img/flags/150.png b/static/img/flags/150.png similarity index 100% rename from webassets/img/flags/150.png rename to static/img/flags/150.png diff --git a/webassets/img/flags/151.png b/static/img/flags/151.png similarity index 100% rename from webassets/img/flags/151.png rename to static/img/flags/151.png diff --git a/webassets/img/flags/152.png b/static/img/flags/152.png similarity index 100% rename from webassets/img/flags/152.png rename to static/img/flags/152.png diff --git a/webassets/img/flags/153.png b/static/img/flags/153.png similarity index 100% rename from webassets/img/flags/153.png rename to static/img/flags/153.png diff --git a/webassets/img/flags/154.png b/static/img/flags/154.png similarity index 100% rename from webassets/img/flags/154.png rename to static/img/flags/154.png diff --git a/webassets/img/flags/155.png b/static/img/flags/155.png similarity index 100% rename from webassets/img/flags/155.png rename to static/img/flags/155.png diff --git a/webassets/img/flags/157.png b/static/img/flags/157.png similarity index 100% rename from webassets/img/flags/157.png rename to static/img/flags/157.png diff --git a/webassets/img/flags/158.png b/static/img/flags/158.png similarity index 100% rename from webassets/img/flags/158.png rename to static/img/flags/158.png diff --git a/webassets/img/flags/159.png b/static/img/flags/159.png similarity index 100% rename from webassets/img/flags/159.png rename to static/img/flags/159.png diff --git a/webassets/img/flags/16.png b/static/img/flags/16.png similarity index 100% rename from webassets/img/flags/16.png rename to static/img/flags/16.png diff --git a/webassets/img/flags/160.png b/static/img/flags/160.png similarity index 100% rename from webassets/img/flags/160.png rename to static/img/flags/160.png diff --git a/webassets/img/flags/161.png b/static/img/flags/161.png similarity index 100% rename from webassets/img/flags/161.png rename to static/img/flags/161.png diff --git a/webassets/img/flags/162.png b/static/img/flags/162.png similarity index 100% rename from webassets/img/flags/162.png rename to static/img/flags/162.png diff --git a/webassets/img/flags/163.png b/static/img/flags/163.png similarity index 100% rename from webassets/img/flags/163.png rename to static/img/flags/163.png diff --git a/webassets/img/flags/164.png b/static/img/flags/164.png similarity index 100% rename from webassets/img/flags/164.png rename to static/img/flags/164.png diff --git a/webassets/img/flags/165.png b/static/img/flags/165.png similarity index 100% rename from webassets/img/flags/165.png rename to static/img/flags/165.png diff --git a/webassets/img/flags/166.png b/static/img/flags/166.png similarity index 100% rename from webassets/img/flags/166.png rename to static/img/flags/166.png diff --git a/webassets/img/flags/167.png b/static/img/flags/167.png similarity index 100% rename from webassets/img/flags/167.png rename to static/img/flags/167.png diff --git a/webassets/img/flags/168.png b/static/img/flags/168.png similarity index 100% rename from webassets/img/flags/168.png rename to static/img/flags/168.png diff --git a/webassets/img/flags/169.png b/static/img/flags/169.png similarity index 100% rename from webassets/img/flags/169.png rename to static/img/flags/169.png diff --git a/webassets/img/flags/17.png b/static/img/flags/17.png similarity index 100% rename from webassets/img/flags/17.png rename to static/img/flags/17.png diff --git a/webassets/img/flags/170.png b/static/img/flags/170.png similarity index 100% rename from webassets/img/flags/170.png rename to static/img/flags/170.png diff --git a/webassets/img/flags/171.png b/static/img/flags/171.png similarity index 100% rename from webassets/img/flags/171.png rename to static/img/flags/171.png diff --git a/webassets/img/flags/172.png b/static/img/flags/172.png similarity index 100% rename from webassets/img/flags/172.png rename to static/img/flags/172.png diff --git a/webassets/img/flags/173.png b/static/img/flags/173.png similarity index 100% rename from webassets/img/flags/173.png rename to static/img/flags/173.png diff --git a/webassets/img/flags/174.png b/static/img/flags/174.png similarity index 100% rename from webassets/img/flags/174.png rename to static/img/flags/174.png diff --git a/webassets/img/flags/175.png b/static/img/flags/175.png similarity index 100% rename from webassets/img/flags/175.png rename to static/img/flags/175.png diff --git a/webassets/img/flags/176.png b/static/img/flags/176.png similarity index 100% rename from webassets/img/flags/176.png rename to static/img/flags/176.png diff --git a/webassets/img/flags/177.png b/static/img/flags/177.png similarity index 100% rename from webassets/img/flags/177.png rename to static/img/flags/177.png diff --git a/webassets/img/flags/178.png b/static/img/flags/178.png similarity index 100% rename from webassets/img/flags/178.png rename to static/img/flags/178.png diff --git a/webassets/img/flags/179.png b/static/img/flags/179.png similarity index 100% rename from webassets/img/flags/179.png rename to static/img/flags/179.png diff --git a/webassets/img/flags/18.png b/static/img/flags/18.png similarity index 100% rename from webassets/img/flags/18.png rename to static/img/flags/18.png diff --git a/webassets/img/flags/180.png b/static/img/flags/180.png similarity index 100% rename from webassets/img/flags/180.png rename to static/img/flags/180.png diff --git a/webassets/img/flags/181.png b/static/img/flags/181.png similarity index 100% rename from webassets/img/flags/181.png rename to static/img/flags/181.png diff --git a/webassets/img/flags/182.png b/static/img/flags/182.png similarity index 100% rename from webassets/img/flags/182.png rename to static/img/flags/182.png diff --git a/webassets/img/flags/183.png b/static/img/flags/183.png similarity index 100% rename from webassets/img/flags/183.png rename to static/img/flags/183.png diff --git a/webassets/img/flags/184.png b/static/img/flags/184.png similarity index 100% rename from webassets/img/flags/184.png rename to static/img/flags/184.png diff --git a/webassets/img/flags/185.png b/static/img/flags/185.png similarity index 100% rename from webassets/img/flags/185.png rename to static/img/flags/185.png diff --git a/webassets/img/flags/186.png b/static/img/flags/186.png similarity index 100% rename from webassets/img/flags/186.png rename to static/img/flags/186.png diff --git a/webassets/img/flags/187.png b/static/img/flags/187.png similarity index 100% rename from webassets/img/flags/187.png rename to static/img/flags/187.png diff --git a/webassets/img/flags/188.png b/static/img/flags/188.png similarity index 100% rename from webassets/img/flags/188.png rename to static/img/flags/188.png diff --git a/webassets/img/flags/189.png b/static/img/flags/189.png similarity index 100% rename from webassets/img/flags/189.png rename to static/img/flags/189.png diff --git a/webassets/img/flags/19.png b/static/img/flags/19.png similarity index 100% rename from webassets/img/flags/19.png rename to static/img/flags/19.png diff --git a/webassets/img/flags/190.png b/static/img/flags/190.png similarity index 100% rename from webassets/img/flags/190.png rename to static/img/flags/190.png diff --git a/webassets/img/flags/191.png b/static/img/flags/191.png similarity index 100% rename from webassets/img/flags/191.png rename to static/img/flags/191.png diff --git a/webassets/img/flags/192.png b/static/img/flags/192.png similarity index 100% rename from webassets/img/flags/192.png rename to static/img/flags/192.png diff --git a/webassets/img/flags/193.png b/static/img/flags/193.png similarity index 100% rename from webassets/img/flags/193.png rename to static/img/flags/193.png diff --git a/webassets/img/flags/194.png b/static/img/flags/194.png similarity index 100% rename from webassets/img/flags/194.png rename to static/img/flags/194.png diff --git a/webassets/img/flags/195.png b/static/img/flags/195.png similarity index 100% rename from webassets/img/flags/195.png rename to static/img/flags/195.png diff --git a/webassets/img/flags/196.png b/static/img/flags/196.png similarity index 100% rename from webassets/img/flags/196.png rename to static/img/flags/196.png diff --git a/webassets/img/flags/197.png b/static/img/flags/197.png similarity index 100% rename from webassets/img/flags/197.png rename to static/img/flags/197.png diff --git a/webassets/img/flags/198.png b/static/img/flags/198.png similarity index 100% rename from webassets/img/flags/198.png rename to static/img/flags/198.png diff --git a/webassets/img/flags/199.png b/static/img/flags/199.png similarity index 100% rename from webassets/img/flags/199.png rename to static/img/flags/199.png diff --git a/webassets/img/flags/2.png b/static/img/flags/2.png similarity index 100% rename from webassets/img/flags/2.png rename to static/img/flags/2.png diff --git a/webassets/img/flags/20.png b/static/img/flags/20.png similarity index 100% rename from webassets/img/flags/20.png rename to static/img/flags/20.png diff --git a/webassets/img/flags/200.png b/static/img/flags/200.png similarity index 100% rename from webassets/img/flags/200.png rename to static/img/flags/200.png diff --git a/webassets/img/flags/201.png b/static/img/flags/201.png similarity index 100% rename from webassets/img/flags/201.png rename to static/img/flags/201.png diff --git a/webassets/img/flags/202.png b/static/img/flags/202.png similarity index 100% rename from webassets/img/flags/202.png rename to static/img/flags/202.png diff --git a/webassets/img/flags/203.png b/static/img/flags/203.png similarity index 100% rename from webassets/img/flags/203.png rename to static/img/flags/203.png diff --git a/webassets/img/flags/204.png b/static/img/flags/204.png similarity index 100% rename from webassets/img/flags/204.png rename to static/img/flags/204.png diff --git a/webassets/img/flags/205.png b/static/img/flags/205.png similarity index 100% rename from webassets/img/flags/205.png rename to static/img/flags/205.png diff --git a/webassets/img/flags/206.png b/static/img/flags/206.png similarity index 100% rename from webassets/img/flags/206.png rename to static/img/flags/206.png diff --git a/webassets/img/flags/207.png b/static/img/flags/207.png similarity index 100% rename from webassets/img/flags/207.png rename to static/img/flags/207.png diff --git a/webassets/img/flags/208.png b/static/img/flags/208.png similarity index 100% rename from webassets/img/flags/208.png rename to static/img/flags/208.png diff --git a/webassets/img/flags/209.png b/static/img/flags/209.png similarity index 100% rename from webassets/img/flags/209.png rename to static/img/flags/209.png diff --git a/webassets/img/flags/21.png b/static/img/flags/21.png similarity index 100% rename from webassets/img/flags/21.png rename to static/img/flags/21.png diff --git a/webassets/img/flags/210.png b/static/img/flags/210.png similarity index 100% rename from webassets/img/flags/210.png rename to static/img/flags/210.png diff --git a/webassets/img/flags/211.png b/static/img/flags/211.png similarity index 100% rename from webassets/img/flags/211.png rename to static/img/flags/211.png diff --git a/webassets/img/flags/212.png b/static/img/flags/212.png similarity index 100% rename from webassets/img/flags/212.png rename to static/img/flags/212.png diff --git a/webassets/img/flags/213.png b/static/img/flags/213.png similarity index 100% rename from webassets/img/flags/213.png rename to static/img/flags/213.png diff --git a/webassets/img/flags/214.png b/static/img/flags/214.png similarity index 100% rename from webassets/img/flags/214.png rename to static/img/flags/214.png diff --git a/webassets/img/flags/215.png b/static/img/flags/215.png similarity index 100% rename from webassets/img/flags/215.png rename to static/img/flags/215.png diff --git a/webassets/img/flags/216.png b/static/img/flags/216.png similarity index 100% rename from webassets/img/flags/216.png rename to static/img/flags/216.png diff --git a/webassets/img/flags/217.png b/static/img/flags/217.png similarity index 100% rename from webassets/img/flags/217.png rename to static/img/flags/217.png diff --git a/webassets/img/flags/218.png b/static/img/flags/218.png similarity index 100% rename from webassets/img/flags/218.png rename to static/img/flags/218.png diff --git a/webassets/img/flags/219.png b/static/img/flags/219.png similarity index 100% rename from webassets/img/flags/219.png rename to static/img/flags/219.png diff --git a/webassets/img/flags/22.png b/static/img/flags/22.png similarity index 100% rename from webassets/img/flags/22.png rename to static/img/flags/22.png diff --git a/webassets/img/flags/220.png b/static/img/flags/220.png similarity index 100% rename from webassets/img/flags/220.png rename to static/img/flags/220.png diff --git a/webassets/img/flags/221.png b/static/img/flags/221.png similarity index 100% rename from webassets/img/flags/221.png rename to static/img/flags/221.png diff --git a/webassets/img/flags/222.png b/static/img/flags/222.png similarity index 100% rename from webassets/img/flags/222.png rename to static/img/flags/222.png diff --git a/webassets/img/flags/223.png b/static/img/flags/223.png similarity index 100% rename from webassets/img/flags/223.png rename to static/img/flags/223.png diff --git a/webassets/img/flags/224.png b/static/img/flags/224.png similarity index 100% rename from webassets/img/flags/224.png rename to static/img/flags/224.png diff --git a/webassets/img/flags/225.png b/static/img/flags/225.png similarity index 100% rename from webassets/img/flags/225.png rename to static/img/flags/225.png diff --git a/webassets/img/flags/226.png b/static/img/flags/226.png similarity index 100% rename from webassets/img/flags/226.png rename to static/img/flags/226.png diff --git a/webassets/img/flags/227.png b/static/img/flags/227.png similarity index 100% rename from webassets/img/flags/227.png rename to static/img/flags/227.png diff --git a/webassets/img/flags/228.png b/static/img/flags/228.png similarity index 100% rename from webassets/img/flags/228.png rename to static/img/flags/228.png diff --git a/webassets/img/flags/229.png b/static/img/flags/229.png similarity index 100% rename from webassets/img/flags/229.png rename to static/img/flags/229.png diff --git a/webassets/img/flags/23.png b/static/img/flags/23.png similarity index 100% rename from webassets/img/flags/23.png rename to static/img/flags/23.png diff --git a/webassets/img/flags/230.png b/static/img/flags/230.png similarity index 100% rename from webassets/img/flags/230.png rename to static/img/flags/230.png diff --git a/webassets/img/flags/231.png b/static/img/flags/231.png similarity index 100% rename from webassets/img/flags/231.png rename to static/img/flags/231.png diff --git a/webassets/img/flags/232.png b/static/img/flags/232.png similarity index 100% rename from webassets/img/flags/232.png rename to static/img/flags/232.png diff --git a/webassets/img/flags/233.png b/static/img/flags/233.png similarity index 100% rename from webassets/img/flags/233.png rename to static/img/flags/233.png diff --git a/webassets/img/flags/234.png b/static/img/flags/234.png similarity index 100% rename from webassets/img/flags/234.png rename to static/img/flags/234.png diff --git a/webassets/img/flags/235.png b/static/img/flags/235.png similarity index 100% rename from webassets/img/flags/235.png rename to static/img/flags/235.png diff --git a/webassets/img/flags/236.png b/static/img/flags/236.png similarity index 100% rename from webassets/img/flags/236.png rename to static/img/flags/236.png diff --git a/webassets/img/flags/237.png b/static/img/flags/237.png similarity index 100% rename from webassets/img/flags/237.png rename to static/img/flags/237.png diff --git a/webassets/img/flags/238.png b/static/img/flags/238.png similarity index 100% rename from webassets/img/flags/238.png rename to static/img/flags/238.png diff --git a/webassets/img/flags/239.png b/static/img/flags/239.png similarity index 100% rename from webassets/img/flags/239.png rename to static/img/flags/239.png diff --git a/webassets/img/flags/24.png b/static/img/flags/24.png similarity index 100% rename from webassets/img/flags/24.png rename to static/img/flags/24.png diff --git a/webassets/img/flags/240.png b/static/img/flags/240.png similarity index 100% rename from webassets/img/flags/240.png rename to static/img/flags/240.png diff --git a/webassets/img/flags/241.png b/static/img/flags/241.png similarity index 100% rename from webassets/img/flags/241.png rename to static/img/flags/241.png diff --git a/webassets/img/flags/242.png b/static/img/flags/242.png similarity index 100% rename from webassets/img/flags/242.png rename to static/img/flags/242.png diff --git a/webassets/img/flags/243.png b/static/img/flags/243.png similarity index 100% rename from webassets/img/flags/243.png rename to static/img/flags/243.png diff --git a/webassets/img/flags/244.png b/static/img/flags/244.png similarity index 100% rename from webassets/img/flags/244.png rename to static/img/flags/244.png diff --git a/webassets/img/flags/245.png b/static/img/flags/245.png similarity index 100% rename from webassets/img/flags/245.png rename to static/img/flags/245.png diff --git a/webassets/img/flags/246.png b/static/img/flags/246.png similarity index 100% rename from webassets/img/flags/246.png rename to static/img/flags/246.png diff --git a/webassets/img/flags/247.png b/static/img/flags/247.png similarity index 100% rename from webassets/img/flags/247.png rename to static/img/flags/247.png diff --git a/webassets/img/flags/248.png b/static/img/flags/248.png similarity index 100% rename from webassets/img/flags/248.png rename to static/img/flags/248.png diff --git a/webassets/img/flags/249.png b/static/img/flags/249.png similarity index 100% rename from webassets/img/flags/249.png rename to static/img/flags/249.png diff --git a/webassets/img/flags/25.png b/static/img/flags/25.png similarity index 100% rename from webassets/img/flags/25.png rename to static/img/flags/25.png diff --git a/webassets/img/flags/250.png b/static/img/flags/250.png similarity index 100% rename from webassets/img/flags/250.png rename to static/img/flags/250.png diff --git a/webassets/img/flags/251.png b/static/img/flags/251.png similarity index 100% rename from webassets/img/flags/251.png rename to static/img/flags/251.png diff --git a/webassets/img/flags/252.png b/static/img/flags/252.png similarity index 100% rename from webassets/img/flags/252.png rename to static/img/flags/252.png diff --git a/webassets/img/flags/253.png b/static/img/flags/253.png similarity index 100% rename from webassets/img/flags/253.png rename to static/img/flags/253.png diff --git a/webassets/img/flags/254.png b/static/img/flags/254.png similarity index 100% rename from webassets/img/flags/254.png rename to static/img/flags/254.png diff --git a/webassets/img/flags/255.png b/static/img/flags/255.png similarity index 100% rename from webassets/img/flags/255.png rename to static/img/flags/255.png diff --git a/webassets/img/flags/256.png b/static/img/flags/256.png similarity index 100% rename from webassets/img/flags/256.png rename to static/img/flags/256.png diff --git a/webassets/img/flags/257.png b/static/img/flags/257.png similarity index 100% rename from webassets/img/flags/257.png rename to static/img/flags/257.png diff --git a/webassets/img/flags/258.png b/static/img/flags/258.png similarity index 100% rename from webassets/img/flags/258.png rename to static/img/flags/258.png diff --git a/webassets/img/flags/259.png b/static/img/flags/259.png similarity index 100% rename from webassets/img/flags/259.png rename to static/img/flags/259.png diff --git a/webassets/img/flags/26.png b/static/img/flags/26.png similarity index 100% rename from webassets/img/flags/26.png rename to static/img/flags/26.png diff --git a/webassets/img/flags/260.png b/static/img/flags/260.png similarity index 100% rename from webassets/img/flags/260.png rename to static/img/flags/260.png diff --git a/webassets/img/flags/261.png b/static/img/flags/261.png similarity index 100% rename from webassets/img/flags/261.png rename to static/img/flags/261.png diff --git a/webassets/img/flags/262.png b/static/img/flags/262.png similarity index 100% rename from webassets/img/flags/262.png rename to static/img/flags/262.png diff --git a/webassets/img/flags/263.png b/static/img/flags/263.png similarity index 100% rename from webassets/img/flags/263.png rename to static/img/flags/263.png diff --git a/webassets/img/flags/264.png b/static/img/flags/264.png similarity index 100% rename from webassets/img/flags/264.png rename to static/img/flags/264.png diff --git a/webassets/img/flags/265.png b/static/img/flags/265.png similarity index 100% rename from webassets/img/flags/265.png rename to static/img/flags/265.png diff --git a/webassets/img/flags/266.png b/static/img/flags/266.png similarity index 100% rename from webassets/img/flags/266.png rename to static/img/flags/266.png diff --git a/webassets/img/flags/267.png b/static/img/flags/267.png similarity index 100% rename from webassets/img/flags/267.png rename to static/img/flags/267.png diff --git a/webassets/img/flags/268.png b/static/img/flags/268.png similarity index 100% rename from webassets/img/flags/268.png rename to static/img/flags/268.png diff --git a/webassets/img/flags/269.png b/static/img/flags/269.png similarity index 100% rename from webassets/img/flags/269.png rename to static/img/flags/269.png diff --git a/webassets/img/flags/27.png b/static/img/flags/27.png similarity index 100% rename from webassets/img/flags/27.png rename to static/img/flags/27.png diff --git a/webassets/img/flags/270.png b/static/img/flags/270.png similarity index 100% rename from webassets/img/flags/270.png rename to static/img/flags/270.png diff --git a/webassets/img/flags/271.png b/static/img/flags/271.png similarity index 100% rename from webassets/img/flags/271.png rename to static/img/flags/271.png diff --git a/webassets/img/flags/272.png b/static/img/flags/272.png similarity index 100% rename from webassets/img/flags/272.png rename to static/img/flags/272.png diff --git a/webassets/img/flags/273.png b/static/img/flags/273.png similarity index 100% rename from webassets/img/flags/273.png rename to static/img/flags/273.png diff --git a/webassets/img/flags/274.png b/static/img/flags/274.png similarity index 100% rename from webassets/img/flags/274.png rename to static/img/flags/274.png diff --git a/webassets/img/flags/275.png b/static/img/flags/275.png similarity index 100% rename from webassets/img/flags/275.png rename to static/img/flags/275.png diff --git a/webassets/img/flags/276.png b/static/img/flags/276.png similarity index 100% rename from webassets/img/flags/276.png rename to static/img/flags/276.png diff --git a/webassets/img/flags/277.png b/static/img/flags/277.png similarity index 100% rename from webassets/img/flags/277.png rename to static/img/flags/277.png diff --git a/webassets/img/flags/278.png b/static/img/flags/278.png similarity index 100% rename from webassets/img/flags/278.png rename to static/img/flags/278.png diff --git a/webassets/img/flags/279.png b/static/img/flags/279.png similarity index 100% rename from webassets/img/flags/279.png rename to static/img/flags/279.png diff --git a/webassets/img/flags/28.png b/static/img/flags/28.png similarity index 100% rename from webassets/img/flags/28.png rename to static/img/flags/28.png diff --git a/webassets/img/flags/280.png b/static/img/flags/280.png similarity index 100% rename from webassets/img/flags/280.png rename to static/img/flags/280.png diff --git a/webassets/img/flags/281.png b/static/img/flags/281.png similarity index 100% rename from webassets/img/flags/281.png rename to static/img/flags/281.png diff --git a/webassets/img/flags/282.png b/static/img/flags/282.png similarity index 100% rename from webassets/img/flags/282.png rename to static/img/flags/282.png diff --git a/webassets/img/flags/283.png b/static/img/flags/283.png similarity index 100% rename from webassets/img/flags/283.png rename to static/img/flags/283.png diff --git a/webassets/img/flags/284.png b/static/img/flags/284.png similarity index 100% rename from webassets/img/flags/284.png rename to static/img/flags/284.png diff --git a/webassets/img/flags/285.png b/static/img/flags/285.png similarity index 100% rename from webassets/img/flags/285.png rename to static/img/flags/285.png diff --git a/webassets/img/flags/286.png b/static/img/flags/286.png similarity index 100% rename from webassets/img/flags/286.png rename to static/img/flags/286.png diff --git a/webassets/img/flags/287.png b/static/img/flags/287.png similarity index 100% rename from webassets/img/flags/287.png rename to static/img/flags/287.png diff --git a/webassets/img/flags/288.png b/static/img/flags/288.png similarity index 100% rename from webassets/img/flags/288.png rename to static/img/flags/288.png diff --git a/webassets/img/flags/289.png b/static/img/flags/289.png similarity index 100% rename from webassets/img/flags/289.png rename to static/img/flags/289.png diff --git a/webassets/img/flags/29.png b/static/img/flags/29.png similarity index 100% rename from webassets/img/flags/29.png rename to static/img/flags/29.png diff --git a/webassets/img/flags/291.png b/static/img/flags/291.png similarity index 100% rename from webassets/img/flags/291.png rename to static/img/flags/291.png diff --git a/webassets/img/flags/292.png b/static/img/flags/292.png similarity index 100% rename from webassets/img/flags/292.png rename to static/img/flags/292.png diff --git a/webassets/img/flags/293.png b/static/img/flags/293.png similarity index 100% rename from webassets/img/flags/293.png rename to static/img/flags/293.png diff --git a/webassets/img/flags/294.png b/static/img/flags/294.png similarity index 100% rename from webassets/img/flags/294.png rename to static/img/flags/294.png diff --git a/webassets/img/flags/295.png b/static/img/flags/295.png similarity index 100% rename from webassets/img/flags/295.png rename to static/img/flags/295.png diff --git a/webassets/img/flags/296.png b/static/img/flags/296.png similarity index 100% rename from webassets/img/flags/296.png rename to static/img/flags/296.png diff --git a/webassets/img/flags/297.png b/static/img/flags/297.png similarity index 100% rename from webassets/img/flags/297.png rename to static/img/flags/297.png diff --git a/webassets/img/flags/298.png b/static/img/flags/298.png similarity index 100% rename from webassets/img/flags/298.png rename to static/img/flags/298.png diff --git a/webassets/img/flags/299.png b/static/img/flags/299.png similarity index 100% rename from webassets/img/flags/299.png rename to static/img/flags/299.png diff --git a/webassets/img/flags/3.png b/static/img/flags/3.png similarity index 100% rename from webassets/img/flags/3.png rename to static/img/flags/3.png diff --git a/webassets/img/flags/30.png b/static/img/flags/30.png similarity index 100% rename from webassets/img/flags/30.png rename to static/img/flags/30.png diff --git a/webassets/img/flags/301.png b/static/img/flags/301.png similarity index 100% rename from webassets/img/flags/301.png rename to static/img/flags/301.png diff --git a/webassets/img/flags/302.png b/static/img/flags/302.png similarity index 100% rename from webassets/img/flags/302.png rename to static/img/flags/302.png diff --git a/webassets/img/flags/303.png b/static/img/flags/303.png similarity index 100% rename from webassets/img/flags/303.png rename to static/img/flags/303.png diff --git a/webassets/img/flags/304.png b/static/img/flags/304.png similarity index 100% rename from webassets/img/flags/304.png rename to static/img/flags/304.png diff --git a/webassets/img/flags/305.png b/static/img/flags/305.png similarity index 100% rename from webassets/img/flags/305.png rename to static/img/flags/305.png diff --git a/webassets/img/flags/306.png b/static/img/flags/306.png similarity index 100% rename from webassets/img/flags/306.png rename to static/img/flags/306.png diff --git a/webassets/img/flags/307.png b/static/img/flags/307.png similarity index 100% rename from webassets/img/flags/307.png rename to static/img/flags/307.png diff --git a/webassets/img/flags/308.png b/static/img/flags/308.png similarity index 100% rename from webassets/img/flags/308.png rename to static/img/flags/308.png diff --git a/webassets/img/flags/309.png b/static/img/flags/309.png similarity index 100% rename from webassets/img/flags/309.png rename to static/img/flags/309.png diff --git a/webassets/img/flags/31.png b/static/img/flags/31.png similarity index 100% rename from webassets/img/flags/31.png rename to static/img/flags/31.png diff --git a/webassets/img/flags/312.png b/static/img/flags/312.png similarity index 100% rename from webassets/img/flags/312.png rename to static/img/flags/312.png diff --git a/webassets/img/flags/315.png b/static/img/flags/315.png similarity index 100% rename from webassets/img/flags/315.png rename to static/img/flags/315.png diff --git a/webassets/img/flags/318.png b/static/img/flags/318.png similarity index 100% rename from webassets/img/flags/318.png rename to static/img/flags/318.png diff --git a/webassets/img/flags/32.png b/static/img/flags/32.png similarity index 100% rename from webassets/img/flags/32.png rename to static/img/flags/32.png diff --git a/webassets/img/flags/321.png b/static/img/flags/321.png similarity index 100% rename from webassets/img/flags/321.png rename to static/img/flags/321.png diff --git a/webassets/img/flags/324.png b/static/img/flags/324.png similarity index 100% rename from webassets/img/flags/324.png rename to static/img/flags/324.png diff --git a/webassets/img/flags/327.png b/static/img/flags/327.png similarity index 100% rename from webassets/img/flags/327.png rename to static/img/flags/327.png diff --git a/webassets/img/flags/33.png b/static/img/flags/33.png similarity index 100% rename from webassets/img/flags/33.png rename to static/img/flags/33.png diff --git a/webassets/img/flags/330.png b/static/img/flags/330.png similarity index 100% rename from webassets/img/flags/330.png rename to static/img/flags/330.png diff --git a/webassets/img/flags/333.png b/static/img/flags/333.png similarity index 100% rename from webassets/img/flags/333.png rename to static/img/flags/333.png diff --git a/webassets/img/flags/336.png b/static/img/flags/336.png similarity index 100% rename from webassets/img/flags/336.png rename to static/img/flags/336.png diff --git a/webassets/img/flags/339.png b/static/img/flags/339.png similarity index 100% rename from webassets/img/flags/339.png rename to static/img/flags/339.png diff --git a/webassets/img/flags/34.png b/static/img/flags/34.png similarity index 100% rename from webassets/img/flags/34.png rename to static/img/flags/34.png diff --git a/webassets/img/flags/342.png b/static/img/flags/342.png similarity index 100% rename from webassets/img/flags/342.png rename to static/img/flags/342.png diff --git a/webassets/img/flags/344.png b/static/img/flags/344.png similarity index 100% rename from webassets/img/flags/344.png rename to static/img/flags/344.png diff --git a/webassets/img/flags/345.png b/static/img/flags/345.png similarity index 100% rename from webassets/img/flags/345.png rename to static/img/flags/345.png diff --git a/webassets/img/flags/348.png b/static/img/flags/348.png similarity index 100% rename from webassets/img/flags/348.png rename to static/img/flags/348.png diff --git a/webassets/img/flags/35.png b/static/img/flags/35.png similarity index 100% rename from webassets/img/flags/35.png rename to static/img/flags/35.png diff --git a/webassets/img/flags/354.png b/static/img/flags/354.png similarity index 100% rename from webassets/img/flags/354.png rename to static/img/flags/354.png diff --git a/webassets/img/flags/36.png b/static/img/flags/36.png similarity index 100% rename from webassets/img/flags/36.png rename to static/img/flags/36.png diff --git a/webassets/img/flags/363.png b/static/img/flags/363.png similarity index 100% rename from webassets/img/flags/363.png rename to static/img/flags/363.png diff --git a/webassets/img/flags/369.png b/static/img/flags/369.png similarity index 100% rename from webassets/img/flags/369.png rename to static/img/flags/369.png diff --git a/webassets/img/flags/37.png b/static/img/flags/37.png similarity index 100% rename from webassets/img/flags/37.png rename to static/img/flags/37.png diff --git a/webassets/img/flags/370.png b/static/img/flags/370.png similarity index 100% rename from webassets/img/flags/370.png rename to static/img/flags/370.png diff --git a/webassets/img/flags/372.png b/static/img/flags/372.png similarity index 100% rename from webassets/img/flags/372.png rename to static/img/flags/372.png diff --git a/webassets/img/flags/375.png b/static/img/flags/375.png similarity index 100% rename from webassets/img/flags/375.png rename to static/img/flags/375.png diff --git a/webassets/img/flags/376.png b/static/img/flags/376.png similarity index 100% rename from webassets/img/flags/376.png rename to static/img/flags/376.png diff --git a/webassets/img/flags/378.png b/static/img/flags/378.png similarity index 100% rename from webassets/img/flags/378.png rename to static/img/flags/378.png diff --git a/webassets/img/flags/379.png b/static/img/flags/379.png similarity index 100% rename from webassets/img/flags/379.png rename to static/img/flags/379.png diff --git a/webassets/img/flags/38.png b/static/img/flags/38.png similarity index 100% rename from webassets/img/flags/38.png rename to static/img/flags/38.png diff --git a/webassets/img/flags/381.png b/static/img/flags/381.png similarity index 100% rename from webassets/img/flags/381.png rename to static/img/flags/381.png diff --git a/webassets/img/flags/382.png b/static/img/flags/382.png similarity index 100% rename from webassets/img/flags/382.png rename to static/img/flags/382.png diff --git a/webassets/img/flags/384.png b/static/img/flags/384.png similarity index 100% rename from webassets/img/flags/384.png rename to static/img/flags/384.png diff --git a/webassets/img/flags/386.png b/static/img/flags/386.png similarity index 100% rename from webassets/img/flags/386.png rename to static/img/flags/386.png diff --git a/webassets/img/flags/387.png b/static/img/flags/387.png similarity index 100% rename from webassets/img/flags/387.png rename to static/img/flags/387.png diff --git a/webassets/img/flags/39.png b/static/img/flags/39.png similarity index 100% rename from webassets/img/flags/39.png rename to static/img/flags/39.png diff --git a/webassets/img/flags/390.png b/static/img/flags/390.png similarity index 100% rename from webassets/img/flags/390.png rename to static/img/flags/390.png diff --git a/webassets/img/flags/391.png b/static/img/flags/391.png similarity index 100% rename from webassets/img/flags/391.png rename to static/img/flags/391.png diff --git a/webassets/img/flags/4.png b/static/img/flags/4.png similarity index 100% rename from webassets/img/flags/4.png rename to static/img/flags/4.png diff --git a/webassets/img/flags/40.png b/static/img/flags/40.png similarity index 100% rename from webassets/img/flags/40.png rename to static/img/flags/40.png diff --git a/webassets/img/flags/400.png b/static/img/flags/400.png similarity index 100% rename from webassets/img/flags/400.png rename to static/img/flags/400.png diff --git a/webassets/img/flags/401.png b/static/img/flags/401.png similarity index 100% rename from webassets/img/flags/401.png rename to static/img/flags/401.png diff --git a/webassets/img/flags/402.png b/static/img/flags/402.png similarity index 100% rename from webassets/img/flags/402.png rename to static/img/flags/402.png diff --git a/webassets/img/flags/404.png b/static/img/flags/404.png similarity index 100% rename from webassets/img/flags/404.png rename to static/img/flags/404.png diff --git a/webassets/img/flags/406.png b/static/img/flags/406.png similarity index 100% rename from webassets/img/flags/406.png rename to static/img/flags/406.png diff --git a/webassets/img/flags/408.png b/static/img/flags/408.png similarity index 100% rename from webassets/img/flags/408.png rename to static/img/flags/408.png diff --git a/webassets/img/flags/409.png b/static/img/flags/409.png similarity index 100% rename from webassets/img/flags/409.png rename to static/img/flags/409.png diff --git a/webassets/img/flags/41.png b/static/img/flags/41.png similarity index 100% rename from webassets/img/flags/41.png rename to static/img/flags/41.png diff --git a/webassets/img/flags/410.png b/static/img/flags/410.png similarity index 100% rename from webassets/img/flags/410.png rename to static/img/flags/410.png diff --git a/webassets/img/flags/411.png b/static/img/flags/411.png similarity index 100% rename from webassets/img/flags/411.png rename to static/img/flags/411.png diff --git a/webassets/img/flags/412.png b/static/img/flags/412.png similarity index 100% rename from webassets/img/flags/412.png rename to static/img/flags/412.png diff --git a/webassets/img/flags/414.png b/static/img/flags/414.png similarity index 100% rename from webassets/img/flags/414.png rename to static/img/flags/414.png diff --git a/webassets/img/flags/416.png b/static/img/flags/416.png similarity index 100% rename from webassets/img/flags/416.png rename to static/img/flags/416.png diff --git a/webassets/img/flags/42.png b/static/img/flags/42.png similarity index 100% rename from webassets/img/flags/42.png rename to static/img/flags/42.png diff --git a/webassets/img/flags/420.png b/static/img/flags/420.png similarity index 100% rename from webassets/img/flags/420.png rename to static/img/flags/420.png diff --git a/webassets/img/flags/422.png b/static/img/flags/422.png similarity index 100% rename from webassets/img/flags/422.png rename to static/img/flags/422.png diff --git a/webassets/img/flags/424.png b/static/img/flags/424.png similarity index 100% rename from webassets/img/flags/424.png rename to static/img/flags/424.png diff --git a/webassets/img/flags/428.png b/static/img/flags/428.png similarity index 100% rename from webassets/img/flags/428.png rename to static/img/flags/428.png diff --git a/webassets/img/flags/43.png b/static/img/flags/43.png similarity index 100% rename from webassets/img/flags/43.png rename to static/img/flags/43.png diff --git a/webassets/img/flags/430.png b/static/img/flags/430.png similarity index 100% rename from webassets/img/flags/430.png rename to static/img/flags/430.png diff --git a/webassets/img/flags/432.png b/static/img/flags/432.png similarity index 100% rename from webassets/img/flags/432.png rename to static/img/flags/432.png diff --git a/webassets/img/flags/434.png b/static/img/flags/434.png similarity index 100% rename from webassets/img/flags/434.png rename to static/img/flags/434.png diff --git a/webassets/img/flags/436.png b/static/img/flags/436.png similarity index 100% rename from webassets/img/flags/436.png rename to static/img/flags/436.png diff --git a/webassets/img/flags/438.png b/static/img/flags/438.png similarity index 100% rename from webassets/img/flags/438.png rename to static/img/flags/438.png diff --git a/webassets/img/flags/44.png b/static/img/flags/44.png similarity index 100% rename from webassets/img/flags/44.png rename to static/img/flags/44.png diff --git a/webassets/img/flags/440.png b/static/img/flags/440.png similarity index 100% rename from webassets/img/flags/440.png rename to static/img/flags/440.png diff --git a/webassets/img/flags/442.png b/static/img/flags/442.png similarity index 100% rename from webassets/img/flags/442.png rename to static/img/flags/442.png diff --git a/webassets/img/flags/444.png b/static/img/flags/444.png similarity index 100% rename from webassets/img/flags/444.png rename to static/img/flags/444.png diff --git a/webassets/img/flags/446.png b/static/img/flags/446.png similarity index 100% rename from webassets/img/flags/446.png rename to static/img/flags/446.png diff --git a/webassets/img/flags/45.png b/static/img/flags/45.png similarity index 100% rename from webassets/img/flags/45.png rename to static/img/flags/45.png diff --git a/webassets/img/flags/450.png b/static/img/flags/450.png similarity index 100% rename from webassets/img/flags/450.png rename to static/img/flags/450.png diff --git a/webassets/img/flags/452.png b/static/img/flags/452.png similarity index 100% rename from webassets/img/flags/452.png rename to static/img/flags/452.png diff --git a/webassets/img/flags/453.png b/static/img/flags/453.png similarity index 100% rename from webassets/img/flags/453.png rename to static/img/flags/453.png diff --git a/webassets/img/flags/454.png b/static/img/flags/454.png similarity index 100% rename from webassets/img/flags/454.png rename to static/img/flags/454.png diff --git a/webassets/img/flags/456.png b/static/img/flags/456.png similarity index 100% rename from webassets/img/flags/456.png rename to static/img/flags/456.png diff --git a/webassets/img/flags/458.png b/static/img/flags/458.png similarity index 100% rename from webassets/img/flags/458.png rename to static/img/flags/458.png diff --git a/webassets/img/flags/46.png b/static/img/flags/46.png similarity index 100% rename from webassets/img/flags/46.png rename to static/img/flags/46.png diff --git a/webassets/img/flags/460.png b/static/img/flags/460.png similarity index 100% rename from webassets/img/flags/460.png rename to static/img/flags/460.png diff --git a/webassets/img/flags/462.png b/static/img/flags/462.png similarity index 100% rename from webassets/img/flags/462.png rename to static/img/flags/462.png diff --git a/webassets/img/flags/464.png b/static/img/flags/464.png similarity index 100% rename from webassets/img/flags/464.png rename to static/img/flags/464.png diff --git a/webassets/img/flags/466.png b/static/img/flags/466.png similarity index 100% rename from webassets/img/flags/466.png rename to static/img/flags/466.png diff --git a/webassets/img/flags/468.png b/static/img/flags/468.png similarity index 100% rename from webassets/img/flags/468.png rename to static/img/flags/468.png diff --git a/webassets/img/flags/47.png b/static/img/flags/47.png similarity index 100% rename from webassets/img/flags/47.png rename to static/img/flags/47.png diff --git a/webassets/img/flags/470.png b/static/img/flags/470.png similarity index 100% rename from webassets/img/flags/470.png rename to static/img/flags/470.png diff --git a/webassets/img/flags/474.png b/static/img/flags/474.png similarity index 100% rename from webassets/img/flags/474.png rename to static/img/flags/474.png diff --git a/webassets/img/flags/478.png b/static/img/flags/478.png similarity index 100% rename from webassets/img/flags/478.png rename to static/img/flags/478.png diff --git a/webassets/img/flags/48.png b/static/img/flags/48.png similarity index 100% rename from webassets/img/flags/48.png rename to static/img/flags/48.png diff --git a/webassets/img/flags/480.png b/static/img/flags/480.png similarity index 100% rename from webassets/img/flags/480.png rename to static/img/flags/480.png diff --git a/webassets/img/flags/482.png b/static/img/flags/482.png similarity index 100% rename from webassets/img/flags/482.png rename to static/img/flags/482.png diff --git a/webassets/img/flags/483.png b/static/img/flags/483.png similarity index 100% rename from webassets/img/flags/483.png rename to static/img/flags/483.png diff --git a/webassets/img/flags/488.png b/static/img/flags/488.png similarity index 100% rename from webassets/img/flags/488.png rename to static/img/flags/488.png diff --git a/webassets/img/flags/489.png b/static/img/flags/489.png similarity index 100% rename from webassets/img/flags/489.png rename to static/img/flags/489.png diff --git a/webassets/img/flags/49.png b/static/img/flags/49.png similarity index 100% rename from webassets/img/flags/49.png rename to static/img/flags/49.png diff --git a/webassets/img/flags/490.png b/static/img/flags/490.png similarity index 100% rename from webassets/img/flags/490.png rename to static/img/flags/490.png diff --git a/webassets/img/flags/492.png b/static/img/flags/492.png similarity index 100% rename from webassets/img/flags/492.png rename to static/img/flags/492.png diff --git a/webassets/img/flags/493.png b/static/img/flags/493.png similarity index 100% rename from webassets/img/flags/493.png rename to static/img/flags/493.png diff --git a/webassets/img/flags/497.png b/static/img/flags/497.png similarity index 100% rename from webassets/img/flags/497.png rename to static/img/flags/497.png diff --git a/webassets/img/flags/499.png b/static/img/flags/499.png similarity index 100% rename from webassets/img/flags/499.png rename to static/img/flags/499.png diff --git a/webassets/img/flags/5.png b/static/img/flags/5.png similarity index 100% rename from webassets/img/flags/5.png rename to static/img/flags/5.png diff --git a/webassets/img/flags/50.png b/static/img/flags/50.png similarity index 100% rename from webassets/img/flags/50.png rename to static/img/flags/50.png diff --git a/webassets/img/flags/501.png b/static/img/flags/501.png similarity index 100% rename from webassets/img/flags/501.png rename to static/img/flags/501.png diff --git a/webassets/img/flags/502.png b/static/img/flags/502.png similarity index 100% rename from webassets/img/flags/502.png rename to static/img/flags/502.png diff --git a/webassets/img/flags/503.png b/static/img/flags/503.png similarity index 100% rename from webassets/img/flags/503.png rename to static/img/flags/503.png diff --git a/webassets/img/flags/504.png b/static/img/flags/504.png similarity index 100% rename from webassets/img/flags/504.png rename to static/img/flags/504.png diff --git a/webassets/img/flags/505.png b/static/img/flags/505.png similarity index 100% rename from webassets/img/flags/505.png rename to static/img/flags/505.png diff --git a/webassets/img/flags/506.png b/static/img/flags/506.png similarity index 100% rename from webassets/img/flags/506.png rename to static/img/flags/506.png diff --git a/webassets/img/flags/507.png b/static/img/flags/507.png similarity index 100% rename from webassets/img/flags/507.png rename to static/img/flags/507.png diff --git a/webassets/img/flags/508.png b/static/img/flags/508.png similarity index 100% rename from webassets/img/flags/508.png rename to static/img/flags/508.png diff --git a/webassets/img/flags/509.png b/static/img/flags/509.png similarity index 100% rename from webassets/img/flags/509.png rename to static/img/flags/509.png diff --git a/webassets/img/flags/51.png b/static/img/flags/51.png similarity index 100% rename from webassets/img/flags/51.png rename to static/img/flags/51.png diff --git a/webassets/img/flags/510.png b/static/img/flags/510.png similarity index 100% rename from webassets/img/flags/510.png rename to static/img/flags/510.png diff --git a/webassets/img/flags/511.png b/static/img/flags/511.png similarity index 100% rename from webassets/img/flags/511.png rename to static/img/flags/511.png diff --git a/webassets/img/flags/512.png b/static/img/flags/512.png similarity index 100% rename from webassets/img/flags/512.png rename to static/img/flags/512.png diff --git a/webassets/img/flags/513.png b/static/img/flags/513.png similarity index 100% rename from webassets/img/flags/513.png rename to static/img/flags/513.png diff --git a/webassets/img/flags/514.png b/static/img/flags/514.png similarity index 100% rename from webassets/img/flags/514.png rename to static/img/flags/514.png diff --git a/webassets/img/flags/515.png b/static/img/flags/515.png similarity index 100% rename from webassets/img/flags/515.png rename to static/img/flags/515.png diff --git a/webassets/img/flags/516.png b/static/img/flags/516.png similarity index 100% rename from webassets/img/flags/516.png rename to static/img/flags/516.png diff --git a/webassets/img/flags/517.png b/static/img/flags/517.png similarity index 100% rename from webassets/img/flags/517.png rename to static/img/flags/517.png diff --git a/webassets/img/flags/518.png b/static/img/flags/518.png similarity index 100% rename from webassets/img/flags/518.png rename to static/img/flags/518.png diff --git a/webassets/img/flags/519.png b/static/img/flags/519.png similarity index 100% rename from webassets/img/flags/519.png rename to static/img/flags/519.png diff --git a/webassets/img/flags/52.png b/static/img/flags/52.png similarity index 100% rename from webassets/img/flags/52.png rename to static/img/flags/52.png diff --git a/webassets/img/flags/520.png b/static/img/flags/520.png similarity index 100% rename from webassets/img/flags/520.png rename to static/img/flags/520.png diff --git a/webassets/img/flags/521.png b/static/img/flags/521.png similarity index 100% rename from webassets/img/flags/521.png rename to static/img/flags/521.png diff --git a/webassets/img/flags/522.png b/static/img/flags/522.png similarity index 100% rename from webassets/img/flags/522.png rename to static/img/flags/522.png diff --git a/webassets/img/flags/53.png b/static/img/flags/53.png similarity index 100% rename from webassets/img/flags/53.png rename to static/img/flags/53.png diff --git a/webassets/img/flags/54.png b/static/img/flags/54.png similarity index 100% rename from webassets/img/flags/54.png rename to static/img/flags/54.png diff --git a/webassets/img/flags/55.png b/static/img/flags/55.png similarity index 100% rename from webassets/img/flags/55.png rename to static/img/flags/55.png diff --git a/webassets/img/flags/56.png b/static/img/flags/56.png similarity index 100% rename from webassets/img/flags/56.png rename to static/img/flags/56.png diff --git a/webassets/img/flags/57.png b/static/img/flags/57.png similarity index 100% rename from webassets/img/flags/57.png rename to static/img/flags/57.png diff --git a/webassets/img/flags/58.png b/static/img/flags/58.png similarity index 100% rename from webassets/img/flags/58.png rename to static/img/flags/58.png diff --git a/webassets/img/flags/59.png b/static/img/flags/59.png similarity index 100% rename from webassets/img/flags/59.png rename to static/img/flags/59.png diff --git a/webassets/img/flags/6.png b/static/img/flags/6.png similarity index 100% rename from webassets/img/flags/6.png rename to static/img/flags/6.png diff --git a/webassets/img/flags/60.png b/static/img/flags/60.png similarity index 100% rename from webassets/img/flags/60.png rename to static/img/flags/60.png diff --git a/webassets/img/flags/61.png b/static/img/flags/61.png similarity index 100% rename from webassets/img/flags/61.png rename to static/img/flags/61.png diff --git a/webassets/img/flags/62.png b/static/img/flags/62.png similarity index 100% rename from webassets/img/flags/62.png rename to static/img/flags/62.png diff --git a/webassets/img/flags/63.png b/static/img/flags/63.png similarity index 100% rename from webassets/img/flags/63.png rename to static/img/flags/63.png diff --git a/webassets/img/flags/64.png b/static/img/flags/64.png similarity index 100% rename from webassets/img/flags/64.png rename to static/img/flags/64.png diff --git a/webassets/img/flags/65.png b/static/img/flags/65.png similarity index 100% rename from webassets/img/flags/65.png rename to static/img/flags/65.png diff --git a/webassets/img/flags/66.png b/static/img/flags/66.png similarity index 100% rename from webassets/img/flags/66.png rename to static/img/flags/66.png diff --git a/webassets/img/flags/67.png b/static/img/flags/67.png similarity index 100% rename from webassets/img/flags/67.png rename to static/img/flags/67.png diff --git a/webassets/img/flags/68.png b/static/img/flags/68.png similarity index 100% rename from webassets/img/flags/68.png rename to static/img/flags/68.png diff --git a/webassets/img/flags/69.png b/static/img/flags/69.png similarity index 100% rename from webassets/img/flags/69.png rename to static/img/flags/69.png diff --git a/webassets/img/flags/7.png b/static/img/flags/7.png similarity index 100% rename from webassets/img/flags/7.png rename to static/img/flags/7.png diff --git a/webassets/img/flags/70.png b/static/img/flags/70.png similarity index 100% rename from webassets/img/flags/70.png rename to static/img/flags/70.png diff --git a/webassets/img/flags/71.png b/static/img/flags/71.png similarity index 100% rename from webassets/img/flags/71.png rename to static/img/flags/71.png diff --git a/webassets/img/flags/72.png b/static/img/flags/72.png similarity index 100% rename from webassets/img/flags/72.png rename to static/img/flags/72.png diff --git a/webassets/img/flags/74.png b/static/img/flags/74.png similarity index 100% rename from webassets/img/flags/74.png rename to static/img/flags/74.png diff --git a/webassets/img/flags/75.png b/static/img/flags/75.png similarity index 100% rename from webassets/img/flags/75.png rename to static/img/flags/75.png diff --git a/webassets/img/flags/76.png b/static/img/flags/76.png similarity index 100% rename from webassets/img/flags/76.png rename to static/img/flags/76.png diff --git a/webassets/img/flags/77.png b/static/img/flags/77.png similarity index 100% rename from webassets/img/flags/77.png rename to static/img/flags/77.png diff --git a/webassets/img/flags/78.png b/static/img/flags/78.png similarity index 100% rename from webassets/img/flags/78.png rename to static/img/flags/78.png diff --git a/webassets/img/flags/79.png b/static/img/flags/79.png similarity index 100% rename from webassets/img/flags/79.png rename to static/img/flags/79.png diff --git a/webassets/img/flags/8.png b/static/img/flags/8.png similarity index 100% rename from webassets/img/flags/8.png rename to static/img/flags/8.png diff --git a/webassets/img/flags/80.png b/static/img/flags/80.png similarity index 100% rename from webassets/img/flags/80.png rename to static/img/flags/80.png diff --git a/webassets/img/flags/81.png b/static/img/flags/81.png similarity index 100% rename from webassets/img/flags/81.png rename to static/img/flags/81.png diff --git a/webassets/img/flags/82.png b/static/img/flags/82.png similarity index 100% rename from webassets/img/flags/82.png rename to static/img/flags/82.png diff --git a/webassets/img/flags/84.png b/static/img/flags/84.png similarity index 100% rename from webassets/img/flags/84.png rename to static/img/flags/84.png diff --git a/webassets/img/flags/85.png b/static/img/flags/85.png similarity index 100% rename from webassets/img/flags/85.png rename to static/img/flags/85.png diff --git a/webassets/img/flags/86.png b/static/img/flags/86.png similarity index 100% rename from webassets/img/flags/86.png rename to static/img/flags/86.png diff --git a/webassets/img/flags/88.png b/static/img/flags/88.png similarity index 100% rename from webassets/img/flags/88.png rename to static/img/flags/88.png diff --git a/webassets/img/flags/89.png b/static/img/flags/89.png similarity index 100% rename from webassets/img/flags/89.png rename to static/img/flags/89.png diff --git a/webassets/img/flags/9.png b/static/img/flags/9.png similarity index 100% rename from webassets/img/flags/9.png rename to static/img/flags/9.png diff --git a/webassets/img/flags/90.png b/static/img/flags/90.png similarity index 100% rename from webassets/img/flags/90.png rename to static/img/flags/90.png diff --git a/webassets/img/flags/91.png b/static/img/flags/91.png similarity index 100% rename from webassets/img/flags/91.png rename to static/img/flags/91.png diff --git a/webassets/img/flags/93.png b/static/img/flags/93.png similarity index 100% rename from webassets/img/flags/93.png rename to static/img/flags/93.png diff --git a/webassets/img/flags/94.png b/static/img/flags/94.png similarity index 100% rename from webassets/img/flags/94.png rename to static/img/flags/94.png diff --git a/webassets/img/flags/95.png b/static/img/flags/95.png similarity index 100% rename from webassets/img/flags/95.png rename to static/img/flags/95.png diff --git a/webassets/img/flags/96.png b/static/img/flags/96.png similarity index 100% rename from webassets/img/flags/96.png rename to static/img/flags/96.png diff --git a/webassets/img/flags/97.png b/static/img/flags/97.png similarity index 100% rename from webassets/img/flags/97.png rename to static/img/flags/97.png diff --git a/webassets/img/flags/98.png b/static/img/flags/98.png similarity index 100% rename from webassets/img/flags/98.png rename to static/img/flags/98.png diff --git a/webassets/img/flags/99.png b/static/img/flags/99.png similarity index 100% rename from webassets/img/flags/99.png rename to static/img/flags/99.png diff --git a/webassets/img/flags/999.png b/static/img/flags/999.png similarity index 100% rename from webassets/img/flags/999.png rename to static/img/flags/999.png diff --git a/webassets/img/flags/generate.py b/static/img/flags/generate.py similarity index 100% rename from webassets/img/flags/generate.py rename to static/img/flags/generate.py diff --git a/webassets/img/icon-16.png b/static/img/icon-16.png similarity index 100% rename from webassets/img/icon-16.png rename to static/img/icon-16.png diff --git a/webassets/img/icon-192-pwa.png b/static/img/icon-192-pwa.png similarity index 100% rename from webassets/img/icon-192-pwa.png rename to static/img/icon-192-pwa.png diff --git a/webassets/img/icon-192.png b/static/img/icon-192.png similarity index 100% rename from webassets/img/icon-192.png rename to static/img/icon-192.png diff --git a/webassets/img/icon-32.png b/static/img/icon-32.png similarity index 100% rename from webassets/img/icon-32.png rename to static/img/icon-32.png diff --git a/webassets/img/icon-512-pwa.png b/static/img/icon-512-pwa.png similarity index 100% rename from webassets/img/icon-512-pwa.png rename to static/img/icon-512-pwa.png diff --git a/webassets/img/icon-512.png b/static/img/icon-512.png similarity index 100% rename from webassets/img/icon-512.png rename to static/img/icon-512.png diff --git a/webassets/img/logo.png b/static/img/logo.png similarity index 100% rename from webassets/img/logo.png rename to static/img/logo.png diff --git a/webassets/js/add-spot.js b/static/js/add-spot.js similarity index 99% rename from webassets/js/add-spot.js rename to static/js/add-spot.js index 5741d5f..1dbe594 100644 --- a/webassets/js/add-spot.js +++ b/static/js/add-spot.js @@ -209,7 +209,7 @@ function addSpot() { spot["freq"] = parseFloat(freqStr) * 1000; if (mode !== "") spot["mode"] = mode; if (sig !== "") spot["sig"] = sig; - if (sigRef !== "") spot["sig_refs"] = [{id: sigRef}]; + if (sigRef !== "") spot["sig_refs"] = [{sig: sig, id: sigRef}]; if (dxGrid !== "") spot["dx_grid"] = dxGrid; if (comment !== "") spot["comment"] = comment; spot["de_call"] = de; diff --git a/webassets/js/alerts.js b/static/js/alerts.js similarity index 92% rename from webassets/js/alerts.js rename to static/js/alerts.js index 9cb9205..a03b7e7 100644 --- a/webassets/js/alerts.js +++ b/static/js/alerts.js @@ -1,5 +1,7 @@ // How often to query the server? const REFRESH_INTERVAL_SEC = 60 * 10; +// Last time the alerts list was updated on display. +let lastUpdateTime; // Storage for the alert data that the server gives us. let alerts = []; @@ -179,7 +181,7 @@ function addAlertRowsToTable(tbody, alerts) { // Format DX flag let dx_flag = ""; if (a["dx_dxcc_id"] && a["dx_dxcc_id"] != null && a["dx_dxcc_id"] !== 0) { - dx_flag = `${dx_country}`; + dx_flag = `${dx_country}`; } // Format dx calls @@ -306,6 +308,27 @@ function filtersUpdated() { saveSettings(); } +// Update the refresh timing display +function updateRefreshDisplay() { + if (lastUpdateTime != null) { + let secSinceUpdate = moment.duration(moment().diff(lastUpdateTime)).asSeconds(); + let count = REFRESH_INTERVAL_SEC; + let updatingString = "Updating..." + if (secSinceUpdate < REFRESH_INTERVAL_SEC) { + count = REFRESH_INTERVAL_SEC - secSinceUpdate; + let number; + if (count <= 60) { + number = count.toFixed(0); + updatingString = "Updating in " + number + " second" + (number !== "1" ? "s" : "") + "."; + } else { + number = Math.round(count / 60.0).toFixed(0); + updatingString = "Updating in " + number + " minute" + (number !== "1" ? "s" : "") + "."; + } + } + $("#timing-container").html("Last updated at " + lastUpdateTime.format('HH:mm') + " UTC. " + updatingString); + } +} + // Startup $(document).ready(function () { // Call loadOptions(), this will then trigger loading alerts and setting up timers. diff --git a/webassets/js/bands.js b/static/js/bands.js similarity index 85% rename from webassets/js/bands.js rename to static/js/bands.js index ef10f64..15f0ba6 100644 --- a/webassets/js/bands.js +++ b/static/js/bands.js @@ -1,5 +1,9 @@ -// How often to query the server? -const REFRESH_INTERVAL_SEC = 60; +// SSE connection for live spot updates +let evtSource; +let restartSSEOnErrorTimeoutId; +// Debounce timer so rapid SSE bursts only trigger one updateBands() call, as these could trigger a flash of the user +// visible content +let updateBandsDebounceId; // A couple of constants that must match what's in CSS. We need to know them before the content actually renders, so we // can't just ask the elements themselves for their dimensions. @@ -12,18 +16,61 @@ BAND_COLUMN_SPOT_DIV_HEIGHT_PX = BAND_COLUMN_FONT_SIZE * 1.6; // Load spots and populate the bands display. function loadSpots() { + // Close any existing SSE connection before fetching fresh data + if (evtSource != null) { + evtSource.close(); + } $.ajax({url: '/api/v2/spots' + buildQueryString(), dataType: 'json', headers: getCredentialHeaders(), success: function (jsonData) { - // Store last updated time - lastUpdateTime = moment.utc(); - updateRefreshDisplay(); // Store data spots = jsonData; // Update bands display updateBands(); - }}); + // Start the ongoing SSE connection + startSSEConnection(); + }); } -// Build a query string for the API, based on the filters that the user has selected. +// Start an SSE connection to receive new spots as they arrive. +function startSSEConnection() { + if (evtSource != null) { + evtSource.close(); + } + evtSource = new EventSource('/api/v2/spots/stream' + buildQueryString()); + + evtSource.onmessage = function (event) { + const newSpot = JSON.parse(event.data); + + // Replace any existing spot for this callsign + spots = spots.filter(s => newSpot["dx_call"] !== s["dx_call"]); + spots.unshift(newSpot); + + // Debounce. Wait 500ms after the last message before re-rendering to avoid too many ugly flashes + clearTimeout(updateBandsDebounceId); + updateBandsDebounceId = setTimeout(updateBands, 5000); + }; + + evtSource.onerror = function () { + if (evtSource != null) { + evtSource.close(); + } + clearTimeout(restartSSEOnErrorTimeoutId); + restartSSEOnErrorTimeoutId = setTimeout(startSSEConnection, 1000); + }; +} + +// Remove spots from the display that are older than the selected max age. +function expireOldSpots() { + const maxAgeSeconds = parseInt($("#max-spot-age option:selected").val()); + const cutoff = (Date.now() / 1000) - maxAgeSeconds; + const before = spots.length; + spots = spots.filter(s => s["time"] && s["time"] >= cutoff); + if (spots.length !== before) { + updateBands(); + } +} + +// Build a query string for the API, based on the filters that the user has selected. There's no need for credentials +// in the bands page's version of this, because nothing QRZ.com/HamQTH can provide will affect the display. function buildQueryString() { let str = "?"; ["dx_continent", "de_continent", "mode", "source", "band", "sig"].forEach(fn => { @@ -254,22 +301,28 @@ function loadOptions() { // Load settings from settings storage now all the controls are available loadSettings(); - // Load spots and set up the timer + // Load spots and start SSE for live updates loadSpots(); - setInterval(loadSpots, REFRESH_INTERVAL_SEC * 1000); + // Set up spot expiry checker + setInterval(expireOldSpots, 60 * 1000); }); } -// Method called when any display property is changed to reload the map and persist the display settings. +// Method called when any display property is changed to reload the bands display and persist settings. function displayUpdated() { - updateMap(); + updateBands(); saveSettings(); } // Startup $(document).ready(function () { + // Close SSE connection cleanly when navigating away + window.addEventListener('beforeunload', function () { + if (evtSource != null) { + evtSource.close(); + } + }); + // Call loadOptions(), this will then trigger loading spots and setting up timers. loadOptions(); - // Update the refresh timing display every second - setInterval(updateRefreshDisplay, 1000); }); \ No newline at end of file diff --git a/webassets/js/common.js b/static/js/common.js similarity index 91% rename from webassets/js/common.js rename to static/js/common.js index 0b0b43c..a881720 100644 --- a/webassets/js/common.js +++ b/static/js/common.js @@ -1,7 +1,5 @@ // Storage for the options that the server gives us. This will define our filters. let options = {}; -// Last time we updated the spots/alerts list on display. -let lastUpdateTime; // Normally load user settings from local storage, unless embedded mode is in use let useLocalStorage = true; @@ -145,27 +143,6 @@ function toggleFilterButtons(filterQuery, state) { filtersUpdated(); } -// Update the refresh timing display -function updateRefreshDisplay() { - if (lastUpdateTime != null) { - let secSinceUpdate = moment.duration(moment().diff(lastUpdateTime)).asSeconds(); - let count = REFRESH_INTERVAL_SEC; - let updatingString = "Updating..." - if (secSinceUpdate < REFRESH_INTERVAL_SEC) { - count = REFRESH_INTERVAL_SEC - secSinceUpdate; - let number; - if (count <= 60) { - number = count.toFixed(0); - updatingString = "Updating in " + number + " second" + (number !== "1" ? "s" : "") + "."; - } else { - number = Math.round(count / 60.0).toFixed(0); - updatingString = "Updating in " + number + " minute" + (number !== "1" ? "s" : "") + "."; - } - } - $("#timing-container").html("Last updated at " + lastUpdateTime.format('HH:mm') + " UTC. " + updatingString); - } -} - // When the "use local time" field is changed, reload the table and save settings function timeZoneUpdated() { updateTable(); diff --git a/webassets/js/conditions.js b/static/js/conditions.js similarity index 100% rename from webassets/js/conditions.js rename to static/js/conditions.js diff --git a/webassets/js/geo.js b/static/js/geo.js similarity index 100% rename from webassets/js/geo.js rename to static/js/geo.js diff --git a/webassets/js/map.js b/static/js/map.js similarity index 73% rename from webassets/js/map.js rename to static/js/map.js index 9e25907..280ce00 100644 --- a/webassets/js/map.js +++ b/static/js/map.js @@ -1,6 +1,3 @@ -// How often to query the server? -const REFRESH_INTERVAL_SEC = 60; - // Colours const MAIDENHEAD_GRID_COLOR_LIGHT = 'rgba(200, 140, 140, 1.0)'; const CQ_ZONES_COLOR_LIGHT = 'rgba(140, 200, 140, 1.0)'; @@ -11,6 +8,15 @@ const CQ_ZONES_COLOR_DARK = 'rgba(60, 120, 60, 1.0)'; const ITU_ZONES_COLOR_DARK = 'rgba(120, 120, 60, 1.0)'; const WAB_WAI_GRID_COLOR_DARK = 'rgba(60, 60, 120, 1.0)'; +// SSE connection for live spot updates +let evtSource; +let restartSSEOnErrorTimeoutId; +// Map dx_call to a pair of marker & geodesic line, so we can de-duplicate spots as they arrive and only show +// one marker per dx_call. The key here is actually dx_call + SSID if the spot gives us an SSID; this is mostly for +// if APRS spots are enabled so we can ID a home and mobile station separately rather than our marker oscillating +// between the two as updates come in. +let spotMarkers = new Map(); + // Map layers let backgroundTileLayer; let markersLayer; @@ -28,6 +34,17 @@ let firstLoad = true; // Load spots and populate the map. function loadSpots() { + // Close any existing SSE connection before fetching fresh data + if (evtSource != null) { + evtSource.close(); + } + // Including QRZ/HamQTH lookups to improve positions causes the load to be really slow, so on first load + // the user would be waiting ages without data and will think it's broken. We therefore load in several + // stages: + // 1) Load without any credentials + // 2) (If we have credentials) reload with them, replacing what's already there, + // 3) Subscribe to the SSE endpoint (with credentials if we have them) so that updates come with augmented + // data if they can. $.ajax({url: '/api/v2/spots' + buildQueryString(), dataType: 'json', headers: getCredentialHeaders(), success: function (jsonData) { // Store data spots = jsonData; @@ -36,9 +53,115 @@ function loadSpots() { if ($("#showTerminator")[0].checked) { terminator.setTime(); } + // Check if we have any credentials to use + if (getCredentialQueryString() !== "") { + // OK, we have credentials and have loaded once without them so the user has a basic map. Now reload + // with the credentials and replace what's on the map, so we can improve the data. + $.getJSON('/api/v1/spots' + buildQueryString(true), function (jsonData2) { + spots = jsonData2; + updateMap(); + // Now start the ongoing SSE connection + startSSEConnection(); + }); + } else { + // We had no credentials with which to augment the data anyway, so just start the SSE connection + // now + startSSEConnection(); + } }}); } +// Start the SSE connection to receive new spots as they arrive +function startSSEConnection() { + if (evtSource != null) { + evtSource.close(); + } + // SSE is going to fetch only a few spots at a time, so now we include QRZ/HamQTH credentials because the delay won't be significant. + evtSource = new EventSource('/api/v1/spots/stream' + buildQueryString(true)); + + evtSource.onmessage = function (event) { + const newSpot = JSON.parse(event.data); + const key = spotKey(newSpot); + + // Remove existing marker/geodesic for this callsign if present + removeSpotFromMap(key); + spots = spots.filter(s => spotKey(s) !== key); + + // Skip spots with no map coordinates + if (newSpot["dx_latitude"] == null || newSpot["dx_longitude"] == null) { + return; + } + + // Add to data store and map + spots.unshift(newSpot); + addSpotToMap(newSpot); + }; + + evtSource.onerror = function () { + if (evtSource != null) { + evtSource.close(); + } + clearTimeout(restartSSEOnErrorTimeoutId); + restartSSEOnErrorTimeoutId = setTimeout(startSSEConnection, 1000); + }; +} + +// Remove spots from the map that are older than the selected max age. +function expireOldSpots() { + const maxAgeSeconds = parseInt($("#max-spot-age option:selected").val()); + const cutoff = (Date.now() / 1000) - maxAgeSeconds; + spots = spots.filter(function (s) { + if (s["time"] && s["time"] < cutoff) { + removeSpotFromMap(spotKey(s)); + return false; + } + return true; + }); +} + +// Returns a unique key for a spot based on its callsign and SSID. This is in case the APRS spot source is enabled and +// we want to display all SSIDs for a given callsign, not just the latest one. +function spotKey(s) { + return s["dx_call"] + (s["dx_ssid"] ? "-" + s["dx_ssid"] : ""); +} + +// Add a single spot's marker and geodesic to the map and to spotMarkers +function addSpotToMap(s) { + const m = L.marker([s["dx_latitude"], s["dx_longitude"]], {icon: getIcon(s)}); + m.bindPopup(getTooltipText(s)); + markersLayer.addLayer(m); + oms.addMarker(m); + + let geodesic = null; + if ($("#mapShowGeodesics")[0].checked && s["de_latitude"] != null && s["de_longitude"] != null) { + try { + geodesic = L.geodesic([[s["de_latitude"], s["de_longitude"]], m.getLatLng()], { + color: bandToColor(s['band']), + wrap: false, + steps: 5 + }); + geodesicsLayer.addLayer(geodesic); + } catch (e) { + // Not sure what causes these but better to continue than to crash out + } + } + + spotMarkers.set(spotKey(s), {marker: m, geodesic: geodesic}); +} + +// Remove a spot's marker and geodesic from the map and from spotMarkers. +function removeSpotFromMap(key) { + const entry = spotMarkers.get(key); + if (entry) { + markersLayer.removeLayer(entry.marker); + oms.removeMarker(entry.marker); + if (entry.geodesic) { + geodesicsLayer.removeLayer(entry.geodesic); + } + spotMarkers.delete(key); + } +} + // Build a query string for the API, based on the filters that the user has selected. function buildQueryString() { let str = "?"; @@ -59,27 +182,14 @@ function updateMap() { markersLayer.clearLayers(); geodesicsLayer.clearLayers(); oms.clearMarkers(); + spotMarkers.clear(); - // Make new markers for all spots that match the filter + // Make new markers for all spots spots.forEach(function (s) { - const m = L.marker([s["dx_latitude"], s["dx_longitude"]], {icon: getIcon(s)}); - m.bindPopup(getTooltipText(s)); - markersLayer.addLayer(m); - oms.addMarker(m); - - // Create geodesics if required - if ($("#mapShowGeodesics")[0].checked && s["de_latitude"] != null && s["de_longitude"] != null) { - try { - const geodesic = L.geodesic([[s["de_latitude"], s["de_longitude"]], m.getLatLng()], { - color: bandToColor(s['band']), - wrap: false, - steps: 5 - }); - geodesicsLayer.addLayer(geodesic); - } catch (e) { - // Not sure what causes these but better to continue than to crash out - } + if (s["dx_latitude"] == null || s["dx_longitude"] == null) { + return; } + addSpotToMap(s); }); // On first load, zoom to the extent of the markers @@ -236,9 +346,10 @@ function loadOptions() { enableITUZones($("#showITUZones")[0].checked); enableWABWAIGrid($("#showWABWAIGrid")[0].checked); - // Load spots and set up the timer + // Load spots and start SSE for live updates loadSpots(); - setInterval(loadSpots, REFRESH_INTERVAL_SEC * 1000); + // Set up spot expiry checker + setInterval(expireOldSpots, 60 * 1000); }); } @@ -478,14 +589,33 @@ function setUpMap() { map.setView([30, 0], 3); } +// Display the intro box, unless the user has already dismissed it once. +function displayIntroBox() { + if (localStorage.getItem("map-intro-box-dismissed") == null) { + $("#map-intro-box").show(); + } + $("#map-intro-box-dismiss").click(function () { + localStorage.setItem("map-intro-box-dismissed", true); + }); +} + // Startup $(document).ready(function () { + // Close SSE connection cleanly when navigating away + window.addEventListener('beforeunload', function () { + if (evtSource != null) { + evtSource.close(); + } + }); + // Hide the extra things that need to be hidden on this page $(".hideonmap").hide(); // Set up map setUpMap(); // Call loadOptions(), this will then trigger loading spots and setting up timers. loadOptions(); + // Display intro box + displayIntroBox(); // Prevent mouse scroll and touch actions in the popup menus being passed through to the map L.DomEvent.disableScrollPropagation(document.getElementById('settingsButtonRowMap')); L.DomEvent.disableClickPropagation(document.getElementById('settingsButtonRowMap')); diff --git a/webassets/js/spots.js b/static/js/spots.js similarity index 98% rename from webassets/js/spots.js rename to static/js/spots.js index 5586865..2907dc8 100644 --- a/webassets/js/spots.js +++ b/static/js/spots.js @@ -246,7 +246,7 @@ function createNewTableRowsForSpot(s, highlightNew) { dx_flag = ""; } if (s["dx_dxcc_id"] && s["dx_dxcc_id"] != null && s["dx_dxcc_id"] !== 0) { - dx_flag = `${dx_country}`; + dx_flag = `${dx_country}`; } // Format the frequency @@ -316,7 +316,7 @@ function createNewTableRowsForSpot(s, highlightNew) { // Format DE flag let de_flag = ""; if (s["de_dxcc_id"] && s["de_dxcc_id"] != null && s["de_dxcc_id"] !== 0) { - de_flag = `${de_country}`; + de_flag = `${de_country}`; } // Format de call diff --git a/webassets/js/spotsbandsandmap.js b/static/js/spotsbandsandmap.js similarity index 100% rename from webassets/js/spotsbandsandmap.js rename to static/js/spotsbandsandmap.js diff --git a/webassets/js/status.js b/static/js/status.js similarity index 100% rename from webassets/js/status.js rename to static/js/status.js diff --git a/webassets/js/ui-ham.js b/static/js/ui-ham.js similarity index 100% rename from webassets/js/ui-ham.js rename to static/js/ui-ham.js diff --git a/webassets/js/utils.js b/static/js/utils.js similarity index 100% rename from webassets/js/utils.js rename to static/js/utils.js diff --git a/static/service-worker.js b/static/service-worker.js new file mode 100644 index 0000000..8340792 --- /dev/null +++ b/static/service-worker.js @@ -0,0 +1,78 @@ +const CACHE_NAME = 'Spothole'; +const CACHE_URLS = [ + 'static/apidocs/openapi.yml', + 'static/audio/ping.mp3', + 'static/css/style.css', + 'static/js/add-spot.js', + 'static/js/alerts.js', + 'static/js/bands.js', + 'static/js/common.js', + 'static/js/geo.js', + 'static/js/map.js', + 'static/js/spots.js', + 'static/js/spotsbandsandmap.js', + 'static/js/status.js', + 'static/js/ui-ham.js', + 'static/js/utils.js', + 'static/img/logo.png', + 'static/img/favicon.ico', + 'static/img/icon-32.png', + 'static/img/icon-192.png', + 'static/img/icon-512.png', + 'static/vendor/img/markers_default@2x.png', + 'static/vendor/img/markers_shadow.png', + 'static/vendor/img/markers_default.png', + 'static/vendor/img/markers_shadow@2x.png', + 'static/vendor/css/bootstrap-5.3.8.min.css', + 'static/vendor/css/images/marker-icon.png', + 'static/vendor/css/images/layers-2x.png', + 'static/vendor/css/images/marker-shadow.png', + 'static/vendor/css/images/layers.png', + 'static/vendor/css/images/marker-icon-2x.png', + 'static/vendor/css/leaflet-extra-markers-1.2.2.min.css', + 'static/vendor/css/leaflet-1.9.4.min.css', + 'static/vendor/css/fontawesome-6.7.2.min.css', + 'static/vendor/css/solid-6.7.2.min.css', + 'static/vendor/js/jquery-3.7.1.min.js', + 'static/vendor/js/chart-4.4.9.umd.min.js', + 'static/vendor/js/oms-leaflet-0.2.7.min.js', + 'static/vendor/js/leaflet-workedallbritainireland.js', + 'static/vendor/js/leaflet-providers-2.0.0.js', + 'static/vendor/js/text-image-0.7.0.js', + 'static/vendor/js/leaflet-vectorgrid-1.3.0.js', + 'static/vendor/js/leaflet-ituzones.js', + 'static/vendor/js/tinycolor2-1.6.0.min.js', + 'static/vendor/js/bootstrap-5.3.8.bundle.min.js', + 'static/vendor/js/moment-2.29.4.min.js', + 'static/vendor/js/leaflet-cqzones.js', + 'static/vendor/js/leaflet-extra-markers-1.2.2.min.js', + 'static/vendor/js/leaflet-terminator-1.1.0.min.js', + 'static/vendor/js/leaflet-1.9.4.min.js', + 'static/vendor/js/leaflet-geodesic-2.7.2.umd.min.js', + 'static/vendor/js/leaflet-maidenhead.js', + 'static/vendor/webfonts/fa-solid-900.ttf', + 'static/vendor/webfonts/fa-solid-900.woff2' +]; + +self.addEventListener('fetch', (event) => { + // Is this an asset we can cache? + const url = new URL(event.request.url); + const isCacheableRequest = CACHE_URLS.includes(url.pathname); + + if (isCacheableRequest) { + // Open the cache + event.respondWith(caches.open(CACHE_NAME).then((cache) => { + // Go to the network first, cacheing the response + return fetch(event.request.url).then((fetchedResponse) => { + cache.put(event.request, fetchedResponse.clone()); + + return fetchedResponse; + }).catch(() => { + // If the network is unavailable, get from cache. + return cache.match(event.request.url); + }); + })); + } else { + // Not a cacheable request, must be a call to the API, so no cache involved just go to the network + } +}); diff --git a/webassets/vendor/css/bootstrap-5.3.8.min.css b/static/vendor/css/bootstrap-5.3.8.min.css similarity index 100% rename from webassets/vendor/css/bootstrap-5.3.8.min.css rename to static/vendor/css/bootstrap-5.3.8.min.css diff --git a/webassets/vendor/css/fontawesome-6.7.2.min.css b/static/vendor/css/fontawesome-6.7.2.min.css similarity index 100% rename from webassets/vendor/css/fontawesome-6.7.2.min.css rename to static/vendor/css/fontawesome-6.7.2.min.css diff --git a/webassets/vendor/css/images/layers-2x.png b/static/vendor/css/images/layers-2x.png similarity index 100% rename from webassets/vendor/css/images/layers-2x.png rename to static/vendor/css/images/layers-2x.png diff --git a/webassets/vendor/css/images/layers.png b/static/vendor/css/images/layers.png similarity index 100% rename from webassets/vendor/css/images/layers.png rename to static/vendor/css/images/layers.png diff --git a/webassets/vendor/css/images/marker-icon-2x.png b/static/vendor/css/images/marker-icon-2x.png similarity index 100% rename from webassets/vendor/css/images/marker-icon-2x.png rename to static/vendor/css/images/marker-icon-2x.png diff --git a/webassets/vendor/css/images/marker-icon.png b/static/vendor/css/images/marker-icon.png similarity index 100% rename from webassets/vendor/css/images/marker-icon.png rename to static/vendor/css/images/marker-icon.png diff --git a/webassets/vendor/css/images/marker-shadow.png b/static/vendor/css/images/marker-shadow.png similarity index 100% rename from webassets/vendor/css/images/marker-shadow.png rename to static/vendor/css/images/marker-shadow.png diff --git a/webassets/vendor/css/leaflet-1.9.4.min.css b/static/vendor/css/leaflet-1.9.4.min.css similarity index 100% rename from webassets/vendor/css/leaflet-1.9.4.min.css rename to static/vendor/css/leaflet-1.9.4.min.css diff --git a/webassets/vendor/css/leaflet-extra-markers-1.2.2.min.css b/static/vendor/css/leaflet-extra-markers-1.2.2.min.css similarity index 100% rename from webassets/vendor/css/leaflet-extra-markers-1.2.2.min.css rename to static/vendor/css/leaflet-extra-markers-1.2.2.min.css diff --git a/webassets/vendor/css/solid-6.7.2.min.css b/static/vendor/css/solid-6.7.2.min.css similarity index 100% rename from webassets/vendor/css/solid-6.7.2.min.css rename to static/vendor/css/solid-6.7.2.min.css diff --git a/webassets/vendor/img/markers_default.png b/static/vendor/img/markers_default.png similarity index 100% rename from webassets/vendor/img/markers_default.png rename to static/vendor/img/markers_default.png diff --git a/webassets/vendor/img/markers_default@2x.png b/static/vendor/img/markers_default@2x.png similarity index 100% rename from webassets/vendor/img/markers_default@2x.png rename to static/vendor/img/markers_default@2x.png diff --git a/webassets/vendor/img/markers_shadow.png b/static/vendor/img/markers_shadow.png similarity index 100% rename from webassets/vendor/img/markers_shadow.png rename to static/vendor/img/markers_shadow.png diff --git a/webassets/vendor/img/markers_shadow@2x.png b/static/vendor/img/markers_shadow@2x.png similarity index 100% rename from webassets/vendor/img/markers_shadow@2x.png rename to static/vendor/img/markers_shadow@2x.png diff --git a/webassets/vendor/js/bootstrap-5.3.8.bundle.min.js b/static/vendor/js/bootstrap-5.3.8.bundle.min.js similarity index 100% rename from webassets/vendor/js/bootstrap-5.3.8.bundle.min.js rename to static/vendor/js/bootstrap-5.3.8.bundle.min.js diff --git a/webassets/vendor/js/chart-4.4.9.umd.min.js b/static/vendor/js/chart-4.4.9.umd.min.js similarity index 100% rename from webassets/vendor/js/chart-4.4.9.umd.min.js rename to static/vendor/js/chart-4.4.9.umd.min.js diff --git a/webassets/vendor/js/jquery-3.7.1.min.js b/static/vendor/js/jquery-3.7.1.min.js similarity index 100% rename from webassets/vendor/js/jquery-3.7.1.min.js rename to static/vendor/js/jquery-3.7.1.min.js diff --git a/webassets/vendor/js/leaflet-1.9.4.min.js b/static/vendor/js/leaflet-1.9.4.min.js similarity index 100% rename from webassets/vendor/js/leaflet-1.9.4.min.js rename to static/vendor/js/leaflet-1.9.4.min.js diff --git a/webassets/vendor/js/leaflet-cqzones.js b/static/vendor/js/leaflet-cqzones.js similarity index 100% rename from webassets/vendor/js/leaflet-cqzones.js rename to static/vendor/js/leaflet-cqzones.js diff --git a/webassets/vendor/js/leaflet-extra-markers-1.2.2.min.js b/static/vendor/js/leaflet-extra-markers-1.2.2.min.js similarity index 100% rename from webassets/vendor/js/leaflet-extra-markers-1.2.2.min.js rename to static/vendor/js/leaflet-extra-markers-1.2.2.min.js diff --git a/webassets/vendor/js/leaflet-geodesic-2.7.2.umd.min.js b/static/vendor/js/leaflet-geodesic-2.7.2.umd.min.js similarity index 100% rename from webassets/vendor/js/leaflet-geodesic-2.7.2.umd.min.js rename to static/vendor/js/leaflet-geodesic-2.7.2.umd.min.js diff --git a/webassets/vendor/js/leaflet-ituzones.js b/static/vendor/js/leaflet-ituzones.js similarity index 100% rename from webassets/vendor/js/leaflet-ituzones.js rename to static/vendor/js/leaflet-ituzones.js diff --git a/webassets/vendor/js/leaflet-maidenhead.js b/static/vendor/js/leaflet-maidenhead.js similarity index 100% rename from webassets/vendor/js/leaflet-maidenhead.js rename to static/vendor/js/leaflet-maidenhead.js diff --git a/webassets/vendor/js/leaflet-providers-2.0.0.js b/static/vendor/js/leaflet-providers-2.0.0.js similarity index 100% rename from webassets/vendor/js/leaflet-providers-2.0.0.js rename to static/vendor/js/leaflet-providers-2.0.0.js diff --git a/webassets/vendor/js/leaflet-terminator-1.1.0.min.js b/static/vendor/js/leaflet-terminator-1.1.0.min.js similarity index 100% rename from webassets/vendor/js/leaflet-terminator-1.1.0.min.js rename to static/vendor/js/leaflet-terminator-1.1.0.min.js diff --git a/webassets/vendor/js/leaflet-vectorgrid-1.3.0.js b/static/vendor/js/leaflet-vectorgrid-1.3.0.js similarity index 100% rename from webassets/vendor/js/leaflet-vectorgrid-1.3.0.js rename to static/vendor/js/leaflet-vectorgrid-1.3.0.js diff --git a/webassets/vendor/js/leaflet-workedallbritainireland.js b/static/vendor/js/leaflet-workedallbritainireland.js similarity index 100% rename from webassets/vendor/js/leaflet-workedallbritainireland.js rename to static/vendor/js/leaflet-workedallbritainireland.js diff --git a/webassets/vendor/js/moment-2.29.4.min.js b/static/vendor/js/moment-2.29.4.min.js similarity index 100% rename from webassets/vendor/js/moment-2.29.4.min.js rename to static/vendor/js/moment-2.29.4.min.js diff --git a/webassets/vendor/js/oms-leaflet-0.2.7.min.js b/static/vendor/js/oms-leaflet-0.2.7.min.js similarity index 100% rename from webassets/vendor/js/oms-leaflet-0.2.7.min.js rename to static/vendor/js/oms-leaflet-0.2.7.min.js diff --git a/webassets/vendor/js/text-image-0.7.0.js b/static/vendor/js/text-image-0.7.0.js similarity index 100% rename from webassets/vendor/js/text-image-0.7.0.js rename to static/vendor/js/text-image-0.7.0.js diff --git a/webassets/vendor/js/tinycolor2-1.6.0.min.js b/static/vendor/js/tinycolor2-1.6.0.min.js similarity index 100% rename from webassets/vendor/js/tinycolor2-1.6.0.min.js rename to static/vendor/js/tinycolor2-1.6.0.min.js diff --git a/webassets/vendor/webfonts/fa-solid-900.ttf b/static/vendor/webfonts/fa-solid-900.ttf similarity index 100% rename from webassets/vendor/webfonts/fa-solid-900.ttf rename to static/vendor/webfonts/fa-solid-900.ttf diff --git a/webassets/vendor/webfonts/fa-solid-900.woff2 b/static/vendor/webfonts/fa-solid-900.woff2 similarity index 100% rename from webassets/vendor/webfonts/fa-solid-900.woff2 rename to static/vendor/webfonts/fa-solid-900.woff2 diff --git a/templates/about.html b/templates/about.html index 5982461..bbeda55 100644 --- a/templates/about.html +++ b/templates/about.html @@ -159,13 +159,22 @@ modify it however you like, you can claim you wrote it and charge people £1000 for a copy, I don't really mind. (Please don't do the last one. But if you're using my code for something cool, it would be nice to hear from you!)

-

Data Accuracy

+

Data Accuracy

Please note that the data coming out of Spothole is only as good as the data going in. People mis-hear and make typos when spotting callsigns all the time. There are also plenty of cases where Spothole's data, particularly location data, may be inaccurate. For example, there are POTA parks that span multiple US states, countries that span multiple CQ zones, portable operators with no requirement to sign /P, etc. If you are doing something where accuracy is important, such as contesting, you should not rely on Spothole's data to fill in any gaps in your log.

+

In the Spothole user interface, under "Your Data", you can enter your credentials for QRZ.com and/or HamQTH if + you have them. This allows Spothole to augment its data with lookups from these services. See the following + section, Privacy, for details of how these are handled. If you are looking at the map and see lots of spots in + the geographic centre of countries, allowing these lookups will help. For QRZ.com a paid account is required to + look up operator locations.

+

IOTA in particular causes a mapping problem, firstly that there is no official set of geodata beyond the names of + islands. We also have no way of telling whether priority should be given to IOTA reference or e.g. QRZ lookup, + for example for a G callsign in EU-005 Great Britain the QRZ home address is probably more accurate, but in + EU-002 Aaland islands, it's more likely a DXpedition and the IOTA would be more accurate.

Privacy

Spothole collects no data about you on a permanent basis. All spots and alerts are "timed out" and deleted from the system after a set interval, which by default is one hour for spots and one week for alerts.

diff --git a/templates/add_spot.html b/templates/add_spot.html index 88ffc88..4339002 100644 --- a/templates/add_spot.html +++ b/templates/add_spot.html @@ -76,7 +76,7 @@ - + diff --git a/templates/alerts.html b/templates/alerts.html index f3720ca..75f3540 100644 --- a/templates/alerts.html +++ b/templates/alerts.html @@ -75,7 +75,7 @@ - + diff --git a/templates/api_only_home.html b/templates/api_only_home.html index 97139c9..8d8782c 100644 --- a/templates/api_only_home.html +++ b/templates/api_only_home.html @@ -1,13 +1,13 @@ {% extends "skeleton.html" %} {% block head_extra %} - + {% end %} {% block body %}
- Spothole + Spothole
diff --git a/templates/apidocs.html b/templates/apidocs.html index 3f4325d..9266dcb 100644 --- a/templates/apidocs.html +++ b/templates/apidocs.html @@ -1,5 +1,5 @@ {% extends "skeleton.html" %} {% block body %} - + {% end %} diff --git a/templates/bands.html b/templates/bands.html index 758c822..5c181bd 100644 --- a/templates/bands.html +++ b/templates/bands.html @@ -3,9 +3,7 @@
-
- {% module Template("widgets/refresh-timer.html", web_ui_options=web_ui_options) %} -
+
{% module Template("widgets/filters-display-data-buttons.html", web_ui_options=web_ui_options) %} @@ -77,8 +75,8 @@ - - + + diff --git a/templates/base.html b/templates/base.html index 375148b..9dd03cb 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,26 +1,26 @@ {% extends "skeleton.html" %} {% block head_extra %} - - - - + + + + - - - - + + + + - - - - + + + + {% end %} {% block body %}
- - + + diff --git a/webassets/manifest.webmanifest b/templates/manifest.webmanifest similarity index 81% rename from webassets/manifest.webmanifest rename to templates/manifest.webmanifest index be33ce4..1d07df2 100644 --- a/webassets/manifest.webmanifest +++ b/templates/manifest.webmanifest @@ -5,7 +5,7 @@ "short_name": "Spothole", "scope": "/", "display": "standalone", - "start_url": "https://spothole.app/", + "start_url": "{{ baseurl }}", "background_color": "white", "theme_color": "white", "description": "An Amateur Radio spotting tool bringing together DX clusters and outdoor programmes, providing a universal JSON API and web interface.", @@ -13,17 +13,17 @@ "prefer_related_applications": false, "icons": [ { - "src": "/img/icon-192-pwa.png", + "src": "/static/img/icon-192-pwa.png", "type": "image/png", "sizes": "192x192", "purpose": "maskable" }, { - "src": "/img/icon-512-pwa.png", + "src": "/static/img/icon-512-pwa.png", "type": "image/png", "sizes": "512x512", "purpose": "maskable" } ], - "url": "https://spothole.app" + "url": "{{ baseurl }}" } diff --git a/templates/map.html b/templates/map.html index a21d7bf..ad5e684 100644 --- a/templates/map.html +++ b/templates/map.html @@ -1,6 +1,19 @@ {% extends "base.html" %} {% block content %} +
+ +
+
@@ -77,26 +90,26 @@
- - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + + diff --git a/templates/skeleton.html b/templates/skeleton.html index bdbcd36..73dc006 100644 --- a/templates/skeleton.html +++ b/templates/skeleton.html @@ -15,10 +15,10 @@ content="An Amateur Radio spotting tool bringing together DX clusters and outdoor programmes, providing a universal JSON API and web interface."/> - - - - + + + + @@ -26,14 +26,14 @@ Spothole - - - - - - + + + + + + - + {% block head_extra %}{% end %} diff --git a/templates/spots.html b/templates/spots.html index 4b26010..b659a50 100644 --- a/templates/spots.html +++ b/templates/spots.html @@ -116,8 +116,8 @@ - - + + diff --git a/templates/status.html b/templates/status.html index 3e8cfd9..8b0a5ea 100644 --- a/templates/status.html +++ b/templates/status.html @@ -59,7 +59,7 @@
- +