diff --git a/rocketpy/environment/fetchers/__init__.py b/rocketpy/environment/fetchers/__init__.py index d74ea55a9..067208717 100644 --- a/rocketpy/environment/fetchers/__init__.py +++ b/rocketpy/environment/fetchers/__init__.py @@ -32,6 +32,19 @@ fetch_open_meteo_ensemble, fetch_open_meteo_forecast, ) +from rocketpy.environment.fetchers.noaa_catalog import ( + FORECAST_MODELS_CATALOG_URL, + NOAA_MODEL_COLLECTIONS, + THREDDS_OPENDAP_ROOT, + build_noaa_opendap_url, + collection_catalog_url, + fetch_latest_noaa_dataset, + get_latest_noaa_dataset_identifier, + get_latest_noaa_opendap_url, + list_noaa_atmosphere_datasets, + list_noaa_dataset_identifiers, + resolve_noaa_collection_path, +) from rocketpy.environment.fetchers.opendap_fetchers import ( fetch_aigfs_file_return_dataset, fetch_cmc_ensemble, @@ -51,9 +64,11 @@ __all__ = [ "MAX_RETRY_DELAY_SECONDS", + "FORECAST_MODELS_CATALOG_URL", "METEOMATICS_BASE_URL", "METEOMATICS_LOGIN_URL", "METEOMATICS_TIMEOUT_SECONDS", + "NOAA_MODEL_COLLECTIONS", "OPEN_METEO_ENSEMBLE_MODELS", "OPEN_METEO_ENSEMBLE_URL", "OPEN_METEO_FORECAST_URL", @@ -61,8 +76,11 @@ "OPEN_METEO_HISTORICAL_URL", "OPEN_METEO_PRESSURE_LEVELS", "OPEN_METEO_TIMEOUT_SECONDS", + "THREDDS_OPENDAP_ROOT", "MeteomaticsFetcher", "build_hourly_variables", + "build_noaa_opendap_url", + "collection_catalog_url", "fetch_aigfs_file_return_dataset", "fetch_atmospheric_data_from_meteomatics", "fetch_atmospheric_data_from_windy", @@ -71,6 +89,7 @@ "fetch_gfs_file_return_dataset", "fetch_hiresw_file_return_dataset", "fetch_hrrr_file_return_dataset", + "fetch_latest_noaa_dataset", "fetch_meteomatics_token", "fetch_nam_file_return_dataset", "fetch_open_elevation", @@ -78,8 +97,13 @@ "fetch_open_meteo_forecast", "fetch_rap_file_return_dataset", "fetch_wyoming_sounding", + "get_latest_noaa_dataset_identifier", + "get_latest_noaa_opendap_url", + "list_noaa_atmosphere_datasets", + "list_noaa_dataset_identifiers", "logger", "netCDF4", "requests", + "resolve_noaa_collection_path", "time", ] diff --git a/rocketpy/environment/fetchers/noaa_catalog.py b/rocketpy/environment/fetchers/noaa_catalog.py new file mode 100644 index 000000000..20a7e3262 --- /dev/null +++ b/rocketpy/environment/fetchers/noaa_catalog.py @@ -0,0 +1,412 @@ +"""Best-effort helpers for listing NOAA/NCEP atmosphere datasets on THREDDS. + +RocketPy's forecast shortcuts historically hard-coded OPeNDAP URLs (and, for +NOMADS GrADS models, probed calendar times). NOMADS OPeNDAP has been retired, +so discovery is based on the Unidata THREDDS catalogs that already back the +GFS/NAM/RAP/HRRR/AIGFS fetchers. + +Limitations +----------- +- Catalog layout can change without notice; treat results as advisory. +- Listing uses HTTP XML catalogs, not NOMADS OpenDAP directory pages. +- ``fetch_latest_noaa_dataset`` opens the resolved OPeNDAP URL via netCDF4 + (same pattern as the existing ``fetch_*_file_return_dataset`` helpers). It + does not write a local GRIB/NetCDF file to disk. +""" + +from __future__ import annotations + +import re +import time +import xml.etree.ElementTree as ET +from datetime import datetime +from urllib.parse import urljoin, urlparse + +import netCDF4 +import requests + +from rocketpy.environment.fetchers.base import MAX_RETRY_DELAY_SECONDS + +THREDDS_ROOT = "https://thredds.ucar.edu/thredds" +THREDDS_CATALOG_ROOT = f"{THREDDS_ROOT}/catalog/" +THREDDS_OPENDAP_ROOT = f"{THREDDS_ROOT}/dodsC/" +FORECAST_MODELS_CATALOG_URL = f"{THREDDS_CATALOG_ROOT}idd/forecastModels.xml" + +# Collections used by Environment's latest-model shortcuts. +NOAA_MODEL_COLLECTIONS = { + "GFS": "grib/NCEP/GFS/Global_0p25deg", + "NAM": "grib/NCEP/NAM/CONUS_12km", + "RAP": "grib/NCEP/RAP/CONUS_13km", + "HRRR": "grib/NCEP/HRRR/CONUS_2p5km", + "AIGFS": "grib/NCEP/AIGFS/Global_0p25deg", +} + +_THREDDS_NS = { + "thredds": "http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0", + "xlink": "http://www.w3.org/1999/xlink", +} +_RUN_TIMESTAMP_RE = re.compile(r"(?P\d{8}_\d{4})") +_DEFAULT_TIMEOUT_SECONDS = 30 + + +def _local_tag(tag: str) -> str: + """Strip Clark-notation namespace from an ElementTree tag.""" + if "}" in tag: + return tag.rsplit("}", 1)[-1] + return tag + + +def _absolute_catalog_url(href: str, base_url: str) -> str: + """Resolve a catalogRef href against a catalog URL.""" + if href.startswith("http://") or href.startswith("https://"): + return href + if href.startswith("/"): + parsed = urlparse(base_url) + return f"{parsed.scheme}://{parsed.netloc}{href}" + return urljoin(base_url, href) + + +def _collection_path_from_catalog_url(catalog_url: str) -> str | None: + """Extract ``grib/NCEP/...`` path from a THREDDS catalog URL, if present.""" + marker = "/catalog/" + if marker not in catalog_url: + return None + path = catalog_url.split(marker, 1)[1] + if path.endswith("/catalog.xml"): + path = path[: -len("/catalog.xml")] + elif path.endswith("catalog.xml"): + path = path[: -len("catalog.xml")].rstrip("/") + return path or None + + +def _fetch_catalog_xml(catalog_url: str, timeout: float = _DEFAULT_TIMEOUT_SECONDS): + """GET a THREDDS catalog XML document and return its root element.""" + try: + response = requests.get(catalog_url, timeout=timeout) + except requests.exceptions.RequestException as exc: + raise RuntimeError( + f"Unable to reach NOAA/THREDDS catalog at {catalog_url}." + ) from exc + + if response.status_code != 200: + raise RuntimeError( + "Unable to list NOAA/THREDDS datasets: " + f"HTTP {response.status_code} for {catalog_url}." + ) + + try: + return ET.fromstring(response.content) + except ET.ParseError as exc: + raise RuntimeError( + f"Invalid THREDDS catalog XML received from {catalog_url}." + ) from exc + + +def _opendap_url_from_path(url_path: str) -> str: + """Build an OPeNDAP URL for a THREDDS dataset ``urlPath``.""" + return urljoin(THREDDS_OPENDAP_ROOT, url_path.lstrip("/")) + + +def _run_sort_key(identifier: str): + """Sort key that prefers identifiers embedding ``YYYYMMDD_HHMM``.""" + match = _RUN_TIMESTAMP_RE.search(identifier) + if match: + try: + return datetime.strptime(match.group("stamp"), "%Y%m%d_%H%M") + except ValueError: + pass + return datetime.min + + +def resolve_noaa_collection_path(model_or_path: str) -> str: + """Map a model shortcut or collection path to a THREDDS collection path. + + Parameters + ---------- + model_or_path : str + Shortcut such as ``\"GFS\"`` or a collection path such as + ``\"grib/NCEP/GFS/Global_0p25deg\"``. + + Returns + ------- + str + Collection path without a leading slash. + + Raises + ------ + ValueError + If ``model_or_path`` is not a known shortcut and does not look like a + collection path. + """ + if not isinstance(model_or_path, str) or not model_or_path.strip(): + raise ValueError("model_or_path must be a non-empty string.") + + key = model_or_path.strip() + mapped = NOAA_MODEL_COLLECTIONS.get(key.upper()) + if mapped is not None: + return mapped + + normalized = key.lstrip("/") + if normalized.startswith("grib/"): + return normalized.removesuffix("/catalog.xml").rstrip("/") + + raise ValueError( + f"Unknown NOAA model collection {model_or_path!r}. " + f"Known shortcuts: {sorted(NOAA_MODEL_COLLECTIONS)}. " + "Pass a THREDDS collection path such as " + "'grib/NCEP/GFS/Global_0p25deg' instead." + ) + + +def collection_catalog_url(model_or_path: str) -> str: + """Return the THREDDS ``catalog.xml`` URL for a model collection.""" + collection = resolve_noaa_collection_path(model_or_path) + return f"{THREDDS_CATALOG_ROOT}{collection}/catalog.xml" + + +def list_noaa_atmosphere_datasets( + catalog_url: str = FORECAST_MODELS_CATALOG_URL, + timeout: float = _DEFAULT_TIMEOUT_SECONDS, +) -> list[dict]: + """List available NCEP atmosphere model collections from THREDDS. + + Parameters + ---------- + catalog_url : str, optional + Root forecast-models catalog URL. Defaults to Unidata's NCEP models + catalog (the same server family used by RocketPy's GFS/NAM/… fetchers). + timeout : float, optional + HTTP timeout in seconds. + + Returns + ------- + list of dict + Each entry has ``name``, ``catalog_url``, ``collection_path`` and + ``opendap_best_url`` (the latter is ``None`` until a collection + catalog is inspected). + """ + root = _fetch_catalog_xml(catalog_url, timeout=timeout) + datasets = [] + seen = set() + + for ref in root.findall(".//thredds:catalogRef", _THREDDS_NS): + href = ref.get(f"{{{_THREDDS_NS['xlink']}}}href") or ref.get("href") + if not href: + continue + name = ( + ref.get(f"{{{_THREDDS_NS['xlink']}}}title") + or ref.get("name") + or ref.get("ID") + or href + ) + absolute = _absolute_catalog_url(href, catalog_url) + collection_path = _collection_path_from_catalog_url(absolute) + key = (name, absolute) + if key in seen: + continue + seen.add(key) + datasets.append( + { + "name": name, + "catalog_url": absolute, + "collection_path": collection_path, + "opendap_best_url": ( + _opendap_url_from_path(f"{collection_path}/Best") + if collection_path + else None + ), + } + ) + + if not datasets: + raise RuntimeError( + f"No NOAA/THREDDS dataset collections found in {catalog_url}." + ) + + return datasets + + +def list_noaa_dataset_identifiers( + model_or_path: str, + timeout: float = _DEFAULT_TIMEOUT_SECONDS, + include_aggregates: bool = True, +) -> list[str]: + """List dataset identifiers available inside a NOAA/NCEP collection. + + Parameters + ---------- + model_or_path : str + Shortcut (``\"GFS\"``, ``\"NAM\"``, …) or a THREDDS collection path. + timeout : float, optional + HTTP timeout in seconds. + include_aggregates : bool, optional + When ``True`` (default), include virtual aggregates such as ``Best`` + and ``TwoD`` alongside individual forecast-run identifiers. + + Returns + ------- + list of str + Identifiers suitable for :func:`build_noaa_opendap_url`. Forecast-run + filenames (``*.grib2``) are sorted newest-first when timestamps can be + parsed; aggregate names are appended afterward. + """ + catalog_url = collection_catalog_url(model_or_path) + root = _fetch_catalog_xml(catalog_url, timeout=timeout) + + runs = [] + aggregates = [] + + for element in root.iter(): + tag = _local_tag(element.tag) + if tag not in {"dataset", "catalogRef"}: + continue + + url_path = element.get("urlPath") + name = ( + element.get(f"{{{_THREDDS_NS['xlink']}}}title") + or element.get("name") + or element.get("ID") + or "" + ) + + if url_path in {None, "", "latest.xml"}: + continue + + identifier = url_path.rsplit("/", 1)[-1] + if identifier.lower().endswith(".grib2") or _RUN_TIMESTAMP_RE.search( + identifier + ): + runs.append(identifier) + elif include_aggregates and identifier in {"Best", "TwoD"}: + aggregates.append(identifier) + elif include_aggregates and name and "Best" in name and identifier: + aggregates.append(identifier) + + # Preserve order while dropping duplicates. + unique_runs = list(dict.fromkeys(runs)) + unique_runs.sort(key=_run_sort_key, reverse=True) + unique_aggregates = list(dict.fromkeys(aggregates)) + identifiers = unique_runs + unique_aggregates + + if not identifiers: + raise RuntimeError( + "No dataset identifiers found in NOAA/THREDDS collection catalog " + f"{catalog_url}." + ) + + return identifiers + + +def build_noaa_opendap_url(model_or_path: str, identifier: str) -> str: + """Build an OPeNDAP URL for a collection dataset identifier.""" + collection = resolve_noaa_collection_path(model_or_path) + return _opendap_url_from_path(f"{collection}/{identifier}") + + +def get_latest_noaa_dataset_identifier( + model_or_path: str, + timeout: float = _DEFAULT_TIMEOUT_SECONDS, +) -> str: + """Return the newest forecast-run identifier for a collection. + + Prefers THREDDS ``latest.xml`` when present. Falls back to the maximum + timestamp among listed ``*.grib2`` runs, then to the ``Best`` aggregate. + + Parameters + ---------- + model_or_path : str + Shortcut or collection path. + timeout : float, optional + HTTP timeout in seconds. + + Returns + ------- + str + Dataset identifier (for example + ``\"GFS_Global_0p25deg_20260810_1800.grib2\"`` or ``\"Best\"``). + """ + collection = resolve_noaa_collection_path(model_or_path) + latest_catalog_url = f"{THREDDS_CATALOG_ROOT}{collection}/latest.xml" + + try: + root = _fetch_catalog_xml(latest_catalog_url, timeout=timeout) + except RuntimeError: + root = None + + if root is not None: + for element in root.iter(): + if _local_tag(element.tag) != "dataset": + continue + url_path = element.get("urlPath") + name = element.get("name") + if url_path and url_path != "latest.xml": + return url_path.rsplit("/", 1)[-1] + if name and name.lower().endswith(".grib2"): + return name + + identifiers = list_noaa_dataset_identifiers( + model_or_path, timeout=timeout, include_aggregates=True + ) + for identifier in identifiers: + if identifier.lower().endswith(".grib2") or _RUN_TIMESTAMP_RE.search( + identifier + ): + return identifier + + if "Best" in identifiers: + return "Best" + + return identifiers[0] + + +def get_latest_noaa_opendap_url( + model_or_path: str = "GFS", + timeout: float = _DEFAULT_TIMEOUT_SECONDS, +) -> str: + """Resolve the OPeNDAP URL for the latest dataset in a collection.""" + identifier = get_latest_noaa_dataset_identifier(model_or_path, timeout=timeout) + return build_noaa_opendap_url(model_or_path, identifier) + + +def fetch_latest_noaa_dataset( + model_or_path: str = "GFS", + max_attempts: int = 10, + base_delay: float = 2, + timeout: float = _DEFAULT_TIMEOUT_SECONDS, +): + """Open the latest NOAA/NCEP dataset for a model via OPeNDAP. + + Parameters + ---------- + model_or_path : str, optional + Shortcut such as ``\"GFS\"`` (default) or a THREDDS collection path. + max_attempts : int, optional + Maximum netCDF4 open attempts. Default is 10. + base_delay : float, optional + Base exponential backoff delay in seconds. Default is 2. + timeout : float, optional + HTTP timeout used while resolving the catalog entry. + + Returns + ------- + netCDF4.Dataset + Open dataset handle. + + Raises + ------ + RuntimeError + If the catalog cannot be resolved or all open attempts fail. + """ + file_url = get_latest_noaa_opendap_url(model_or_path, timeout=timeout) + attempt_count = 0 + while attempt_count < max_attempts: + try: + return netCDF4.Dataset(file_url) + except OSError: + attempt_count += 1 + time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) + + raise RuntimeError( + "Unable to load the latest NOAA/THREDDS weather dataset through " + + file_url + ) diff --git a/tests/unit/environment/test_fetchers.py b/tests/unit/environment/test_fetchers.py index c226076db..5eef10e79 100644 --- a/tests/unit/environment/test_fetchers.py +++ b/tests/unit/environment/test_fetchers.py @@ -350,3 +350,156 @@ def test_fetch_meteomatics_data_converts_date_to_utc(monkeypatch): data_calls = [c for c in calls if c[0] != fetchers.METEOMATICS_LOGIN_URL] assert data_calls, "expected at least one data request" assert all("2024-01-01T09:00:00Z" in url for url, _ in data_calls) + + +_FORECAST_MODELS_XML = """ + + + + +""" + +_GFS_COLLECTION_XML = """ + + + + + + + +""" + +_GFS_LATEST_XML = """ + + + +""" + + +class _FakeCatalogResponse: + """Minimal response object for mocked NOAA/THREDDS catalog GETs.""" + + def __init__(self, text, status_code=200): + self.text = text + self.content = text.encode("utf-8") + self.status_code = status_code + + +def _install_noaa_catalog_mocks(monkeypatch, *, include_latest=True): + """Patch ``requests.get`` with canned forecast-model and GFS catalogs.""" + + def fake_get(url, timeout=None, **_kwargs): + del timeout + if url.endswith("forecastModels.xml") or "idd/forecastModels.xml" in url: + return _FakeCatalogResponse(_FORECAST_MODELS_XML) + if url.endswith("/latest.xml"): + if include_latest: + return _FakeCatalogResponse(_GFS_LATEST_XML) + return _FakeCatalogResponse("missing", status_code=404) + if url.endswith("/Global_0p25deg/catalog.xml"): + return _FakeCatalogResponse(_GFS_COLLECTION_XML) + return _FakeCatalogResponse("unexpected", status_code=404) + + monkeypatch.setattr(fetchers.requests, "get", fake_get) + return fake_get + + +def test_list_noaa_atmosphere_datasets_parses_catalog_refs(monkeypatch): + """List NCEP collections from the mocked forecast-models catalog.""" + _install_noaa_catalog_mocks(monkeypatch) + + datasets = fetchers.list_noaa_atmosphere_datasets() + + assert [entry["name"] for entry in datasets] == [ + "GFS Quarter Degree Forecast", + "NAM CONUS 12km from NOAAPORT", + ] + assert datasets[0]["collection_path"] == "grib/NCEP/GFS/Global_0p25deg" + assert datasets[0]["opendap_best_url"].endswith( + "grib/NCEP/GFS/Global_0p25deg/Best" + ) + + +def test_list_noaa_dataset_identifiers_sorts_runs_newest_first(monkeypatch): + """GFS run identifiers should sort by embedded timestamp, newest first.""" + _install_noaa_catalog_mocks(monkeypatch) + + identifiers = fetchers.list_noaa_dataset_identifiers("GFS") + + assert identifiers[0] == "GFS_Global_0p25deg_20260810_1800.grib2" + assert identifiers[1] == "GFS_Global_0p25deg_20260810_1200.grib2" + assert "Best" in identifiers + + +def test_get_latest_noaa_opendap_url_uses_latest_xml(monkeypatch): + """Prefer THREDDS latest.xml when resolving the newest GFS run.""" + _install_noaa_catalog_mocks(monkeypatch, include_latest=True) + + url = fetchers.get_latest_noaa_opendap_url("GFS") + + assert url == ( + "https://thredds.ucar.edu/thredds/dodsC/" + "grib/NCEP/GFS/Global_0p25deg/GFS_Global_0p25deg_20260810_1800.grib2" + ) + + +def test_get_latest_noaa_dataset_identifier_falls_back_without_latest_xml( + monkeypatch, +): + """When latest.xml is missing, choose the newest listed grib2 run.""" + _install_noaa_catalog_mocks(monkeypatch, include_latest=False) + + identifier = fetchers.get_latest_noaa_dataset_identifier("gfs") + + assert identifier == "GFS_Global_0p25deg_20260810_1800.grib2" + + +def test_fetch_latest_noaa_dataset_opens_resolved_url(monkeypatch): + """fetch_latest_noaa_dataset should open the resolved OPeNDAP URL.""" + _install_noaa_catalog_mocks(monkeypatch) + calls = [] + sentinel = object() + + def fake_dataset(url): + calls.append(url) + return sentinel + + monkeypatch.setattr(fetchers.netCDF4, "Dataset", fake_dataset) + + dataset = fetchers.fetch_latest_noaa_dataset("GFS", max_attempts=2, base_delay=2) + + assert dataset is sentinel + assert calls == [ + "https://thredds.ucar.edu/thredds/dodsC/" + "grib/NCEP/GFS/Global_0p25deg/GFS_Global_0p25deg_20260810_1800.grib2" + ] + + +def test_resolve_noaa_collection_path_rejects_unknown_model(): + """Unknown shortcuts must fail with an actionable ValueError.""" + with pytest.raises(ValueError, match="Unknown NOAA model collection"): + fetchers.resolve_noaa_collection_path("not-a-model")