diff --git a/.gitignore b/.gitignore index 3dfbb55f..d3b9beea 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,11 @@ ENV/ # pyscn analysis reports (rebuildable: `pyscn analyze dataretrieval`) .pyscn/ + +# Working design note for the layered-configuration work; the durable +# record is ADR 0009 + docs/source/userguide/configuration.rst. +CONFIG-PLAN.md + +# Resolver lock for local dev; the package ships a range-based pyproject and +# is not deployed from a pinned set. +uv.lock diff --git a/.importlinter b/.importlinter index f6b95138..bcc34a9b 100644 --- a/.importlinter +++ b/.importlinter @@ -21,7 +21,11 @@ type = layers containers = dataretrieval layers = - ngwmn | nldi | nwis | streamstats | waterdata | wateruse | wqp +; The deprecated ``wateruse`` alias re-exports ``nwdc``, so it sits above the +; adapters rather than beside them. A compatibility facade may depend on the +; adapter it forwards to; nothing may depend on the facade. + wateruse + ngwmn | nldi | nwdc | nwis | streamstats | waterdata | wqp ogc utils _querying @@ -32,6 +36,7 @@ layers = _wqx _ambient | _response_metadata | codes | combining | interruptions | rdb credentials + settings exceptions ; Every top-level module must be placed in the stack deliberately. A new ; top-level module fails this contract until someone decides where it sits. @@ -114,6 +119,7 @@ source_modules = dataretrieval.streamstats dataretrieval.transport dataretrieval.utils + dataretrieval.nwdc dataretrieval.waterdata dataretrieval.wateruse dataretrieval.wqp diff --git a/AGENTS.md b/AGENTS.md index 455c108c..7c821ecf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ - Exclude `.claude/worktrees/` from searches and edits; it contains stale worktrees that pollute results. ## Example Notebooks -- `demos/*.ipynb` — top-level Water Data tour: `USGS_WaterData_Introduction_Examples.ipynb` is the entry point; `_ContinuousData_`, `_DailyStatistics_`, `_DiscreteSamples_`, `_ReferenceLists_` cover individual collections; `WaterData_demo.ipynb`, `peak_streamflow_trends.ipynb`, `USGS_WaterUse_Examples.ipynb` (NWDC water-use data via `wateruse.get_wateruse`), and `R Python Vignette equivalents.ipynb` are standalone walkthroughs. +- `demos/*.ipynb` — top-level Water Data tour: `USGS_WaterData_Introduction_Examples.ipynb` is the entry point; `_ContinuousData_`, `_DailyStatistics_`, `_DiscreteSamples_`, `_ReferenceLists_` cover individual collections; `WaterData_demo.ipynb`, `peak_streamflow_trends.ipynb`, `USGS_WaterUse_Examples.ipynb` (NWDC water-use data via `nwdc.get_wateruse`), and `R Python Vignette equivalents.ipynb` are standalone walkthroughs. - `demos/hydroshare/*.ipynb` — per-service HydroShare examples (NLDI, NWIS WaterUse, and Water Data DailyValues / GroundwaterLevels / Measurements / ParameterCodes / Peaks / Ratings / Samples / SiteInfo / SiteInventory / Statistics / UnitValues). Mirror these when adding examples for a new collection. - `demos/nwqn_data_pull/` — non-notebook example: a lithops/Docker batch pipeline (`retrieve_nwqn_samples.py`, `retrieve_nwqn_streamflow.py`) with its own `README.md`. - Any `Untitled*.ipynb`, `*_test.ipynb`, or notebooks not listed here are untracked local scratch; ignore them. diff --git a/CONTEXT.md b/CONTEXT.md index 8b9e2032..c112c426 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -71,8 +71,12 @@ statistics. The package's primary target. **NGWMN** — The National Ground-Water Monitoring Network, a distinct OGC API covering sites, water levels, lithology, well construction, and providers. -**NWDC** — The National Water Availability Assessment Data Companion, providing -modeled national-scale water-use data. +**NWDC** — The National Water Availability Assessment Data Companion. Serves +ten modeled national-scale datasets, of which the water-use models are five; +the rest are hydrologic, atmospheric-forcing, and assessment outputs. The +package reaches it through the `nwdc` adapter, named for the service like every +other adapter. Legacy: that module was `wateruse`, which named one subset of +what the service offers. **WQP** — The Water Quality Portal, a multi-agency water-quality clearinghouse. @@ -105,6 +109,77 @@ term. Legacy: the deprecated NWIS getters and the WQP profiles call this a **Metadata** — The second half of every getter's return: the request URL, the elapsed time, and the response headers. Describes the *retrieval*, not the data. +## Settings + +**Settings profile** — A named set of settings for one adapter, stored in the +settings file or built in code. **Profile** is the short form; the class that +carries one is `Settings`. A profile is an *input* to resolution, +never its result. + +**Default profile** — The profile an adapter uses when no other is selected: +the `[]` table's own keys. Always in effect. A **named profile** +(`[.bulk]`) is in effect only when a caller selects it, so adding one +to a file never changes an existing script. + +**Effective settings** — The resolved set of settings a call will use: what the +chain produces after every profile, variable and default has been applied. +Distinct from a settings profile, which is one contribution to it. +**Configure** is the verb for applying one, and keeps that spelling: it is what +the caller writes, and the settings library has no competing name for it. + +**Settings file** — `~/.dataretrieval/config.toml`, or the path in +`DATARETRIEVAL_CONFIG`. The file keeps the name `config.toml` while the +vocabulary around it says *settings*: the path is a compatibility surface, and +`config` is the conventional name for a file on disk. + +**Setting** — One named tunable the caller may adjust: the API key, the +concurrency cap, the retry count, the progress line, the fan-out baseline. A +setting means the same thing wherever it applies, but it does not apply +everywhere: `concurrency` and `parallel_chunks` are meaningless to an adapter +that issues one request, and `ssl_check` is meaningful to only three. Which +settings an adapter accepts is part of that adapter's vocabulary. + +**Package-wide setting** — A setting that applies to every adapter: the retry +count, the progress line, the stall timeout. Set once, honored everywhere. + +**Adapter-scoped setting** — A setting named under one adapter, applying to +that adapter and no other. It overrides the package-wide value for that adapter +alone; it does not replace the package-wide tier. An adapter rejects a setting +it has no use for, rather than accepting and ignoring it. + +The scope is the *adapter*, not the service and not the host, because the +adapter is what owns the conventions being tuned. The API key is the +counter-example that fixes the distinction: it belongs to the gateway fronting +a host, so Water Data and NGWMN — two adapters, one host — necessarily share +one key and one quota pool. Credentials are host-scoped; tunables are +adapter-scoped. + +**Source** — Where a setting's value came from. Sources are ordered, and the +order is resolved per setting rather than per source: a value supplied for one +setting does not displace another setting's value from a lower source. + +**Selection** — Naming which profile an adapter should use. Done in code; a +profile is never selected by the environment or implied by the file, so the +set of profiles in a file is inert until something asks for one. + +**Built-in default** — The value a setting takes when no source supplies one. +Package-wide. + +**Adapter default** — The value a *particular adapter* prefers when no source +supplies one, because that adapter warrants a different figure — NWDC asks for +4 concurrent requests where the OGC getters take 32. Supplied by the adapter in +code, not by the user. It replaces the built-in default for calls through that +adapter and nothing else. A value from any source outranks it: an adapter able +to override an explicit setting would make that setting a lie. + +Distinct from an **adapter-scoped setting**, which is the *user* naming a value +for one adapter. Both narrow to a single adapter; only one of them is something +the caller wrote. + +All three are called "the default" in casual use, and they are not the same +value. Where the distinction matters — reporting what a call will actually use +— say which one is meant. + ## Boundaries **Adapter** — A module owning one service's conventions: its URLs, parameters, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c6ce3b8f..4416e8d7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,9 +115,10 @@ about the upstream service rather than about this package. **Before adding a small helper, check whether a leaf already generalizes it.** This package keeps its general mechanisms in dependency-free leaves -- -`_ambient.Ambient` for scoped context values, `transport.retry._read_env_number` -for `API_USGS_*` settings, `transport.links.resolve_next_url` for pagination -cursors. Each of those has been re-implemented at least once by someone who +`_ambient.Ambient` for scoped context values, `config` for every setting +(`API_USGS_*`, the config file, and `configure()` blocks all resolve through +it, and it is the only module that reads the environment for one), +`transport.links.resolve_next_url` for pagination cursors. Each of those has been re-implemented at least once by someone who did not know it was there, and the copies drift: the same question gets a different cycle guard, a different error message, a different edge case. None of the automated checks catch it, because two eight-line helpers are below the diff --git a/NEWS.md b/NEWS.md index 2f1b52d2..8a850223 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +**08/11/2026:** Settings resolve through a layered chain instead of the environment alone, and a *settings profile* is a named set of settings for **one adapter**. The new `dataretrieval.settings` module resolves every setting in one order, highest first: a settings profile passed to an active `dataretrieval.configure(...)` block, a profile that block selected, the setting's `API_USGS_*` environment variable, the adapter's `[]` table in `~/.dataretrieval/config.toml` (or `DATARETRIEVAL_CONFIG`), the file's top-level keys, the adapter's own built-in preference, then the package default. Precedence applies **per setting**, so a file that sets only `concurrency` leaves an environment `API_USGS_PAT` in effect, and a `[ngwmn]` table still inherits every top-level key it does not name. `configure()` takes settings profiles positionally, at most one per adapter and nothing else: `configure(Settings(api_key=vault.read("usgs/pat")), WaterdataSettings.load("bulk"), NgwmnSettings(concurrency=4))`. The adapter a profile targets is a property of its class, so a caller never restates it, and each adapter owns its class in the module that *reads* those settings (`waterdata.WaterdataSettings`, `ngwmn.NgwmnSettings`, `nwdc.NwdcSettings`, `wqp.WqpSettings`, `nldi.NldiSettings`, `streamstats.StreamstatsSettings`) — an adapter accepts only the settings it reads, so `[streamstats] parallel_chunks = 8` is an error rather than a line that quietly does nothing. The block is delivered through a `ContextVar`, so a credential set inside it cannot leak across threads or asyncio tasks, which is what makes it safe for a server or notebook handling several users' keys and is the thing assigning to `os.environ` could never do (issue #352). The file gains named profiles beside each adapter's default profile: `[waterdata]` is always in effect, `[waterdata.bulk]` only when a caller selects it with `WaterdataSettings.load("bulk")`, and a selected profile still inherits the default profile and the package-wide keys per setting. A profile named in code outranks the setting's environment variable — the one place the ladder inverts the environment-above-file rule, because losing a deliberate selection to a stale shell export is what a caller would file a bug about. An adapter's settings may also carry a `base_url`, which redirects that adapter's requests for the duration of the block — a staging instance, a mirror, a recording proxy — and no other adapter's; for Water Data one value moves the OGC collections, the Samples database, the statistics service and the STAC catalog together. It is settable in a `configure()` block only: a `base_url` key in the settings file and an exported `API_USGS_BASE_URL` each raise rather than being read, since a redirect a config file or a shell profile can set is one no reader of the script can see. The API key does not follow a redirect — it is scoped to the single host that honors it — and is deliberately not per-adapter: it authenticates to the gateway fronting a host, and Water Data and NGWMN share that host, one key, and one hourly quota. `dataretrieval.show_settings()` reports each setting's effective value and where it came from, naming the profile behind each value (`configure() block [waterdata.bulk]` rather than a bare block), listing the profiles a file defines whether or not this run selected any, and naming any adapter this process has not imported rather than omitting it — without ever printing the key. One parser per setting owns its grammar, so a value means the same thing whichever source wrote it. **Breaking change:** `RetryPolicy.from_env()` is now `RetryPolicy.from_settings()` and resolves through the whole chain rather than the environment alone. **Behavior change:** a credential-shaped keyword passed to a getter's `**kwargs` query passthrough — Water Data's `**queryables` and every WQP getter's search filters — now raises `TypeError` naming `configure(Settings(api_key=...))` instead of putting a secret in a URL that clients, proxies and logs retain. The names refused are `api_key=`, `token=`, `x_api_key=`, `password=`, `auth=`, `pat=` and similar spellings; a filter the server actually defines is unaffected. **Bug fix:** `API_USGS_STALL_TIMEOUT` was read straight from `os.environ`, so it could not be set by a `configure()` block or the settings file and never appeared in `show_settings()`; it now resolves through the chain like every other setting. Settings are declared and resolved with [pydantic-settings](https://docs.pydantic.dev/latest/concepts/pydantic_settings/), which is now a required dependency: each adapter's profile is a `BaseSettings` subclass, so its field annotations are enforced rather than decorative, and each rung of the ladder above is a `PydanticBaseSettingsSource`. **Behavior change:** a setting an adapter does not read, or a misspelled one, now raises `ConfigurationError` rather than a bare `TypeError` — the same mistake written into the settings file has always raised `ConfigurationError`, so the two surfaces now agree, and the message lists the settings the adapter does accept. Rationale in ADRs 0009, 0010, 0011 and 0012; terms in `CONTEXT.md`. + +**08/11/2026:** `dataretrieval.wateruse` is now `dataretrieval.nwdc`. Every other adapter is named for the service it retrieves from — `ngwmn`, `nldi`, `wqp`, `streamstats`, `nwis` — and this one was named for one subset of what its service offers. The National Water Availability Assessment Data Companion serves ten modeled datasets; the water-use models are five of them, the rest being hydrologic, atmospheric-forcing, and assessment outputs (`GET https://api.water.usgs.gov/nwaa-data/models`). **Deprecation:** `dataretrieval.wateruse` still works and re-exports `dataretrieval.nwdc` unchanged, emitting a `DeprecationWarning` on import; it will be removed on or after 2027-08-11. The alias forwards rather than copies, so `wateruse.get_wateruse is nwdc.get_wateruse` — monkeypatching or identity comparison through either spelling behaves the same. `import dataretrieval` stays silent: the package imports `nwdc` directly, so only code naming `wateruse` itself sees the warning. Function and constant names are unchanged (`get_wateruse`, `MODELS`, `WATERUSE_URL`, `DEFAULT_CONCURRENT_REQUESTS`). Terms are defined in `CONTEXT.md`. + **08/09/2026:** `waterdata.get_cql` takes `collection` rather than `service`. OGC API - Features (17-069r4) normatively names this value the `collectionId`: Requirement 20 fixes the path template `/collections/{collectionId}/items`, and Requirement 18 defines `collectionId` as each `id` in the collections response -- which is literally how the package builds the URL, and what the live API returns. *Service* names the API itself (Water Data, NGWMN). **Deprecation:** `service=` still works and resolves to `collection`, with a `DeprecationWarning`; it will be removed on or after 2027-08-09. Positional callers (`get_cql("daily", cql)`) are unaffected. The `WATERDATA_SERVICES` type alias is now `WATERDATA_COLLECTIONS`, with `WATERDATA_SERVICES` retained as a permanent alias for the same object. Terms are defined in `CONTEXT.md`. **08/09/2026:** Every retrieval path now runs through one executor. `waterdata.get_cql` (via the OGC `fetch_ogc_request`) and `waterdata.get_stats_por` / `get_stats_date_range` (via the Statistics page walk) previously bypassed `dataretrieval.transport.fanout.FanOut` through a private sync bridge, which meant they were the only getters in the package with **no retry**: a mid-page-walk 429 or 503 failed the whole call while every typed getter and Water Use rode it out. Both now run as a one-item fan-out and the 25-line `transport/sync.py` is gone. **Behavior change:** those three getters now retry transient failures (`API_USGS_RETRIES`, default 4) and, when the retries are exhausted, raise the resumable `ServiceInterrupted` / `QuotaExhausted` rather than `ServiceUnavailable` / `RateLimited` / `NetworkError` — all remain `DataRetrievalError`, so broad handlers are unaffected, but narrow handlers around those calls must widen, and `.call.resume()` is now available on the interruption. A failure that retrying cannot fix (bad scheme, a hostname that does not resolve) still surfaces as `NetworkError` immediately. The progress line moved with it: `FanOut.resume()` opens the reporter it ticks into, so a driver can no longer run the shared executor and silently print nothing, and a `.call.resume()` fired long after the interruption now reports progress instead of running mute. Internal tidying with no public effect: the WQX3 / legacy-WQP CSV datetime shaping moved out of `dataretrieval.utils` (whose docstring reserves it for non-service-specific shaping) into the `dataretrieval._wqx` leaf; the five Water Data endpoint URLs are declared once in `dataretrieval.waterdata.endpoints` instead of being derived in three modules; the OGC queryables document is parsed by `dataretrieval.ogc.schema` so every OGC adapter can offer the table, with `waterdata.get_queryables` unchanged as its documented wrapper; and `ogc/engine.py` imports each symbol from the module that defines it. diff --git a/README.md b/README.md index 41b3299c..542b0072 100644 --- a/README.md +++ b/README.md @@ -42,16 +42,57 @@ pip install git+https://github.com/DOI-USGS/dataretrieval-python.git Access USGS water-monitoring data. -**Important:** We strongly encourage you to obtain an API key for higher -rate limits. [Register for an API key](https://api.waterdata.usgs.gov/signup/) -and set it as an environment variable: +**Important:** Users are strongly encouraged to obtain an API key for higher +rate limits. [Register for an API key](https://api.waterdata.usgs.gov/signup/), +then supply it in whichever of these ways suits you. They are listed from +highest to lowest precedence, so an explicit block or deployment environment +can override a file without editing it: ```python -import os +# 1. a configure() block - for one call, an interactive prompt, or when +# different threads/tasks need different credentials. +from getpass import getpass -os.environ["API_USGS_PAT"] = "your_api_key_here" +import dataretrieval +from dataretrieval import Settings, waterdata + +with dataretrieval.configure(Settings(api_key=getpass("USGS API key: "))): + df, metadata = waterdata.get_daily(monitoring_location_id="USGS-01646500") +``` + +```bash +# 2. an environment variable (the R dataRetrieval package uses the same +# variable, so one export serves both) +export API_USGS_PAT="your_api_key_here" +``` + +```toml +# 3. ~/.dataretrieval/config.toml - keeps the key out of your shell +# environment, where every process you start inherits it. +# Restrict it afterwards: chmod 600 ~/.dataretrieval/config.toml +api_key = "your_api_key_here" ``` +`dataretrieval.show_settings()` reports what is in effect and where each setting +came from, without printing the key. Concurrency, retries, and the progress +line are configured the same way, and can be narrowed to one service: a +`configure()` block takes at most one configuration per adapter, each either +built in code or loaded by name from a profile in the file. + +```python +from dataretrieval.ngwmn import NgwmnSettings +from dataretrieval.waterdata import WaterdataSettings + +with dataretrieval.configure( + WaterdataSettings.load("overnight"), # a profile in config.toml + NgwmnSettings(concurrency=2), # built here +): + ... +``` + +See the +[settings guide](https://doi-usgs.github.io/dataretrieval-python/userguide/settings.html). + The following example retrieves daily streamflow data for a specific monitoring location. The `/` in the `time` argument separates the start and end of the desired range: @@ -127,7 +168,7 @@ from dataretrieval import waterdata # enough to span many pages, so it profits from a finer split. sites, _ = waterdata.get_monitoring_locations(state="Ohio", site_type_code="ST") -with waterdata.parallel_chunks(32): # fan out into 32 sub-requests +with waterdata.parallel_chunks(32): # request up to 32 optional chunks df, md = waterdata.get_daily( monitoring_location_id=sites["monitoring_location_id"], parameter_code="00060", # discharge @@ -147,11 +188,11 @@ Benchmark — a fixed 271-site subset of Ohio stream gages the effect of parallelism). Each `n` ran against its own cold 1-year time window, so no run is served from the server's data-window cache: -| `n` | parallelism | pages | wall-clock | speedup | -| ---- | ----------- | ----- | ----------------------- | ------- | -| off | 1 | ~30 | 9.5 s / 9.1 s (2 runs) | 1× | -| `8` | 8 | ~32 | 2.2 s / 1.9 s | ~4.5× | -| `32` | 32 | 54 | 1.2 s | ~8× | +| `n` | optional fan-out | pages | wall-clock | speedup | +| ---- | ---------------- | ----- | ---------------------- | ------- | +| off | 1 | ~30 | 9.5 s / 9.1 s (2 runs) | 1× | +| `8` | 8 | ~32 | 2.2 s / 1.9 s | ~4.5× | +| `32` | 32 | 54 | 1.2 s | ~8× | The gain comes from overlapping each sub-request's per-page latency and server-side work. The exact multiplier therefore scales with how many pages the @@ -252,11 +293,11 @@ Retrieve modeled water-use estimates from the National Water Availability Assessment Data Companion: ```python -from dataretrieval import wateruse +from dataretrieval import nwdc # Monthly public-supply withdrawals for Rhode Island, split into # groundwater and surface-water sources (returns a DataFrame and metadata). -df, metadata = wateruse.get_wateruse( +df, metadata = nwdc.get_wateruse( model="wu-public-supply-wd", variable=["pswdtot", "pswdgw", "pswdsw"], state="RI", # name/postal/FIPS; pass a list to fan out over several areas @@ -312,7 +353,7 @@ print(statewide.head()) - `get_features`: Find monitoring sites, dams, and other features along the network - `get_features_by_data_source`: Features from a specific data source -### Water Use (NWDC) — `dataretrieval.wateruse` +### NWDC (National Water Availability Assessment Data Companion) — `dataretrieval.nwdc` - `get_wateruse`: Modeled water-use estimates — public-supply, irrigation, and thermoelectric withdrawals and consumptive use — on a national 12-digit hydrologic-unit (HUC12) grid, summarizable to counties, states, or coarser hydrologic units ## More Examples diff --git a/dataretrieval/__init__.py b/dataretrieval/__init__.py index 29e288d7..728362de 100644 --- a/dataretrieval/__init__.py +++ b/dataretrieval/__init__.py @@ -11,12 +11,21 @@ df, meta = nwis.get_dv(sites="05427718") Available service modules: ``waterdata``, ``wqp`` (Water Quality Portal), -``wateruse`` (NWDC water-use data), ``nldi``, ``streamstats``, and the +``nwdc`` (National Water Availability Assessment Data Companion, incl. +water use), ``nldi``, ``streamstats``, and the deprecated ``nwis``. ``nldi`` requires geopandas (``pip install dataretrieval[nldi]``) and is imported on demand: ``from dataretrieval import nldi``. +Settings -- the Water Data API key, fan-out concurrency, retries, the progress +line -- resolve through :mod:`dataretrieval.settings`: a +``with dataretrieval.configure(Settings(...))`` block, then the +``API_USGS_*`` environment variables, then ``~/.dataretrieval/config.toml``. +A setting for one service goes on that adapter's own settings profile, such as +``waterdata.WaterdataSettings``. ``dataretrieval.show_settings()`` +reports what is in effect and where each value came from. + A failed request raises a subclass of :class:`dataretrieval.DataRetrievalError` (the taxonomy lives in ``dataretrieval.exceptions``); connection-level failures (timeouts, DNS) are wrapped as :class:`dataretrieval.NetworkError`. A fanned-out @@ -32,6 +41,15 @@ except PackageNotFoundError: __version__ = "version-unknown" +# Layered settings: a ``with configure(...)`` block, the environment, then the +# settings file. The canonical home is ``dataretrieval.settings``, which is +# built on pydantic-settings (ADR 0012); the callable is named ``configure`` so +# it doesn't shadow that module. +# +# The module itself is deliberately absent from ``__all__`` below: it and the +# ``Settings`` class differ only by case, and keeping the module out of the +# package's exports means ``from dataretrieval import settings, Settings`` +# never arises (ADR 0011, carried forward by ADR 0012). from dataretrieval.exceptions import ( ConfigurationError, DataRetrievalError, @@ -63,31 +81,36 @@ # ``dataretrieval.ogc.chunking``; surfaced here for a stable public path # ``from dataretrieval import parallel_chunks``. from dataretrieval.ogc.chunking import parallel_chunks +from dataretrieval.settings import Settings, configure, show_settings from . import ( exceptions, ngwmn, + nwdc, nwis, streamstats, utils, waterdata, - wateruse, wqp, ) __all__ = [ + # layered configuration (canonical home: ``dataretrieval.settings``) + "Settings", + "configure", + "show_settings", + "ConfigurationError", # service modules "ngwmn", + "nwdc", "nwis", "streamstats", "utils", "waterdata", - "wateruse", "wqp", # error taxonomy (canonical home: ``dataretrieval.exceptions``), re-exported # so callers can ``except dataretrieval.DataRetrievalError`` "exceptions", - "ConfigurationError", "DataRetrievalError", "HTTPError", "NetworkError", diff --git a/dataretrieval/_querying.py b/dataretrieval/_querying.py index 9fcde81a..81e8232a 100644 --- a/dataretrieval/_querying.py +++ b/dataretrieval/_querying.py @@ -2,7 +2,7 @@ "Compose a USGS query URL, send it, map the status, retry a transient" -- the half of the old ``utils`` module that talks to the network, as used by ``nwis``, -``wqp``, ``nldi``, ``streamstats`` and ``wateruse``. Its other half (pandas +``wqp``, ``nldi``, ``streamstats`` and ``nwdc``. Its other half (pandas column munging) shared nothing with this but a filename: no caller wanted both, and the two have disjoint dependencies -- this one needs ``exceptions`` and ``transport``, that one needs ``codes`` and pandas. @@ -102,7 +102,7 @@ def _raise_for_status( """Raise the typed :class:`DataRetrievalError` for an HTTP error response. A success status returns ``None``. Shared by the legacy :func:`query` path - (and ``streamstats`` / ``wateruse``). Delegates the status-to-type mapping to + (and ``streamstats`` / ``nwdc``). Delegates the status-to-type mapping to :func:`dataretrieval.exceptions.error_for_status`, except a too-long-URL status (413 / 414): that gets the same actionable "split your query" remediation as the client-side over-long-URL case below, rather than a bare @@ -131,14 +131,20 @@ def _raise_for_status( ) -def _single_request_policy() -> RetryPolicy: +def _single_request_policy(adapter: str | None = None) -> RetryPolicy: """Retry policy for the one-shot adapters (WQP, NLDI, StreamStats). These services answer a rejected query with a 500, so only the gateway statuses are worth re-sending; the Water Data chunker keeps the broader default, where a 5xx is an upstream hiccup worth riding out. + + ``adapter`` names which settings table supplies ``retries`` and + ``stall_timeout`` -- these three services share a retry *shape* but not + a settings scope. """ - return RetryPolicy.from_env(retryable_statuses=_GATEWAY_STATUSES) + return RetryPolicy.from_settings( + retryable_statuses=_GATEWAY_STATUSES, adapter=adapter + ) def _get_with_retry( @@ -146,6 +152,7 @@ def _get_with_retry( *, detail_from: Callable[[httpx.Response], str | None] | None = None, retry_policy: RetryPolicy | None = None, + adapter: str | None = None, **kwargs: Any, ) -> httpx.Response: """GET with status mapping and bounded retry on typed transients.""" @@ -158,7 +165,7 @@ def attempt() -> httpx.Response: try: return retry_sync( attempt, - _single_request_policy() if retry_policy is None else retry_policy, + _single_request_policy(adapter) if retry_policy is None else retry_policy, ) except httpx.InvalidURL as exc: raise _url_too_long_error(f"httpx rejected the URL client-side: {exc}") from exc @@ -171,6 +178,7 @@ def _query_with_retry( ssl_check: bool = True, *, retry_policy: RetryPolicy | None = None, + adapter: str | None = None, ) -> httpx.Response: """Send an active-service query with bounded transient retry by default.""" @@ -188,6 +196,7 @@ def _query_with_retry( headers=user_agent, verify=ssl_check, retry_policy=retry_policy, + adapter=adapter, **HTTPX_DEFAULTS, ) diff --git a/dataretrieval/credentials.py b/dataretrieval/credentials.py index ffb3df02..2bab4662 100644 --- a/dataretrieval/credentials.py +++ b/dataretrieval/credentials.py @@ -1,25 +1,35 @@ """Which host honors the USGS API key, and how it is attached and withheld. One leaf owns every answer about the ``API_USGS_PAT`` credential: the host that -accepts it, whether a given destination qualifies, and how it is stripped back -off a request bound somewhere else. Splitting those answers across the layers -that happen to need them is how a credential reaches a host nobody authorized: -the code that attaches a key and the code that removes it have to agree, and the -only way to guarantee they agree is to have them read the same predicate. - -This is deliberately a leaf. It sits below HTTP mechanics (which attaches the -header) and below progress reporting (which tells an unauthenticated caller where -to register), so neither has to depend on the other to learn the same fact. +accepts it, whether a given destination qualifies, how it is stripped back off a +request bound somewhere else, and which keyword names are a caller *asking* to +send it. Splitting those answers across the layers that happen to need them is +how a credential reaches a host nobody authorized: the code that attaches a key +and the code that removes it have to agree, and the only way to guarantee they +agree is to have them read the same predicate. + +This sits below HTTP mechanics (which attaches the header) and below progress +reporting (which tells an unauthenticated caller where to register), so neither +has to depend on the other to learn the same fact. Its only first-party +dependency is :mod:`dataretrieval.settings`, which is itself a +standard-library-only leaf and sits directly beneath this module in the layers +contract -- it supplies the key's *value*, while the questions this module +owns are which host may receive it and how it is withheld from every other. """ from __future__ import annotations -import os +from collections.abc import Iterable import httpx +from dataretrieval import settings as _settings + #: Environment variable holding the USGS Water Data personal access token. -API_KEY_ENV = "API_USGS_PAT" +#: Taken from the chain that reads it rather than spelled again here -- the +#: same rule ``test_credential_policy_has_one_definition`` enforces for the +#: authorized host, and for the same reason: two copies stop agreeing silently. +API_KEY_ENV = _settings.ENV_VARS["api_key"] #: Where to register for a key. Surfaced once, by the progress reporter, when a #: query against the authorized host runs without one -- unauthenticated callers @@ -77,13 +87,77 @@ def without_embedded_credentials(url: httpx.URL) -> httpx.URL: return url.copy_with(userinfo=b"") if url.userinfo else url +# Credential-shaped keyword names must never reach a getter's generic query +# passthrough: URLs are retained by clients, proxies, logs, and response +# metadata. Kept here rather than in the adapter that first needed it, because +# the fact that motivates the check is package-wide -- ``configure()`` now takes +# ``Settings(api_key=...)``, so a caller who has not read that far reaches +# for ``api_key=`` on whichever getter they are already calling, and every +# adapter with a ``**kwargs`` passthrough is that getter. +# +# Matched as *substrings* of the separator-stripped name, not as exact names: +# an exact-match list missed the spelling the library's own docs make most +# tempting -- ``x_api_key``, after the ``X-Api-Key`` header. +_CREDENTIAL_MARKERS = ( + "apikey", + "authorization", + "credential", + "password", + "passwd", + "secret", + "token", +) + +# Whole names that are credentials on their own but too short to match as +# substrings without catching legitimate query parameters. +# +# ``session`` is deliberately absent from both lists: it carries no secret, so +# rejecting it with a credentials message told users the wrong thing, and as a +# substring it claimed part of a namespace the *server* owns -- any future +# query parameter containing it would have been unreachable behind that message. +_CREDENTIAL_NAMES = frozenset({"auth", "key", "pat", "pw"}) + + +def refuse_credential_keywords(names: Iterable[str]) -> None: + """Raise ``TypeError`` if any of *names* reads as a request for the key. + + For the ``**kwargs`` passthroughs -- Water Data's ``**queryables`` and + WQP's search filters -- where a name the caller invents is forwarded to the + server as a query parameter. Both call this rather than each keeping its + own list, so a spelling learned from one adapter's mistake is refused by + the other on the same day. + + This catches the plausible mistake; it is not a security control. Nothing + inspects *values*, so a secret pasted into ``state_name=`` travels just the + same, and the name space belongs to the server (``get_queryables``) rather + than to us. The point is to answer the caller who reasonably guesses that a + credential goes here, with a ``TypeError`` naming + ``configure(Settings(api_key=...))`` instead of a token in a URL. It + errs toward rejecting for that reason. + """ + forbidden = set() + for name in names: + flat = name.replace("_", "").replace("-", "").casefold() + if flat in _CREDENTIAL_NAMES or any(m in flat for m in _CREDENTIAL_MARKERS): + forbidden.add(name) + if forbidden: + spellings = ", ".join(f"{name}=" for name in sorted(forbidden)) + raise TypeError( + f"Credentials cannot be passed as query parameters ({spellings}); " + "use dataretrieval.configure(Settings(api_key=...)) instead." + ) + + def api_key() -> str | None: """The configured token, or ``None``. - Read through a function rather than captured at import so a caller that sets - the variable after import -- or a test that patches it -- is still honored. + Lives here, next to the host check and + :func:`strip_api_key_from_untrusted_host`, so reading the key and the rules + governing where it may travel stay in one module. The value itself resolves + through :func:`dataretrieval.settings.api_key`, so host scoping applies + identically no matter which source supplied the key. """ - return os.getenv(API_KEY_ENV) + return _settings.api_key() def strip_api_key_from_untrusted_host(request: httpx.Request) -> None: diff --git a/dataretrieval/exceptions.py b/dataretrieval/exceptions.py index 79160e9e..da02f161 100644 --- a/dataretrieval/exceptions.py +++ b/dataretrieval/exceptions.py @@ -13,7 +13,10 @@ aren't a plain status: :class:`RequestTooLarge` (with :class:`URLTooLong` / :class:`Unchunkable`), :class:`NetworkError` (a failed connection, per above), :class:`NoSitesError`, and :class:`ConfigurationError` for an unusable setting. -:func:`error_for_status` maps a status to its type. +:func:`error_for_status` maps a status to its type. ``ConfigurationError`` is +the one member that is not a request failure at all: it reports an unusable +setting or config file, raised from wherever a setting is first resolved -- +which, because resolution is lazy, is inside whichever getter runs first. This module has no third-party runtime dependencies -- ``httpx`` is imported only for type checking. Any module can therefore import it without pulling in pandas @@ -48,7 +51,15 @@ class DataRetrievalError(Exception): - """Base class for every failed-request error in ``dataretrieval``. + """Base class for every ``dataretrieval`` error. + + Almost every member is a failed request, and the read-anywhere fields below + describe one. The exception is :class:`ConfigurationError`, which reports a + configuration the library cannot use; it appears here because configuration + is resolved lazily on the request path, so it surfaces from inside a getter + and one ``except DataRetrievalError`` should cover it too. It carries no + status and is not retryable, so the branching idiom below routes it to the + final ``raise``. Catch it to handle any USGS or EPA service failure uniformly, and branch on the read-anywhere fields below without needing the concrete subclass:: @@ -256,15 +267,18 @@ class NetworkError(DataRetrievalError): class ConfigurationError(DataRetrievalError, ValueError): - """A ``dataretrieval`` setting holds a value that can't be used. - - The setting may be an environment variable or a policy field; either way, - no request was issued. + """A ``dataretrieval`` setting holds a value that can't be used, so no + request was issued -- an environment variable, a policy field, a malformed + ``config.toml``, or a profile the file does not define. It is a :class:`DataRetrievalError` so ``except`` around a retrieval catches - it rather than letting a bare ``ValueError`` escape a request path, and a - :class:`ValueError` so code that already treats a bad setting as one keeps - working. + it rather than letting a bare ``ValueError`` escape a request path. That + matters because settings resolve lazily, on the request path: a broken + config file surfaces from inside whichever getter runs first, and belongs in + the same handler as any other failure of that call. It is *also* a + :class:`ValueError`, so code that already treats a bad setting as one keeps + working whether the value came from the environment, a file, or a + :func:`dataretrieval.configure` block. """ diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index 154899fa..c737fbe2 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -18,18 +18,28 @@ from __future__ import annotations from collections.abc import Iterable -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import pandas as pd +from dataretrieval import settings as _settings from dataretrieval.codes.states import apply_state from dataretrieval.credentials import WATERDATA_BASE_URL from dataretrieval.ogc import OgcDialect, get_ogc_data, prepare_request_args +from dataretrieval.settings import ( + AdapterSettings, + _Chunked, + _Concurrent, + _Redirectable, + _register, + _Retrying, +) if TYPE_CHECKING: from dataretrieval._response_metadata import BaseMetadata __all__ = [ + "NgwmnSettings", "get_sites", "get_water_level", "get_lithology", @@ -104,8 +114,13 @@ def _get(service: str, local_vars: dict[str, Any]) -> tuple[pd.DataFrame, BaseMe args, service, output_id=_NGWMN_OUTPUT_ID, - base_url=NGWMN_OGC_API_URL, + # A ``NgwmnSettings(base_url=...)`` from an enclosing block, or + # this service's own base. Resolved per call because the block is + # scoped to a ``with`` statement, and read here because this is the one + # place the NGWMN base is named. + base_url=_settings.base_url(adapter="ngwmn", default=NGWMN_OGC_API_URL), dialect=NGWMN_DIALECT, + adapter="ngwmn", ) @@ -428,3 +443,45 @@ def get_providers( ... ) """ return _get("providers", locals()) + + +class NgwmnSettings(_Chunked, _Concurrent, _Redirectable, _Retrying, AdapterSettings): + """Settings for NGWMN calls alone. + + NGWMN is a second OGC API on the Water Data host, so its queries + divide along the same URL byte budget and take the same two fan-out + dials. The API key is not among them: one gateway fronts both + adapters, so one key and one quota pool serve them (ADR 0010). + + Lives here rather than in :mod:`dataretrieval.settings` because + *which* settings a service reads is the service's own knowledge (ADR + 0011); what each of them means is shared, so the fields come from the + setting groups declared beside their grammar. + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying + stops. + base_url : str, optional + OGC API base to send NGWMN requests to, instead of the service's + own (``NGWMN_OGC_API_URL``). Code only: the file and the + environment refuse it. The API key is scoped to the host that + honors it, so a redirected call carries no key. + concurrency : int or str, optional + Cap on simultaneous sub-requests, or ``"unbounded"``. + parallel_chunks : int, optional + Baseline fan-out for multi-value queries. Each sub-request spends + rate-limit quota, so raise it only for pulls you know are large. + """ + + # NGWMN rides the same OGC engine as Water Data, so it reads the same + # groups: retry dials, a redirectable base, and both fan-out dials. The + # settings themselves are declared once in + # :mod:`dataretrieval.settings`, beside the grammar that parses them. + adapter: ClassVar[str] = "ngwmn" + + +_register(NgwmnSettings) diff --git a/dataretrieval/nldi.py b/dataretrieval/nldi.py index c32efd10..c8ea61c5 100644 --- a/dataretrieval/nldi.py +++ b/dataretrieval/nldi.py @@ -11,11 +11,19 @@ from __future__ import annotations from json import JSONDecodeError -from typing import Any, Literal, cast +from typing import Any, ClassVar, Literal, cast +from dataretrieval import settings as _settings from dataretrieval._querying import _query_with_retry +from dataretrieval.settings import ( + AdapterSettings, + _Redirectable, + _register, + _Retrying, +) __all__ = [ + "NldiSettings", "get_flowlines", "get_basin", "get_features", @@ -35,6 +43,23 @@ _VALID_NAVIGATION_MODES = ("UM", "DM", "UT", "DD") +def _api_base() -> str: + """The NLDI base this call targets: a block's redirect, or the service's. + + Every URL below is built from this rather than from + :data:`NLDI_API_BASE_URL` directly, so a ``NldiSettings(base_url=...)`` + reaches every navigation, basin, and catalog request alike -- a redirect + that covered only some of them would leave the library asking the real + service about the mirror's data. Resolved per call, because a ``configure`` + block is scoped to a ``with`` statement rather than to the process. + + Six call sites, which is what this seam is for; choosing between the + redirect and the service's own base is the accessor's job, not each + service's. + """ + return _settings.base_url(adapter="nldi", default=NLDI_API_BASE_URL) + + def _query_nldi( url: str, query_params: dict[str, str], @@ -42,7 +67,7 @@ def _query_nldi( # A helper function to query the NLDI API. ``query()`` already raises a # typed ``DataRetrievalError`` for any HTTP error response, so a returned # response is a success that we only need to parse. - response = _query_with_retry(url, payload=query_params) + response = _query_with_retry(url, payload=query_params, adapter="nldi") response_data: dict[str, Any] | list[Any] = {} try: response_data = response.json() @@ -185,7 +210,7 @@ def get_basin( if not feature_id: raise ValueError("feature_id is required") - url = f"{NLDI_API_BASE_URL}/{feature_source}/{feature_id}/basin" + url = f"{_api_base()}/{feature_source}/{feature_id}/basin" simplified_str = str(simplified).lower() split_catchment_str = str(split_catchment).lower() query_params = { @@ -295,7 +320,7 @@ def _navigation_request( string keeps its documented parameter order. """ origin = f"{feature_source}/{feature_id}" if feature_source else f"comid/{comid}" - url = f"{NLDI_API_BASE_URL}/{origin}/navigation/{navigation_mode}/{tail}" + url = f"{_api_base()}/{origin}/navigation/{navigation_mode}/{tail}" return url, {"distance": str(distance)} @@ -326,7 +351,7 @@ def _get_features_request( "Provide only one origin type - feature_source and feature_id cannot" " be provided with lat or long" ) - return f"{NLDI_API_BASE_URL}/comid/position", {"coords": f"POINT({long} {lat})"} + return f"{_api_base()}/comid/position", {"coords": f"POINT({long} {lat})"} if (comid is not None or data_source is not None) and navigation_mode is None: raise ValueError( @@ -340,7 +365,7 @@ def _get_features_request( _validate_data_source(feature_source) if not navigation_mode: - return f"{NLDI_API_BASE_URL}/{feature_source}/{feature_id}", {} + return f"{_api_base()}/{feature_source}/{feature_id}", {} navigation_mode = _validate_navigation_mode(navigation_mode) url, query_params = _navigation_request( @@ -385,7 +410,7 @@ def get_features_by_data_source(data_source: str) -> gpd.GeoDataFrame: """ # validate the data source _validate_data_source(data_source) - url = f"{NLDI_API_BASE_URL}/{data_source}" + url = f"{_api_base()}/{data_source}" feature_collection = cast("dict[str, Any]", _query_nldi(url, {})) gdf = _features_to_gdf(feature_collection) return gdf @@ -533,7 +558,7 @@ def _validate_data_source(data_source: str) -> None: # get the available data/feature sources - if not already cached if _AVAILABLE_DATA_SOURCES is None: - url = f"{NLDI_API_BASE_URL}/" + url = f"{_api_base()}/" available_data_sources = _query_nldi(url, {}) if not isinstance(available_data_sources, list) or not all( isinstance(ds, dict) and "source" in ds for ds in available_data_sources @@ -583,3 +608,41 @@ def _validate_feature_source_comid( raise ValueError( "Specify one origin type - comid or feature_source is required" ) + + +class NldiSettings(_Redirectable, _Retrying, AdapterSettings): + """Settings for NLDI calls alone. + + No fan-out dials: an NLDI query is answered by a single request. + + This adapter is imported on demand for the geopandas extra, so this + class registers itself later than the rest -- which is exactly why + the adapter roster lives in :data:`~dataretrieval.settings.ADAPTERS` + rather than being derived from what has been imported. + + Lives here rather than in :mod:`dataretrieval.settings` because + *which* settings a service reads is the service's own knowledge (ADR + 0011); what each of them means is shared, so the fields come from the + setting groups declared beside their grammar. + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying + stops. + base_url : str, optional + Linked-data base to send NLDI requests to, instead of the + service's own (``NLDI_API_BASE_URL``). Every navigation, basin + and catalog request is built on it. Code only: the file and the + environment refuse it. + """ + + # One request per call, so this service reads the retry dials and a + # redirectable base and no fan-out dial. Each setting is declared once, + # in :mod:`dataretrieval.settings`, beside its grammar. + adapter: ClassVar[str] = "nldi" + + +_register(NldiSettings) diff --git a/dataretrieval/nwdc.py b/dataretrieval/nwdc.py new file mode 100644 index 00000000..7a7d585c --- /dev/null +++ b/dataretrieval/nwdc.py @@ -0,0 +1,533 @@ +"""Retrieve USGS water-use data from the NWDC web service. + +The National Water Availability Assessment Data Companion (NWDC) web services +provide national-scale, USGS-modeled water-use data that underlie the `USGS +National Water Availability Assessment `_. +Estimates are served on a HUC12 (12-digit hydrologic unit) spatial grid and can +be queried for any county, state, or hydrologic unit. This is the modern +replacement for the defunct legacy NWIS water-use service +(``nwis.get_water_use``). + +Unlike the main Water Data getters (:mod:`dataretrieval.waterdata`) and NGWMN +(:mod:`dataretrieval.ngwmn`), the NWDC is a plain CSV REST service rather than +an OGC API Features collection. This module supplies the NWDC-specific bits — +request building, CSV parsing, the ``Link``-header cursor, and the ``{detail}`` +error envelope. The service-neutral transport layer supplies cursor pagination, +response aggregation, client lifecycle, and sync-from-async dispatch. The module +follows the same conventions: host-scoped request headers, the typed +:class:`~dataretrieval.exceptions.DataRetrievalError` taxonomy, and a +``(DataFrame, BaseMetadata)`` return. + +See https://api.water.usgs.gov/docs/nwaa-data/ for the API reference and +https://water.usgs.gov/nwaa-data/ for the catalog of available models and +variables. + +Examples +-------- +.. code-block:: python + + from dataretrieval import nwdc + + # Monthly public-supply withdrawals for Rhode Island, 2020 onward. + df, md = nwdc.get_wateruse( + model="wu-public-supply-wd", + variable=["pswdtot", "pswdgw", "pswdsw"], + state="RI", + start_date="2020-01", + time_resolution="monthly", + ) + +""" + +from __future__ import annotations + +import io +from collections.abc import Callable, Iterable +from typing import Any, ClassVar + +import httpx +import pandas as pd + +from dataretrieval import settings as _settings +from dataretrieval._querying import _raise_for_status, to_str +from dataretrieval._response_metadata import BaseMetadata +from dataretrieval.codes.states import to_state +from dataretrieval.exceptions import DataRetrievalError +from dataretrieval.settings import ( + AdapterSettings, + _Concurrent, + _Redirectable, + _register, + _Retrying, +) +from dataretrieval.transport.fanout import FanOut, active_client +from dataretrieval.transport.http import default_headers, network_error +from dataretrieval.transport.links import resolve_next_url +from dataretrieval.transport.pagination import paginate +from dataretrieval.transport.retry import RetryPolicy + +__all__ = [ + "NwdcSettings", + "get_wateruse", + "WATERUSE_URL", + "MODELS", + "TIME_RESOLUTIONS", + "DEFAULT_CONCURRENT_REQUESTS", +] + +WATERUSE_URL = "https://api.water.usgs.gov/nwaa-data/data" +_WATERUSE_HOST = httpx.URL(WATERUSE_URL).host +# Hosts a ``rel="next"`` cursor may name for this same service; each is +# rewritten to :data:`_WATERUSE_HOST` rather than followed as given. +_WATERUSE_HOST_ALIASES = frozenset({_WATERUSE_HOST, "water.usgs.gov"}) + +#: Water-use models (categories) served by the NWDC. The catalog at +#: https://water.usgs.gov/nwaa-data/ lists the variables available within each. +MODELS = ( + "wu-public-supply-wd", # public-supply withdrawals + "wu-public-supply-cu", # public-supply consumptive use + "wu-thermoelectric", # thermoelectric-power water use + "wu-irrigation-wd", # irrigation withdrawals + "wu-irrigation-cu", # irrigation consumptive use +) + +#: Temporal resolutions: monthly, annual calendar year, annual water year. +TIME_RESOLUTIONS = ("monthly", "annualcy", "annualwy") + +#: This service's preferred in-flight cap when nothing is configured. Lower +#: than the package default of 32 because every location retries +#: independently, so a rate-limit episode bursts this number times the retry +#: count; the NWDC tolerates this level without rate-limit errors (verified by +#: stress test) and higher has not been tested. Any configured concurrency +#: overrides it -- see :func:`dataretrieval.settings.concurrency` for why the +#: general setting outranks a module's default rather than the reverse. +DEFAULT_CONCURRENT_REQUESTS = 4 + +# Page responses carry the HUC12 identifier in this column; it must stay a +# string so leading zeros (e.g. "010900020502") survive the round trip. +_HUC12_COLUMN = "huc12_id" + + +def get_wateruse( + model: str, + variable: str | Iterable[str] | None = None, + state: str | int | Iterable[str | int] | None = None, + county: str | Iterable[str] | None = None, + huc: str | Iterable[str] | None = None, + time_resolution: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + intersection: str = "overlap", + limit: int = 600, + ssl_check: bool = True, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get USGS water-use data from the NWDC web service. + + Retrieves modeled water-use estimates from the USGS National Water + Availability Assessment Data Companion. The area is given as exactly one of + ``state``, ``county``, or ``huc``; results are always returned on a HUC12 + grid, in a long (tidy) frame with one row per HUC12 and time step. Large + areas (e.g. a whole region or a populous state) are served across multiple + pages; this function follows those pages transparently and concatenates + them into one frame. + + Each selector also accepts a list of values. The NWDC queries one area per + request, so a list is fanned out into one request per value — up to the + effective ``concurrency`` setting in parallel, defaulting to + :data:`DEFAULT_CONCURRENT_REQUESTS` for this service — and the results are + concatenated in the order given. That cap resolves through the + configuration chain, so it can be raised or lowered for this service alone + (``configure(NwdcSettings(concurrency=2))``, or an ``[nwdc]`` table in + the config file) as well as package-wide via ``API_USGS_CONCURRENT``; see + :doc:`the settings guide `. A fan-out + interrupted by a rate limit or an upstream fault raises a resumable + :class:`~dataretrieval.interruptions.FanOutInterrupted`, whose + ``.call.resume()`` re-issues only the locations that did not complete. + + Parameters + ---------- + model : string + Water-use category to query. See :data:`MODELS` for the available + options (e.g. ``"wu-public-supply-wd"``). The full catalog of models + and their variables is at https://water.usgs.gov/nwaa-data/. + variable : string or iterable of strings, optional + One or more variable IDs within ``model`` (e.g. ``"pswdtot"`` for total + public-supply withdrawals, or ``["pswdgw", "pswdsw"]`` for the + groundwater and surface-water components). Multiple variables are + comma-joined into a single request. The service requires at least one + variable; omitting it returns a 400 listing the model's valid variable + IDs (surfaced as a :class:`~dataretrieval.exceptions.DataRetrievalError`). + state : string, int, or iterable, optional + One or more US states/territories to query. Each accepts a full name + (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit + ANSI/FIPS code (``"55"`` or ``55``), mirroring + :func:`dataretrieval.ngwmn.get_sites`. + county : string or iterable, optional + One or more five-digit county FIPS codes — state FIPS + county FIPS, + e.g. ``"55025"`` for Dane County, Wisconsin. + huc : string or iterable, optional + One or more hydrologic unit codes. Each code's level is taken from its + length: a 2-digit code queries a HUC2 region, 8-digit a HUC8 subbasin, + 12-digit a single HUC12, and so on (even lengths 2-12, e.g. ``"04"``, + ``"07070005"``, ``"010900020502"``). + + Provide exactly one of ``state``, ``county``, or ``huc`` (each may be a + single value or a list). + time_resolution : string, optional + Temporal resolution: ``"monthly"``, ``"annualcy"`` (annual, calendar + year), or ``"annualwy"`` (annual, water year). See + :data:`TIME_RESOLUTIONS`. + start_date : string, optional + Start of the query window, formatted ``"YYYY"`` for annual data or + ``"YYYY-MM"`` for monthly data. + end_date : string, optional + End of the query window, in the same format as ``start_date``. + intersection : string, optional + How to select HUC12s that straddle the queried-area boundary: + ``"overlap"`` (any overlap, the default) or ``"envelop"`` (fully + enclosed). + limit : int, optional + Maximum number of HUC12s returned per page. Queries spanning more than + ``limit`` HUC12s are split across pages and reassembled. Default 600. + ssl_check : bool, optional + If True (default), verify SSL certificates; set False to skip + verification (e.g. behind a TLS-intercepting proxy). + + Returns + ------- + df : ``pandas.DataFrame`` + Water-use estimates in long form: a ``huc12_id`` column (string, + leading zeros preserved), a time column (``year_month`` for monthly + data or ``year`` for annual data), and one value column per requested + variable (suffixed with its unit, e.g. ``pswdtot_mgd`` for million + gallons per day). + md : :class:`dataretrieval.utils.BaseMetadata` + Metadata describing the request (URL, query time, response headers). + + Raises + ------ + ValueError + If not exactly one of ``state``, ``county``, or ``huc`` is given, or a + given selector is malformed (an unrecognized state, a county code that + is not five digits, or a HUC of invalid length). + DataRetrievalError + On an HTTP error response, the typed subclass for the status (see + :func:`dataretrieval.exceptions.error_for_status`). A transient 429, + 5xx, or recoverable connection failure that exhausts inline retries is + raised as a resumable + :class:`~dataretrieval.interruptions.FanOutInterrupted`; a deterministic + connection failure (for example, a permanently unresolvable host) + remains a :class:`~dataretrieval.exceptions.NetworkError`. + + Examples + -------- + .. doctest:: + :skipif: True # network + + >>> from dataretrieval import nwdc + >>> df, md = nwdc.get_wateruse( + ... model="wu-public-supply-wd", + ... variable=["pswdtot", "pswdgw", "pswdsw"], + ... state="RI", + ... start_date="2020-01", + ... time_resolution="monthly", + ... ) + + """ + # The public parameters are idiomatic snake_case (consistent with + # ``waterdata.get_samples``); the NWDC service expects compact lowercase + # query names, so map to those here as the request is built. + base_params: dict[str, Any] = { + "format": "csv", + "model": model, + "variable": to_str(variable), + "timeres": time_resolution, + "startdate": start_date, + "enddate": end_date, + "intersection": intersection, + "limit": limit, + } + # Drop params the caller left unset; the service rejects empty values. + base_params = {k: v for k, v in base_params.items() if v is not None} + + # An ``NwdcSettings(base_url=...)`` from an enclosing block, or this + # service's own endpoint. Resolved once per call -- the block is scoped to + # a ``with`` statement -- and threaded through every request and the page + # walk, so a redirected call cannot half-follow the redirect. + service_url = _settings.base_url(adapter="nwdc", default=WATERUSE_URL) + + # The NWDC queries one location per request, so fan a multi-value selector + # out into one request per location, each handled by shared transport + # pagination, and concatenate the results. + headers = default_headers(service_url) + requests = [ + httpx.Request( + "GET", + service_url, + params={**base_params, "location": location}, + headers=headers, + ) + for location in _resolve_locations(state, county, huc) + ] + return _fan_out(requests, headers, ssl_check, host=httpx.URL(service_url).host) + + +# Valid HUC code lengths (digits) → the hydrologic-unit level they query. +_HUC_LENGTHS = (2, 4, 6, 8, 10, 12) + +# Maps each selector to the NWDC ``location=:`` value(s) it produces. +# A value may be a single code or a list; ``_as_list`` normalizes both (``state`` +# additionally normalizes to the two-letter postal code, and ``to_state`` may +# itself return a scalar or list, which ``_as_list`` flattens the same way). +# Since NWDC takes one location per request, a list value fans out — one request +# per location (see :func:`_fan_out`). +_LOCATION_BUILDERS: dict[str, Callable[[Any], list[str]]] = { + "state": lambda v: [f"stateCd:{c}" for c in _as_list(to_state(v, to="postal"))], + "county": lambda v: [f"countyCd:{_validate_county(c)}" for c in _as_list(v)], + "huc": lambda v: [f"huc{len(c)}:{c}" for c in map(_validate_huc, _as_list(v))], +} + + +def _resolve_locations( + state: str | int | Iterable[str | int] | None, + county: str | Iterable[str] | None, + huc: str | Iterable[str] | None, +) -> list[str]: + """Build the NWDC ``location=:`` value(s) from the selectors. + + Exactly one of ``state`` / ``county`` / ``huc`` must be given; each may be a + single value or a list. ``state`` is normalized to the two-letter postal + code ``stateCd`` requires; ``county`` is a five-digit FIPS code; and a + ``huc`` code's length selects its level (``huc2`` … ``huc12``). Returns one + location string per value — the caller issues one request per location. + """ + selected = { + name: value + for name, value in (("state", state), ("county", county), ("huc", huc)) + if value is not None + } + if len(selected) != 1: + raise ValueError( + "Specify exactly one of state, county, or huc " + f"(got: {', '.join(selected) or 'none'})." + ) + [(name, value)] = selected.items() + locations = _LOCATION_BUILDERS[name](value) + if not locations: + raise ValueError( + "The chosen location selector is empty; pass at least one value." + ) + return locations + + +def _as_list(value: object) -> list[Any]: + """Normalize a value to a list. + + A scalar becomes a one-element list; any non-string iterable (list, tuple, + Series, ndarray, generator) is materialized to a list. A string is treated + as a scalar so it isn't exploded into characters. + """ + if isinstance(value, Iterable) and not isinstance(value, str): + return list(value) + return [value] + + +def _validate_county(value: object) -> str: + """Validate and normalize a five-digit state+county FIPS code.""" + code = str(value).strip() + if not (code.isdigit() and len(code) == 5): + raise ValueError( + "county must be a five-digit state+county FIPS code " + f"(e.g. '55025'), got {value!r}." + ) + return code + + +def _validate_huc(value: object) -> str: + """Validate a HUC code (even length 2-12 digits; level set by length).""" + code = str(value).strip() + if not (code.isdigit() and len(code) in _HUC_LENGTHS): + raise ValueError( + "huc must be a hydrologic unit code of even length 2-12 digits " + f"(e.g. '04', '07070005', '010900020502'), got {value!r}." + ) + return code + + +def _fan_out( + requests: list[httpx.Request], + headers: dict[str, str], + ssl_check: bool, + *, + host: str = _WATERUSE_HOST, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Fetch every request (each paginated) over the shared fan-out executor. + + Each request is paginated by :func:`dataretrieval.transport.pagination.paginate` + with NWDC strategies: parse a CSV page and read its ``Link`` header cursor + (``parse``), follow that cursor (``follow``), and raise the typed error + carrying the NWDC ``detail`` (``raise_for_status``). + + Everything else -- bounded concurrency, per-attempt retry, failure + precedence, progress, and resumable interruption -- belongs to + :class:`~dataretrieval.transport.fanout.FanOut`, which Water Data and NGWMN + drive too. This function is now only the NWDC-specific half: what a + chunk is, and how to read one. + + The plan is the request list itself. ``FanOut`` asks a plan only to be + sized and iterable, and the NWDC accepts one ``location=`` per request, so + the caller's locations arrive already separate -- there is nothing to + divide and so nothing for a plan class to hold. + + The broad retry status set is on purpose: NWDC reports a bad query as a 400 + with a ``{"detail": ...}`` envelope, so unlike WQP and StreamStats its 5xx + really is an upstream fault worth re-sending. + """ + + def parse(response: httpx.Response) -> tuple[pd.DataFrame, str | None]: + return _read_csv_page(response), _next_page_url(response, host=host) + + async def follow(cursor: str, sess: httpx.AsyncClient) -> httpx.Response: + return await sess.get(cursor, headers=headers) + + def raise_for_status(response: httpx.Response) -> None: + _raise_for_status(response, detail_from=_nwdc_error_detail) + + async def fetch(request: httpx.Request) -> tuple[pd.DataFrame, httpx.Response]: + """One location's full page walk, over the executor's shared client. + + ``active_client()`` is the client :meth:`FanOut._run` published for this + run; borrowing it keeps every location's pages on one connection pool + instead of opening a client per location. + """ + try: + return await paginate( + request, + parse_response=parse, + follow_up=follow, + client=active_client(), + raise_for_status=raise_for_status, + ) + except httpx.TransportError as exc: + raise network_error(request.url, exc) from exc + + def finalize( + frame: pd.DataFrame, response: httpx.Response + ) -> tuple[pd.DataFrame, BaseMetadata]: + return frame, BaseMetadata(response) + + return FanOut( + requests, + fetch, + RetryPolicy.from_settings(adapter="nwdc"), + finalize=finalize, + client_options={"verify": ssl_check}, + default_concurrent=DEFAULT_CONCURRENT_REQUESTS, + # No single URL expresses "all of these locations" -- the service + # has no such request -- so the aggregate reports the first, + # matching what an un-fanned single-location call would show. + canonical_url=str(requests[0].url) if requests else None, + # Labels the progress line the executor opens for this drive. + service="nwdc", + adapter="nwdc", + ).resume() + + +def _read_csv_page(response: httpx.Response) -> pd.DataFrame: + """Parse one CSV page; ``huc12_id`` stays a string to keep leading zeros.""" + try: + return pd.read_csv(io.BytesIO(response.content), dtype={_HUC12_COLUMN: str}) + except pd.errors.EmptyDataError as exc: + # NWDC normally signals "no data" with a 400 (handled above) or rows of + # zeros, never an empty body — but keep the typed-error contract if it + # ever returns one rather than leaking a bare pandas exception. + raise DataRetrievalError( + f"NWDC returned an empty response body (URL: {response.url})." + ) from exc + + +def _next_page_url( + response: httpx.Response, *, host: str = _WATERUSE_HOST +) -> str | None: + """Return the absolute URL of the next page, or None if this is the last. + + Reads the standard ``Link: <...>; rel="next"`` header (parsed by httpx into + ``response.links``). The cursor is normalized before it is trusted, because + the service spells it inconsistently. A relative reference is resolved + against the page it came from, and the bare ``water.usgs.gov`` host is + rewritten to the public ``api.water.usgs.gov`` gateway (over https, whatever + scheme the link used) so the follow-up request reaches the API. Only a + cursor that still points somewhere else after that is refused -- following + it would send Water Use requests, and any credentials on them, to a host the + caller never asked for. + + ``host`` is the host this call is actually talking to, which is not the + NWDC's when a ``configure`` block redirected the adapter. The alias list and + the rewrite are facts about *this* service -- nothing else answers for + ``water.usgs.gov`` -- so a redirected call gets the general rule instead: + follow a link only back to the host that served the page. Applying the + NWDC's rewrite there would send page two of a mirrored query to the USGS. + """ + url = response.links.get("next", {}).get("url") + if not url: + return None + if host != _WATERUSE_HOST: + return resolve_next_url(url, response, service="Water Use") + return resolve_next_url( + url, + response, + service="Water Use", + allowed_hosts=_WATERUSE_HOST_ALIASES, + rewrite_host=_WATERUSE_HOST, + ) + + +def _nwdc_error_detail(response: httpx.Response) -> str | None: + """Pull the ``detail`` message out of an NWDC JSON error envelope, if any. + + The NWDC reports errors as ``{"detail": "Invalid model name: ..."}``. Passed + to :func:`~dataretrieval.utils._raise_for_status` as ``detail_from`` so the + service's wording surfaces in the typed error message. + """ + try: + body = response.json() + except ValueError: + return None + return body.get("detail") if isinstance(body, dict) else None + + +class NwdcSettings(_Concurrent, _Redirectable, _Retrying, AdapterSettings): + """Settings for NWDC calls alone. + + No ``parallel_chunks``: the NWDC is a plain CSV service, so a query + fans out per location rather than being divided along a URL byte + budget. There is nothing for the planner to divide more finely. + + Lives here rather than in :mod:`dataretrieval.settings` because + *which* settings a service reads is the service's own knowledge (ADR + 0011); what each of them means is shared, so the fields come from the + setting groups declared beside their grammar. + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying + stops. + base_url : str, optional + Endpoint to send NWDC requests to, instead of the service's own + (``WATERUSE_URL``). A ``rel="next"`` cursor is then followed only + back to that host, since the service's own host aliases mean + nothing there. Code only: the file and the environment refuse it. + concurrency : int or str, optional + Cap on simultaneous sub-requests, or ``"unbounded"``. + """ + + # One request per location, fanned out but never chunked, so this service + # reads the retry dials, a redirectable base and ``concurrency`` -- but not + # ``parallel_chunks``, which divides a query it never divides. + adapter: ClassVar[str] = "nwdc" + + +_register(NwdcSettings) diff --git a/dataretrieval/nwis.py b/dataretrieval/nwis.py index e8354d71..58fd3643 100644 --- a/dataretrieval/nwis.py +++ b/dataretrieval/nwis.py @@ -734,16 +734,16 @@ def get_pmcodes(**kwargs: Any) -> NoReturn: def get_water_use(**kwargs: Any) -> NoReturn: - """Defunct: use ``dataretrieval.wateruse.get_wateruse`` instead. + """Defunct: use ``dataretrieval.nwdc.get_wateruse`` instead. The legacy NWIS water-use service has been retired. Modeled water-use estimates are now served by the National Water Availability Assessment Data Companion (NWDC); retrieve them with - :func:`dataretrieval.wateruse.get_wateruse`. + :func:`dataretrieval.nwdc.get_wateruse`. """ raise NameError( "`nwis.get_water_use` is defunct; use " - "`dataretrieval.wateruse.get_wateruse` instead." + "`dataretrieval.nwdc.get_wateruse` instead." ) diff --git a/dataretrieval/ogc/chunking.py b/dataretrieval/ogc/chunking.py index c958b49d..d8a17d0f 100644 --- a/dataretrieval/ogc/chunking.py +++ b/dataretrieval/ogc/chunking.py @@ -24,8 +24,9 @@ :meth:`ChunkPlan._refine`; see ``parallel_chunks`` for the why and the when. Concurrency, retries, and interruption semantics are documented on -:mod:`dataretrieval.transport.fanout`; ``API_USGS_CONCURRENT`` and -``API_USGS_RETRIES`` are read there. +:mod:`dataretrieval.transport.fanout`; the ``concurrency`` and ``retries`` +settings are resolved there, through the chain in +:mod:`dataretrieval.settings`. Dedup: list-axis chunks don't overlap; filter-axis chunks can, so ``_combine_chunk_frames`` dedupes by feature ``id``. ``properties``, @@ -44,7 +45,7 @@ import httpx import pandas as pd -from dataretrieval._ambient import Ambient +from dataretrieval import settings as _settings from dataretrieval.transport.fanout import ( FanOut, _active_client, @@ -56,7 +57,6 @@ from dataretrieval.transport.retry import RetryPolicy from .planning import ChunkPlan -from .policy import _require_positive_int # Compatibility aliases. ``ChunkedCall`` was this module's executor before it # moved down to transport as the API-neutral ``FanOut``; ``get_active_client`` @@ -78,15 +78,6 @@ _OGC_URL_BYTE_LIMIT = 8000 -# Parallel-chunks dial: opt-in to fan a query out *more finely* than the byte -# limit alone requires. Scoped to a ``with parallel_chunks(...):`` block (a -# ContextVar), deliberately NOT an env var (see :func:`parallel_chunks` for -# why). The ambient holds ``n`` — the requested cap on the plan's total -# chunk count; ``1`` (the default, outside any block) means "off — chunk -# only as much as the byte limit needs, no extra fan-out". -_parallel_chunks: Ambient[int] = Ambient("ogc_parallel_chunks", 1) - - @contextmanager def parallel_chunks(n: int) -> Iterator[None]: """ @@ -133,9 +124,9 @@ def parallel_chunks(n: int) -> Iterator[None]: Each chunk fetches at least one page, so it costs at least one request against your hourly rate limit — a larger ``n`` spends more quota. How many chunks run *at once* is capped separately by - ``API_USGS_CONCURRENT`` (default 32), so an ``n`` beyond that adds - quota without adding parallelism; the useful range is roughly ``2`` - up to ``API_USGS_CONCURRENT``. + the ``concurrency`` setting (default 32), so an ``n`` beyond that + adds quota without adding parallelism; the useful range is roughly + ``2`` up to the effective ``concurrency``. Yields ------ @@ -183,10 +174,23 @@ def parallel_chunks(n: int) -> Iterator[None]: -------- ChunkPlan._refine : the planning-side effect of ``n``. """ - # Fail loudly on a bad ``n`` at ``with`` entry, before any request. Shared - # rules with ``max_rows`` via the helper (accepts numpy ints, rejects bool). - _require_positive_int(n, "parallel_chunks(n)", examples="2, 8, 32") - with _parallel_chunks(n): + # Fail loudly on a bad ``n`` at ``with`` entry, before any request -- and + # fail by the *setting's* grammar, not a second one written here. ``n`` is + # ``parallel_chunks``: the same bool/Integral rejection and the same lower + # bound, from the table that owns them, so raising the floor there cannot + # leave this block accepting a value the chain would then refuse. Spelled + # with the source label this block is written as, so the message names + # ``parallel_chunks(n)`` rather than the ``Settings`` built below. + # ``ConfigurationError`` is a ``ValueError``, so callers catching that + # still catch this. + _settings._require("parallel_chunks", n, "parallel_chunks(n)") + # Sugar for a package-wide ``Settings`` rather than a second scope of + # its own: two competing ContextVars would let ``show_settings()`` report a + # value the chunker does not use. Sharing one means the innermost block + # wins, whichever spelling opened it -- and package-wide rather than scoped + # to one adapter, because this block is a per-call request that must reach + # whichever adapter the call goes to. + with _settings.configure(_settings.Settings(parallel_chunks=n)): yield @@ -194,6 +198,7 @@ def multi_value_chunked( *, build_request: Callable[..., httpx.Request], url_limit: int | None = None, + adapter: str | None = None, ) -> Callable[[_Fetch[dict[str, Any]]], Callable[..., tuple[pd.DataFrame, Any]]]: """ Decorate an async fetcher to transparently chunk over-budget requests. @@ -251,17 +256,21 @@ def wrapper( finalize: _Finalize = _passthrough_result, ) -> tuple[pd.DataFrame, Any]: limit = _OGC_URL_BYTE_LIMIT if url_limit is None else url_limit - # Read the parallel_chunks dial ``n`` from the ambient set by - # ``parallel_chunks`` (1 = off outside any such block; otherwise the - # requested total chunk cap). It only affects *planning*, done - # here up front, so a later resume — which re-issues the - # already-planned chunks — needs no snapshot. + # Resolve the parallel_chunks dial ``n`` through the configuration + # chain (1 = off unless a ``parallel_chunks``/``configure`` block or + # the config file raised it; otherwise the requested total chunk + # cap). It only affects *planning*, done here up front, so a later + # resume — which re-issues the already-planned chunks — reuses this + # plan rather than resolving again. plan = ChunkPlan( - args, build_request, limit, max_chunks=_parallel_chunks.get() + args, + build_request, + limit, + max_chunks=_settings.parallel_chunks(adapter=adapter), ) - retry_policy = RetryPolicy.from_env() - # The concurrency cap is resolved inside ``resume()`` from - # ``API_USGS_CONCURRENT``; ``1`` is a sequential gather, + retry_policy = RetryPolicy.from_settings(adapter=adapter) + # The concurrency cap is resolved inside ``resume()`` through the + # configuration chain; ``1`` is a sequential gather, # ``total <= 1`` a one-element gather — no special branch. return ChunkedCall( plan, @@ -272,6 +281,7 @@ def wrapper( # The collection name, for the progress line the executor # opens. ``get_ogc_data`` puts it in ``args``. service=args.get("collection"), + adapter=adapter, ).resume() return wrapper diff --git a/dataretrieval/ogc/engine.py b/dataretrieval/ogc/engine.py index 87e956d7..d0c6b5d0 100644 --- a/dataretrieval/ogc/engine.py +++ b/dataretrieval/ogc/engine.py @@ -231,6 +231,7 @@ def get_ogc_data( max_rows: int | None = None, extra_id_cols: frozenset[str] | set[str] = frozenset(), dialect: OgcDialect | None = None, + adapter: str | None = None, ) -> tuple[pd.DataFrame, BaseMetadata]: """ Retrieves OGC (Open Geospatial Consortium) data as a DataFrame with metadata. @@ -336,7 +337,9 @@ def get_ogc_data( fetch = functools.partial( _fetch_once, build_request=build_request, row_cap=max_rows ) - run = chunking.multi_value_chunked(build_request=build_request)(fetch) + run = chunking.multi_value_chunked(build_request=build_request, adapter=adapter)( + fetch + ) # No progress block here: the executor that emits the events owns the line # (see :meth:`~dataretrieval.transport.fanout.FanOut.resume`). return run(args, finalize=finalize) @@ -358,9 +361,9 @@ async def _fetch_once( URL fits, and iterates the cartesian product. With no chunkable inputs the decorator passes args through unchanged. The decorator gathers every chunk over one shared :class:`httpx.AsyncClient` (concurrency - bounded by a semaphore, sized from ``API_USGS_CONCURRENT``) and - returns a *synchronous* wrapper, so ``get_ogc_data`` drives it - synchronously. The return shape is ``(frame, response)``. + bounded by a semaphore, sized from the effective ``concurrency`` + setting) and returns a *synchronous* wrapper, so ``get_ogc_data`` drives + it synchronously. The return shape is ``(frame, response)``. """ req = build_request(**args) return await _walk_pages(geopd=GEOPANDAS, req=req, row_cap=row_cap) @@ -370,6 +373,7 @@ def fetch_ogc_request( request: httpx.Request, *, collection: str, + adapter: str | None = None, ) -> tuple[pd.DataFrame, httpx.Response]: """Execute a prepared OGC request with pagination, returning (df, response). @@ -402,7 +406,8 @@ async def _fetch(req: httpx.Request) -> tuple[pd.DataFrame, httpx.Response]: return FanOut( [request], _fetch, - RetryPolicy.from_env(), + RetryPolicy.from_settings(adapter=adapter), canonical_url=str(request.url), service=collection, + adapter=adapter, ).resume() diff --git a/dataretrieval/progress.py b/dataretrieval/progress.py index d59bff48..2f80d555 100644 --- a/dataretrieval/progress.py +++ b/dataretrieval/progress.py @@ -26,12 +26,12 @@ from __future__ import annotations -import os import sys from collections.abc import Iterator from contextlib import contextmanager from typing import TYPE_CHECKING, TextIO +from dataretrieval import settings as _settings from dataretrieval._ambient import Ambient from dataretrieval.credentials import SIGNUP_URL, accepts_api_key, api_key @@ -83,9 +83,12 @@ def _enabled_default(stream: TextIO) -> bool: a TTY or a Jupyter/IPython kernel — and stay quiet for redirected output, logs, and CI. """ - override = os.getenv("API_USGS_PROGRESS") + # config owns the grammar, so this is already a bool: the same value means + # the same thing whether it came from a configure() block, the environment, + # or the file. Re-parsing here is what let those three disagree. + override = _settings.progress() if override is not None: - return override.strip().lower() not in {"", "0", "false", "no", "off"} + return override if _in_jupyter_kernel(): return True return hasattr(stream, "isatty") and stream.isatty() diff --git a/dataretrieval/settings.py b/dataretrieval/settings.py new file mode 100644 index 00000000..a3e60762 --- /dev/null +++ b/dataretrieval/settings.py @@ -0,0 +1,2141 @@ +"""Layered settings resolution for ``dataretrieval``, built on pydantic-settings. + +Every tunable setting -- the Water Data API key, the fan-out concurrency cap, +the retry count, and the progress line -- resolves through one ordered chain so +a caller never has to mutate ``os.environ`` to configure a single call. + +Sources, highest precedence first: + +1. A settings profile passed to :func:`configure` -- delivered through a + :class:`~contextvars.ContextVar`, so a setting applies to the current thread + or asyncio task and cannot leak into another one. +2. The environment variable for that setting (``API_USGS_PAT``, + ``API_USGS_CONCURRENT``, ``API_USGS_RETRIES``, ``API_USGS_PROGRESS``). +3. The settings file (TOML): ``~/.dataretrieval/config.toml``, or the path in + ``DATARETRIEVAL_CONFIG``. Top-level keys are the package-wide defaults; a + ``[]`` table is that adapter's *default profile*, always in effect; + a ``[.]`` table is a *named profile*, inert until a caller + selects it with ``Settings.load("")``. +4. The built-in default. + +Those are the four *sources*. ADR 0011 states the same order as seven rungs by +splitting three of them into the scopes inside: source 1 into a settings +instance and a selected profile, which cannot disagree because both name one +adapter and two profiles for one adapter raise; source 3 into the +``[]`` table above the top-level keys; and source 4 into an adapter's +own built-in preference above the package default. That last scope is invisible +here because this module never supplies it -- it arrives as the ``default`` a +read site like :func:`concurrency` passes for its own service. + +Precedence applies **per setting**, not per source: an environment that sets only +``API_USGS_PAT`` leaves a file-provided ``concurrency`` fully in effect. That is +also how pydantic-settings merges sources, which is why the chain is expressed +as a source tuple rather than as hand-written fallbacks -- see +:meth:`AdapterSettings.settings_customise_sources`. + +A caller configures by passing settings profiles, at most one per adapter:: + + with dataretrieval.configure( + Settings(api_key=vault.read("usgs/pat")), + WaterdataSettings.load("bulk"), + NgwmnSettings(concurrency=4), + ): + ... + +Settings are scoped **per adapter** (ADR 0010): a ``[ngwmn]`` table in the file, +or an ``NgwmnSettings``, applies to NGWMN calls and no others. Precedence stays +*source-major*: the chain still walks block, then environment, then file, and an +adapter-scoped value outranks a package-wide one only *within* the same source. + +Which settings an adapter accepts is its own vocabulary -- ``concurrency`` means +nothing to an adapter that issues one request -- so each adapter declares them +on its own :class:`AdapterSettings` subclass, defined in the module that *reads* +them. The API key is not among them: it belongs to the gateway fronting a host, +which Water Data and NGWMN share. + +Why pydantic-settings (ADR 0012) +-------------------------------- + +The field declarations, the type coercion, the bounds, the "unknown setting" +rejection and the source merge are pydantic-settings'. What remains here is what +that library has no opinion about: the TOML *grammar* of adapter tables and +named profiles, the file cache, the provenance labels :func:`show_settings` +reports, and the ``ContextVar`` that carries a block. Those are wired in as +:class:`~pydantic_settings.PydanticBaseSettingsSource` subclasses, which is the +library's own extension point. + +This module is no longer a standard-library-only leaf -- ADR 0012 withdrew that +constraint deliberately. It still imports no adapter (see :data:`ADAPTERS`) and +nothing from ``dataretrieval`` other than the ``exceptions`` taxonomy leaf, so +it cannot cycle. +""" + +from __future__ import annotations + +import math +import os +import stat +import sys +import warnings +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass, field +from numbers import Integral +from pathlib import Path +from types import MappingProxyType +from typing import Any, ClassVar, Literal, TextIO, TypeVar, overload + +from pydantic import ValidationError, field_validator +from pydantic.fields import FieldInfo +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, +) + +from dataretrieval.exceptions import ConfigurationError + +# ``ConfigurationError`` is re-exported; its canonical home and rationale are in +# :mod:`dataretrieval.exceptions`. +__all__ = [ + "ADAPTERS", + # The package-wide settings, and the base every adapter subclasses. Public + # because a caller writes ``Settings(...)`` at every call site that + # configures anything, and an adapter module names the base in its own + # subclass. + "AdapterSettings", + "Settings", + "config_path", + "configure", + "settings_for", + "show_settings", +] + + +#: Settings only an adapter can carry, because they name one service. No +#: package-wide value could mean anything for them: there is no one base URL. +#: +#: The package-wide roster is :data:`SETTINGS`, declared below the class it is +#: derived from. +ADAPTER_ONLY_SETTINGS: tuple[str, ...] = ("base_url",) + +#: Environment variable backing a setting (precedence step 2). +#: +#: Not every setting has one. ``parallel_chunks`` is deliberately absent: it +#: fans a query into more sub-requests, each of which spends rate-limit quota, +#: and ``dataretrieval.parallel_chunks`` documents why that must stay a +#: deliberate choice rather than a process-wide default. +ENV_VARS: dict[str, str] = { + "api_key": "API_USGS_PAT", + "concurrency": "API_USGS_CONCURRENT", + "retries": "API_USGS_RETRIES", + "progress": "API_USGS_PROGRESS", + "stall_timeout": "API_USGS_STALL_TIMEOUT", +} + +#: Variables the environment is *refused* for, by setting. Named rather than +#: simply left out of :data:`ENV_VARS`, because leaving them out only makes the +#: environment silent: a caller who exports ``API_USGS_BASE_URL`` -- the +#: spelling every other setting's variable predicts -- has redirected nothing +#: and would learn that from the traffic rather than from us. +#: +#: Derived from :data:`ADAPTER_ONLY_SETTINGS` rather than written out beside it, +#: because the two would be spelling one fact -- "this setting is code-only" -- +#: in two tables with nothing keeping them in step. +_REFUSED_ENV_VARS: dict[str, str] = { + name: f"API_USGS_{name.upper()}" for name in ADAPTER_ONLY_SETTINGS +} + +#: Environment variable holding an explicit path to the settings file. +CONFIG_PATH_ENV = "DATARETRIEVAL_CONFIG" + +#: Source label for a setting no source supplied. +_BUILT_IN = "built-in default" + +#: The table ADR 0011 retired. Named here only so a file written against the +#: earlier design gets an error that says what to write instead. +_RETIRED_PROFILES_TABLE = "profiles" + +#: Label for the file's top-level table, where keys are the defaults. +_TOP_LEVEL = "top level" + +#: Settings that warn when written at the top level of the file, and what to +#: say. Declared as data, beside the other per-setting policies, so "what is +#: special about ``parallel_chunks``?" is answerable from this block rather than +#: from a condition buried in a validation loop. +_WARN_AT_TOP_LEVEL: dict[str, str] = { + "parallel_chunks": ( + f"'parallel_chunks' at {_TOP_LEVEL} applies to every query in every " + "process and spends rate-limit quota. Prefer a [.] " + "table selected per run, or the dataretrieval.parallel_chunks(n) " + "block for a single call." + ), +} + +# Built-in defaults (precedence step 4). ``concurrency`` and ``retries`` keep the +# values the environment-only implementation used, so behavior is unchanged for +# anyone who configures nothing. +DEFAULT_CONCURRENCY = 32 +DEFAULT_RETRIES = 4 +DEFAULT_PARALLEL_CHUNKS = 1 +DEFAULT_STALL_TIMEOUT = 60.0 +CONCURRENCY_UNBOUNDED = "unbounded" + + +# Values that turn the progress line off. Blank counts: ``API_USGS_PROGRESS=`` +# has always meant "off", not "unset" -- unlike the numeric knobs, where blank +# falls through to the default. +_PROGRESS_FALSEY = frozenset({"", "0", "false", "no", "off"}) +_PROGRESS_TRUTHY = frozenset({"1", "true", "yes", "on"}) + +# Settings for which a *blank* environment variable is a value rather than an +# absence. ``API_USGS_PROGRESS=`` has always meant "off". For every other +# setting a blank variable is what container and CI tooling produces when it has +# nothing to pass, so treating it as configured would let it shadow the settings +# file and silently drop the user's API key. +_BLANK_MEANS_SET = frozenset({"progress"}) + +# Warnings about the settings file report the file, not a call site: settings are +# resolved lazily from wherever a getter first needs one, so the user frame is a +# different depth every time and no fixed ``stacklevel`` can name it. +_WARN_STACKLEVEL = 2 + + +# --- provenance ---------------------------------------------------------- +# +# pydantic-settings merges sources but does not report which one supplied a +# given field, and ``show_settings`` exists to answer exactly that. Sources are +# called highest-precedence first (``BaseSettings._settings_build_values`` +# accumulates with ``deep_update(source_state, state)``, so what is already in +# ``state`` wins), which means the *first* source to claim a key is the one that +# won it. Each source below records its keys into the recorder if not already +# present, so first-writer-wins produces the right label with no second merge. +# +# A ContextVar rather than a parameter because the recorder has to reach two +# places pydantic does not thread state through: the sources, which the library +# constructs, and the field validators, which need it to name the source in an +# error message. +_recorder: ContextVar[dict[str, str] | None] = ContextVar( + "dataretrieval_settings_recorder", default=None +) + + +def _record(name: str, label: str) -> None: + """Note that *label* supplied *name*, unless something already claimed it.""" + recorder = _recorder.get() + if recorder is not None: + recorder.setdefault(name, label) + + +def _label_for(name: str, fallback: str) -> str: + """The source label for *name*, for an error message naming its origin.""" + recorder = _recorder.get() + if recorder is None: + return fallback + return recorder.get(name, fallback) + + +def _unwrap(exc: ValidationError) -> ConfigurationError: + """Recover the :class:`ConfigurationError` a field validator raised. + + Every validator in this module raises :class:`ConfigurationError`, which is + a ``ValueError``, so pydantic wraps it in a ``ValidationError`` carrying a + list of errors. The wrapper's rendering names the model and the field in + pydantic's own vocabulary; ours names the *source* -- ``$API_USGS_RETRIES``, + or the file and table -- which is the thing a caller can act on. So the + original is unwrapped and re-raised. + + An error pydantic itself produced (an unknown setting under + ``extra="forbid"``, a type it rejected before any validator ran) has no + original to recover, so its message is translated instead. + """ + for error in exc.errors(): + original = error.get("ctx", {}).get("error") + if isinstance(original, ConfigurationError): + return original + return ConfigurationError(_describe(exc)) + + +def _describe(exc: ValidationError) -> str: + """Render a pydantic-raised error in this package's vocabulary.""" + parts = [] + for error in exc.errors(): + field = ".".join(str(item) for item in error["loc"]) or "settings" + if error["type"] == "extra_forbidden": + parts.append( + f"{field!r} is not a setting {exc.title} accepts. It accepts: " + f"{', '.join(sorted(_fields_of(exc.title)))}." + ) + else: + parts.append(f"{field}: {error['msg']}") + return "; ".join(parts) + + +def _fields_of(title: str) -> frozenset[str]: + """The settings a registered class named *title* accepts, for a message.""" + known: tuple[type[AdapterSettings], ...] = (Settings, *_REGISTRY.values()) + for cls in known: + if cls.__name__ == title: + return cls.settings() + return frozenset(SETTINGS) + + +# --- value grammar ------------------------------------------------------- +# +# One parser drives each setting's grammar, so a value means the same thing and +# reports the same way whichever source wrote it. Each is a plain function so +# the field validators below and the file's eager top-level check share it. +# +# ``typed`` is the one axis on which the sources differ, and it separates them +# into two groups rather than seven. A settings profile's fields and TOML +# scalars are *typed*: ``retries = "2"`` in the file is a quoted integer, which +# is a mistake worth reporting, and ``Settings(retries="2")`` is the same +# mistake in Python. The environment is *untyped* -- it can only deliver strings +# -- so ``API_USGS_RETRIES=2`` must keep working. Passing ``typed=False`` is +# therefore the environment source's privilege alone, and it parses its own +# values before handing them on, which is what lets every tier below it stay +# strict. (Integers are matched as :class:`numbers.Integral` so a numpy or +# pandas integer is a legitimate count from Python; ``tomllib`` only ever +# yields ``int``, so the wider check cannot change a TOML outcome.) + + +def _parse_int( + value: object, + source: str, + *, + default: int, + minimum: int, + examples: str | None = None, + typed: bool = True, +) -> int: + """Parse a bounded integer setting; blank falls through to *default*.""" + if isinstance(value, bool): + raise ConfigurationError(f"{source} must be an integer (got bool).") + expected = f"an integer >= {minimum}" + (f", e.g. {examples}" if examples else "") + if isinstance(value, str): + if typed: + raise ConfigurationError(f"{source} must be {expected} (got str).") + text = value.strip() + if text == "": + return default + try: + parsed = int(text) + except ValueError as exc: + raise ConfigurationError( + f"{source} must be {expected} (got {value!r})." + ) from exc + elif isinstance(value, Integral): + parsed = int(value) + else: + raise ConfigurationError( + f"{source} must be {expected} (got {type(value).__name__})." + ) + if parsed < minimum: + raise ConfigurationError(f"{source} must be {expected} (got {parsed}).") + return parsed + + +def _parse_seconds(value: object, source: str, *, typed: bool = True) -> float: + """Parse a non-negative duration in seconds; blank falls through. + + Seconds rather than a count, so fractional values are accepted. ``0`` + disables the bound it guards, which is why the floor is zero rather than one. + """ + expected = "a finite, non-negative number of seconds" + if isinstance(value, bool): + raise ConfigurationError(f"{source} must be {expected} (got bool).") + if isinstance(value, str): + if typed: + raise ConfigurationError(f"{source} must be {expected} (got str).") + text = value.strip() + if text == "": + return DEFAULT_STALL_TIMEOUT + try: + parsed = float(text) + except ValueError as exc: + raise ConfigurationError( + f"{source} must be {expected} (got {value!r})." + ) from exc + elif isinstance(value, (Integral, float)): + parsed = float(value) + else: + raise ConfigurationError( + f"{source} must be {expected} (got {type(value).__name__})." + ) + # ``inf`` and ``nan`` both parse as floats and both defeat the bound they + # are meant to set: ``inf`` makes every wait allowed, and ``nan`` compares + # false against every threshold. TOML has literal ``inf``/``nan``, so this + # is reachable from the file as well as from Python. + if not math.isfinite(parsed) or parsed < 0: + raise ConfigurationError(f"{source} must be {expected} (got {parsed}).") + return parsed + + +def _parse_concurrency(value: object, source: str, *, typed: bool = True) -> int | str: + """Parse a concurrency cap: a positive int, or ``unbounded``. + + ``"unbounded"`` is a legitimate *string* value for this setting, so unlike + the other numeric dials a string is not automatically a typed-source + mistake -- only a string that is not that word. + """ + if isinstance(value, str) and value.strip().lower() == CONCURRENCY_UNBOUNDED: + return CONCURRENCY_UNBOUNDED + if typed and isinstance(value, str): + raise ConfigurationError( + f"{source} must be an integer or '{CONCURRENCY_UNBOUNDED}'." + ) + try: + return _parse_int( + value, source, default=DEFAULT_CONCURRENCY, minimum=1, typed=typed + ) + except ConfigurationError as exc: + raise ConfigurationError( + f"{exc} Use '{CONCURRENCY_UNBOUNDED}' to disable the cap." + ) from exc + + +def _parse_base_url(value: object, source: str) -> str: + """Parse a service base URL: an absolute ``http``/``https`` origin. + + Only the scheme is checked, and deliberately so. This module cannot know + what a given service's paths look like, but it can refuse the shapes that + are never a base URL and would fail far from here. + """ + if not isinstance(value, str): + raise ConfigurationError( + f"{source} must be a string, or None (got {type(value).__name__})." + ) + text = value.strip() + if not text.startswith(("http://", "https://")): + raise ConfigurationError( + f"{source} must be an absolute http:// or https:// URL (got {value!r})." + ) + return text + + +def _parse_progress(value: object, source: str, *, strict: bool) -> bool: + """Parse a progress toggle, optionally preserving legacy env truthiness.""" + if isinstance(value, bool): + return value + if not isinstance(value, str): + raise ConfigurationError( + f"{source} must be a bool or recognized string, or None " + f"(got {type(value).__name__})." + ) + text = value.strip().lower() + if strict and not text: + raise ConfigurationError(f"{source} must not be blank.") + if text in _PROGRESS_FALSEY: + return False + if text in _PROGRESS_TRUTHY: + return True + if not strict: + # Preserve the legacy environment behavior: any value outside the false + # set enables progress. New block/file values are validated strictly. + return True + expected = ", ".join(sorted(_PROGRESS_TRUTHY | _PROGRESS_FALSEY)) + raise ConfigurationError(f"{source} must be one of {expected} (got {value!r}).") + + +def _parse_api_key(value: object, source: str) -> str: + """Parse an API key: any string. Whitespace is stripped at the read site.""" + if not isinstance(value, str): + raise ConfigurationError( + f"{source} must be a string, or None (got {type(value).__name__})." + ) + return value + + +def _parse_retries(value: object, source: str, *, typed: bool = True) -> int: + """Retries attempted after the first try; ``0`` disables retrying.""" + return _parse_int(value, source, default=DEFAULT_RETRIES, minimum=0, typed=typed) + + +def _parse_parallel_chunks(value: object, source: str, *, typed: bool = True) -> int: + """Baseline fan-out for multi-value queries; at least one chunk.""" + return _parse_int( + value, + source, + default=DEFAULT_PARALLEL_CHUNKS, + minimum=1, + examples="2, 8, 32", + typed=typed, + ) + + +#: Per-setting grammar, named once. The field validators and the file's eager +#: top-level check both go through this table, so a change to a bound cannot +#: leave a ``configure()`` block validating against different rules than the +#: value it later resolves. +_VALIDATORS: dict[str, Callable[[object, str], object]] = { + "api_key": _parse_api_key, + "concurrency": _parse_concurrency, + "retries": _parse_retries, + "progress": lambda value, source: _parse_progress(value, source, strict=True), + "parallel_chunks": _parse_parallel_chunks, + "stall_timeout": _parse_seconds, + "base_url": _parse_base_url, +} + + +def _validate(name: str, value: object, source: str) -> object: + """Run a setting's grammar. ``None`` is never checked: it clears the tiers.""" + if value is None: + return None + return _VALIDATORS[name](value, source) + + +def _require(name: str, value: object, source: str) -> object: + """Run a setting's grammar where ``None`` is not one of the answers. + + The chain reads ``None`` as "suppress the lower sources", which is a + meaningful thing to configure. A caller writing ``parallel_chunks(None)`` is + not saying that -- there is no lower source for a per-call block to suppress + -- so that surface needs the same grammar without the escape hatch. + """ + return _VALIDATORS[name](value, source) + + +#: How the *untyped* source reads each setting it can supply. The environment +#: delivers strings and nothing else, so it parses its own values here and hands +#: on properly typed ones -- which is what lets every tier below stay strict +#: about a quoted integer, and what keeps the two legacy environment grammars +#: (a blank numeric falling through to the default, an unrecognized ``progress`` +#: value meaning "on") contained in the one source that has to honor them. +_ENV_PARSERS: dict[str, Callable[[str, str], object]] = { + "api_key": lambda raw, source: raw, + "concurrency": lambda raw, source: _parse_concurrency(raw, source, typed=False), + "retries": lambda raw, source: _parse_retries(raw, source, typed=False), + "progress": lambda raw, source: _parse_progress(raw, source, strict=False), + "stall_timeout": lambda raw, source: _parse_seconds(raw, source, typed=False), +} + + +# --- settings profiles ---------------------------------------------------- +# +# A setting means the same thing wherever it applies, but it does not apply +# everywhere (ADR 0010). Each adapter declares the settings it accepts as the +# fields of an :class:`AdapterSettings` subclass, defined *in the adapter's own +# module* so a setting's definition sits with the code that reads it. +# +# Two settings are deliberately absent from every adapter: +# +# ``api_key`` belongs to the gateway fronting a host, not to an adapter. +# Water Data and NGWMN are two adapters on one host sharing one +# key and one quota pool -- measured, see ADR 0010. +# ``progress`` describes the caller's terminal, not a service. There is one +# progress line per call, so scoping it per adapter could only +# produce a contradiction. + +#: Bound to the concrete subclass so ``WaterdataSettings.load(...)`` is typed as +#: a ``WaterdataSettings`` rather than the base. ``typing.Self`` would say this +#: in one word and arrives in 3.11; the floor is 3.10. +_S = TypeVar("_S", bound="AdapterSettings") + +# A settings profile has two roles: the payload a caller hands to +# ``configure()``, which must carry *only* what that caller wrote, and the +# resolved view the chain produces. One class serves both -- which is what keeps +# an adapter declaring itself exactly once (ADR 0011) -- and the two are told +# apart by how the instance is built: ``cls(...)`` for a payload (see +# :meth:`AdapterSettings.settings_customise_sources`), :func:`_resolved` for the +# chain. No second class hierarchy, and no flag. + + +class AdapterSettings(BaseSettings): + """A named set of settings for one adapter -- a *settings profile*. + + Subclasses declare the settings their adapter reads as fields, and set + :attr:`adapter` to that adapter's module name. Every field is optional, so + an empty profile is legal and one can be built up conditionally. + + Frozen, because a settings profile is a value: two with the same settings + are interchangeable, and one already handed to :func:`configure` must not + change under the block that entered it. + + Values are checked when the profile is *constructed*, so a typo raises where + it was written rather than at a later ``with`` statement or, worse, inside a + request. + """ + + model_config = SettingsConfigDict( + # A setting an adapter does not read is a typo, and saying so at + # construction is the whole point of a per-adapter vocabulary. + extra="forbid", + frozen=True, + # The chain supplies raw TOML scalars and environment strings; each + # field's validator is what turns them into values, so pydantic must not + # coerce them first and hide a type error behind a silent cast. + strict=False, + validate_default=False, + ) + + #: The adapter this profile targets, by the name of the module a caller + #: imports. ``None`` on the package-wide :class:`Settings`, which every + #: adapter reads. A ``ClassVar``, not a field: the adapter is a property of + #: the class, which is what stops the caller restating it at every call site + #: and stops the roster being spelled twice. + adapter: ClassVar[str | None] = None + + #: The named profile these settings were read from, or ``None`` for one + #: written in code. Provenance rather than a setting: it records *where the + #: values came from*, which is what lets :func:`show_settings` name the + #: profile that supplied each value instead of reporting every block alike. + #: + #: A ``ClassVar`` shadowed per instance by :meth:`load`, so it is neither a + #: field nor part of equality -- two profiles carrying the same settings stay + #: interchangeable however each was spelled. + profile: ClassVar[str | None] = None + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Constructing a profile directly reads nothing ambient. + + ``WaterdataSettings(concurrency=8)`` written in a script is a *payload* + describing what that caller asked for, and folding the environment into + it would make it describe something else -- ``values()`` would report + settings the caller never wrote, and :func:`configure` would then push + them into the block as though they had. So the caller's own keywords are + the only source here. + + The stock ``env_settings``, ``dotenv_settings`` and + ``file_secret_settings`` are dropped for the same reason, and because + this package reads a specific set of ``API_USGS_*`` variables and one + TOML grammar rather than pydantic-settings' generic conventions. + + The *resolution* chain is :data:`_CHAIN`, applied by :func:`_resolved`. + """ + return (init_settings,) + + @field_validator("*", mode="before") + @classmethod + def _check(cls, value: object, info: Any) -> object: + """Run one setting's grammar, naming the source that supplied it. + + Every field goes through the same table, so the file, the environment + and a ``configure()`` block cannot come to disagree about what a value + means. The label is the source recorded for this field during + resolution, or the constructor that is running now. + """ + name = info.field_name + return _validate(name, value, _label_for(name, f"{name}= in {cls.__name__}()")) + + def __init__(self, **values: Any) -> None: + """Validate the caller's keywords, and nothing ambient. + + Deliberately *not* ``BaseSettings.__init__``. That entry point exists to + build a settings object once at start-up, and it constructs the four + stock sources -- two of which snapshot and case-fold the whole of + ``os.environ`` -- before :meth:`settings_customise_sources` can discard + them. This package resolves lazily, several times per query, so it pays + that cost per read rather than per process: measured at ~330 us against + the ~5 us the rest of a resolution takes. + + Both of this class's roles want it skipped. A profile written in code + must read nothing ambient by definition, and :func:`_resolved` supplies + the chain's merged mapping itself. So this is ``BaseModel.__init__`` -- + validate the given keywords into ``self`` -- plus the unwrap that keeps + a bad value reported in this package's vocabulary rather than pydantic's. + """ + try: + self.__pydantic_validator__.validate_python(values, self_instance=self) + except ValidationError as exc: + raise _unwrap(exc) from None + + def model_post_init(self, context: Any, /) -> None: + self.validate_settings() + + def validate_settings(self) -> None: + """Check rules that span more than one setting. + + Does nothing by default. Per-setting grammar lives in this module's + parsers and is shared with the file and the environment, so a value + means the same thing whichever source wrote it; override this only for a + rule no single setting can express. + """ + + @classmethod + def settings(cls) -> frozenset[str]: + """The setting names this profile accepts.""" + return frozenset(cls.model_fields) + + def values(self) -> dict[str, Any]: + """The settings actually supplied, omitting those left unset. + + An omitted setting inherits from an outer block or a lower source; an + explicit ``None`` suppresses them. ``model_fields_set`` is pydantic's + record of that distinction, which is why no sentinel default is needed. + """ + return {name: getattr(self, name) for name in sorted(self.model_fields_set)} + + @classmethod + def load(cls: type[_S], profile: str) -> _S: + """Read a named profile for this adapter from the settings file. + + ``[.]``. Only the keys that table names are carried, so + the profile still inherits the adapter's default profile and the + package-wide keys per setting from the tiers below. + + Selecting a profile the file does not define raises: a name a caller just + typed is a typo worth reporting, not a silent fall-through to settings + they did not ask for. + + Parameters + ---------- + profile : str + The name after the adapter, so ``[waterdata.bulk]`` is ``"bulk"``. + + Returns + ------- + AdapterSettings + An instance of the class it was called on, remembering the profile + it was read from so :func:`show_settings` can name it. + """ + adapter = cls.adapter + if adapter is None: + raise ConfigurationError( + f"{cls.__name__}.load() names a profile for one adapter, and " + "the package-wide settings have none. Put shared keys at the " + "top level of the file." + ) + loaded = cls(**_named_profile(adapter, profile, cls.settings())) + # The model is frozen, so the provenance goes on the way pydantic sets + # its own attributes. It is deliberately not a field: the profile name is + # where these values came from, not one of the values, and + # :meth:`settings` is built from the fields. + object.__setattr__(loaded, "profile", profile) + return loaded + + def _source(self, name: str) -> str: + """How one of this profile's settings is named in an error.""" + return f"{name}= in {type(self).__name__}()" + + def _provenance(self) -> str: + """How :func:`show_settings` reports a value this supplied. + + The profile is named in the file's own spelling -- ``[waterdata.bulk]`` + -- so the report answers "which profile set this?" rather than only "a + block did", and the answer is greppable in the file that holds it. + """ + if self.adapter is None: + return "configure() block" + scope = self.adapter + if self.profile is not None: + scope = f"{scope}.{self.profile}" + return f"configure() block [{scope}]" + + +# --- shared setting groups ----------------------------------------------- +# +# Which settings an adapter accepts is the adapter's own knowledge, and it says +# so by naming the groups below. What a setting *is* -- its type, the fact that +# ``None`` suppresses the tiers under it -- is this module's. Each group declares +# one shared setting once, and an adapter composes the groups it reads:: +# +# class NgwmnSettings(_Chunked, _Concurrent, _Redirectable, _Retrying, +# AdapterSettings): +# adapter: ClassVar[str] = "ngwmn" +# +# Widening a shared setting's accepted type, or adding one, is one edit rather +# than six. Unlike the dataclass version these annotations are *enforced*: +# pydantic builds its validator from them, so an adapter that drifted to +# ``retries: str | None`` would reject an integer at construction rather than +# type-checking clean and failing when a value reached the chain. +# +# Plain mixins rather than ``AdapterSettings`` subclasses: a group is not a +# settings profile -- it has no adapter and cannot be passed to +# :func:`configure`. Fields are collected in reverse MRO order, so an adapter +# composing all four reads ``retries, stall_timeout, base_url, concurrency, +# parallel_chunks``. + + +class _Retrying(BaseSettings): + """Every adapter's retry dials: transient retries and the stall bound.""" + + retries: int | None = None + stall_timeout: float | int | None = None + + +class _Redirectable(BaseSettings): + """An adapter whose requests can be pointed at another base URL.""" + + base_url: str | None = None + + +class _Concurrent(BaseSettings): + """An adapter that issues more than one request per call.""" + + concurrency: int | Literal["unbounded"] | None = None + + +class _Chunked(BaseSettings): + """An adapter whose queries divide into sub-requests the caller can fan.""" + + parallel_chunks: int | None = None + + +class Settings(AdapterSettings): + """Settings that apply to every adapter. + + The package-wide profile: ``adapter`` stays ``None``, so nothing narrows and + every adapter reads what this sets unless its own profile, or a block nested + inside, overrides that setting. + + Parameters + ---------- + api_key : str, optional + Water Data API key, sent as ``X-Api-Key`` and only ever to + ``api.waterdata.usgs.gov``. Prefer reading it from a secret store, the + environment, or the settings file over writing a literal into a script. + Pass ``None`` to make a call without an ambient key. + concurrency : int or str, optional + Cap on simultaneous sub-requests: a positive integer, or ``"unbounded"`` + to disable the cap. + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + progress : bool or str, optional + Whether to draw the progress line. ``None`` leaves the automatic + behavior (on for a TTY or Jupyter kernel, off otherwise). + parallel_chunks : int, optional + Default optional fan-out for multi-value queries. It limits extra + refinement, but URL-byte safety may already require more sub-requests. + Sets the baseline that :func:`dataretrieval.parallel_chunks` overrides + per call. Each sub-request spends rate-limit quota, so raise it only for + pulls you know are large. + stall_timeout : float, optional + Seconds a call may go without receiving *any* data before retrying stops + and the failure surfaces. Bounds the wall-clock cost of a dead + connection, which ``retries`` does not -- it counts attempts, not + seconds. Progress resets the clock; ``0`` disables the bound. + + Examples + -------- + .. code-block:: python + + with dataretrieval.configure(Settings(api_key=vault.read("usgs"))): + df, md = waterdata.get_daily(monitoring_location_id="USGS-05114000") + """ + + # Spelled out rather than composed from the groups above, because this order + # is also the order :func:`show_settings` reports the settings in -- + # :data:`SETTINGS` is derived from it just below -- and composing would hand + # that reader-facing sequence to MRO linearization. The two adapter-only + # fields the groups carry are absent by construction here: there is no + # package-wide base URL. + api_key: str | None = None + concurrency: int | Literal["unbounded"] | None = None + retries: int | None = None + progress: bool | str | None = None + parallel_chunks: int | None = None + stall_timeout: float | int | None = None + + +#: The package-wide settings, in the order :func:`show_settings` reports them -- +#: the fields of :class:`Settings`, derived rather than restated. An adapter may +#: accept a subset of them plus :data:`ADAPTER_ONLY_SETTINGS`. +SETTINGS: tuple[str, ...] = tuple(Settings.model_fields) + +#: Every setting name this module knows a grammar for. +_ALL_SETTINGS: tuple[str, ...] = SETTINGS + ADAPTER_ONLY_SETTINGS + + +#: The adapters that may be configured, by the name of the module a caller +#: imports. Names only, because this module cannot import an adapter without +#: cycling: every adapter imports it. +#: +#: Holding the names here rather than deriving them from the registry below is +#: what lets a ``[nldi]`` table stay valid in a file: NLDI is imported on demand +#: for the geopandas extra, so a roster built from imports would reject a +#: perfectly good table until something happened to import that module. +ADAPTERS: tuple[str, ...] = ( + "waterdata", + "ngwmn", + "nwdc", + "wqp", + "nldi", + "streamstats", +) + +#: Settings classes that have registered themselves, keyed by adapter. Populated +#: at adapter import, and consulted only to validate a table's *keys* -- which +#: happens the first time that adapter resolves a setting, by which point it is +#: necessarily imported. +_REGISTRY: dict[str, type[AdapterSettings]] = {} + + +def _register(cls: type[AdapterSettings]) -> None: + """Record an adapter's settings class. Called at adapter import. + + The roster in :data:`ADAPTERS` and the class are the two halves of one + declaration, and this is where they are checked to agree: a class naming an + adapter the roster does not list would be a profile no file table and no + report could ever reach. + """ + adapter = cls.adapter + if adapter is None or adapter not in ADAPTERS: + raise ConfigurationError( + f"{cls.__name__}.adapter is {adapter!r}, which is not one of " + f"{', '.join(ADAPTERS)}." + ) + _REGISTRY[adapter] = cls + + +def settings_for(adapter: str) -> frozenset[str] | None: + """The settings *adapter* accepts, or ``None`` if it has not been imported. + + ``None`` is not an error and callers must not treat it as one: a file may + name an adapter this process has never loaded, and rejecting that would make + a settings file conditionally valid depending on which optional extras + happened to be installed. It means "cannot validate these keys yet". + """ + cls = _REGISTRY.get(adapter) + return None if cls is None else cls.settings() + + +# --- the configure() block ------------------------------------------------ + +# Overrides from the active ``configure`` blocks. A package-wide override is +# keyed by the setting's name; an adapter-scoped one by ``(adapter, name)``. One +# flat mapping rather than a nested one so that nesting, per-key inheritance and +# restore-on-exit keep falling out of a single lookup. +_ScopeKey = str | tuple[str, str] +# One frame per ``configure`` block, stacked outermost-first. Frames rather than +# a merged mapping are what makes "the innermost block wins" true across *both* +# scopes: an adapter-scoped value outranks a package-wide one only within the +# same frame. Merged, an outer ``configure(WaterdataSettings(...))`` would beat +# an inner ``configure(Settings(concurrency=1))`` -- inverting nesting, and +# silently discarding the per-call ``parallel_chunks(n)`` block. +# +# Each entry pairs the value with the label naming where it came from, built +# while the profile is still in hand because that is the only place the +# *profile* is known: a value from ``WaterdataSettings.load("bulk")`` and one +# from ``WaterdataSettings(...)`` are indistinguishable by the time they reach +# the frame. +_Frame = Mapping[_ScopeKey, tuple[Any, str]] +_scope: ContextVar[tuple[_Frame, ...]] = ContextVar( + "dataretrieval_settings_scope", default=() +) + + +@contextmanager +def configure(*profiles: AdapterSettings) -> Iterator[None]: + """Apply settings profiles for the duration of a ``with`` block. + + The highest-precedence source. Takes settings profiles positionally, at most + one per adapter, and nothing else:: + + with dataretrieval.configure( + Settings(api_key=secrets["usgs"]), + WaterdataSettings.load("bulk"), + NgwmnSettings(concurrency=4), + ): + df, md = waterdata.get_daily(monitoring_location_id=sites) + + The adapter a profile targets is a property of its class, so the caller never + restates it -- which is what keeps the adapter roster from being spelled once + per call site. Naming two profiles for one adapter raises: they are the one + pairing with no defined order between them. + + Because the block is delivered through a :class:`~contextvars.ContextVar`, a + value set here applies to the current thread and to asyncio tasks started + inside the block, and cannot leak into another thread, task, or unrelated + call the way ``os.environ`` does -- which is what makes it safe for a server + or notebook handling several users' credentials at once. + + Blocks nest and merge per setting: an inner block that sets only + ``concurrency`` keeps the outer block's ``api_key``, and an adapter profile + in an outer block loses to a package-wide value set by a block nested inside + it, so the innermost block always decides. + + Parameters + ---------- + *profiles : AdapterSettings + A package-wide :class:`Settings` and/or one profile per adapter, in any + order. Each adapter's class lives in that adapter's module -- + ``WaterdataSettings`` in :mod:`dataretrieval.waterdata`, + ``NgwmnSettings`` in :mod:`dataretrieval.ngwmn`, and so on. + + Yields + ------ + None + + Raises + ------ + ConfigurationError + If an argument is not a settings profile, or two of them target the same + adapter. Raised on entry, before any request. A bad *value* raises + earlier still, where the profile was constructed. + + See Also + -------- + show_settings : Report the effective settings and where they came from. + """ + token = _scope.set((*_scope.get(), _frame(profiles))) + try: + yield + finally: + _scope.reset(token) + + +def _frame(profiles: tuple[AdapterSettings, ...]) -> _Frame: + """Flatten one ``configure`` call's profiles into a scope frame. + + One frame per block, holding both scopes: a package-wide setting keyed by + its name, an adapter-scoped one by ``(adapter, name)``. Values were already + checked when each profile was constructed -- which is where a typo should + raise, at the line that wrote it rather than at a later ``with`` statement -- + so nothing new can fail here except the two call-shaped mistakes below. + + Each value is stored with the label naming the profile it came from, because + this is the last point where that is known (see :data:`_Frame`). + """ + overrides: dict[_ScopeKey, tuple[Any, str]] = {} + seen: set[str | None] = set() + for profile in profiles: + if not isinstance(profile, AdapterSettings): + raise ConfigurationError( + "configure() takes settings profiles, not " + f"{type(profile).__name__}. Package-wide settings go on " + "Settings(...); a setting for one service goes on that " + "adapter's profile, e.g. WaterdataSettings(...)." + ) + adapter = profile.adapter + if adapter in seen: + where = f"the {adapter} adapter" if adapter else "the package-wide settings" + raise ConfigurationError( + f"configure() got two settings profiles for {where}. Precedence " + "between them would be undefined, so combine them into one." + ) + seen.add(adapter) + label = profile._provenance() + for name, value in profile.values().items(): + key: _ScopeKey = name if adapter is None else (adapter, name) + overrides[key] = (value, label) + return overrides + + +# --- settings sources ----------------------------------------------------- +# +# One :class:`~pydantic_settings.PydanticBaseSettingsSource` per tier of the +# chain. Each returns the settings it can supply for the class being resolved +# and records the label that :func:`show_settings` will report; pydantic-settings +# does the merge, keeping the first value it sees for each key. + + +class _ChainSource(PydanticBaseSettingsSource): + """Shared plumbing for this package's four tiers.""" + + def __init__(self, settings_cls: type[BaseSettings], adapter: str) -> None: + super().__init__(settings_cls) + # ``""`` is how :func:`_resolved` spells "package-wide", because the + # ContextVar's ``None`` already means "not resolving at all". + self.adapter: str | None = adapter or None + + def get_field_value( + self, field: FieldInfo, field_name: str + ) -> tuple[Any, str, bool]: + # Required by the ABC, but this package's sources answer for every field + # at once in ``__call__`` -- a per-field walk would re-read the + # environment and re-stat the file once per setting. + raise NotImplementedError # pragma: no cover + + def _names(self) -> tuple[str, ...]: + return tuple(self.settings_cls.model_fields) + + +class _BlockSource(_ChainSource): + """Tier 1: the innermost active ``configure()`` block.""" + + def __call__(self) -> dict[str, Any]: + values: dict[str, Any] = {} + frames = _scope.get() + if not frames: + return values + for name in self._names(): + # Innermost block first: a value set by a nested block wins over both + # scopes of an enclosing one. Within one block the adapter-scoped + # value is the more specific of the two, so it is asked first. + for frame in reversed(frames): + if self.adapter is not None and (self.adapter, name) in frame: + value, label = frame[(self.adapter, name)] + elif name in frame: + value, label = frame[name] + else: + continue + values[name] = value + _record(name, label) + break + return values + + +class _EnvSource(_ChainSource): + """Tier 2: the setting's ``API_USGS_*`` variable. + + Package-wide by construction: seven adapters times four settings would be a + namespace nobody can hold in mind, and an exported variable is inherited by + every subprocess and invisible at the call site (ADR 0010). + """ + + def __call__(self) -> dict[str, Any]: + values: dict[str, Any] = {} + for name in self._names(): + variable = ENV_VARS.get(name) + if variable is None: + continue + raw = os.environ.get(variable) + if raw is None: + continue + # A blank variable is what container and CI tooling produces when it + # has nothing to pass, so it does not count as configured -- except + # for ``progress``, where blank has always meant "off". + if not raw.strip() and name not in _BLANK_MEANS_SET: + continue + label = _env_source_label(variable) + # Parsed here rather than by the field validator, because this is the + # only point at which the value is known to have come from the + # environment: by the time pydantic validates the merged mapping, + # which tier supplied a value is no longer visible. That matters for + # the two grammars this tier alone is lenient about -- a blank + # numeric falls through to the default, and any unrecognized + # ``progress`` value means "on". + values[name] = _ENV_PARSERS[name](raw, label) + _record(name, label) + return values + + +class _AdapterTableSource(_ChainSource): + """Tier 3: the ``[]`` table -- that adapter's default profile.""" + + def __call__(self) -> dict[str, Any]: + if self.adapter is None: + return {} + path, parsed = _current_file() + table = _adapter_file_settings(self.adapter, path, parsed) + values: dict[str, Any] = {} + for name in self._names(): + if name in table: + value, label = table[name] + values[name] = value + _record(name, label) + return values + + +class _TopLevelSource(_ChainSource): + """Tier 4: the file's top-level keys -- the package-wide defaults.""" + + def __call__(self) -> dict[str, Any]: + path, parsed = _current_file() + if not parsed.base: + return {} + values: dict[str, Any] = {} + label = str(path) + for name in self._names(): + if name in parsed.base: + values[name] = parsed.base[name] + _record(name, label) + return values + + +def _env_source_label(variable: str) -> str: + """How a value read from *variable* is reported as a source.""" + return f"${variable}" + + +# --- resolution ---------------------------------------------------------- + + +#: The resolution chain, highest precedence first. Declared as data so the +#: ordering is one fact in one place: ADR 0009's "precedence is per setting, not +#: per source" is expressed by walking these in order and keeping the first +#: value seen for each key, rather than by hand-written fallbacks. +_CHAIN: tuple[type[_ChainSource], ...] = ( + _BlockSource, + _EnvSource, + _AdapterTableSource, + _TopLevelSource, +) + + +def _resolved(adapter: str | None) -> tuple[AdapterSettings, dict[str, str]]: + """Run the chain for one adapter, returning the values and their sources. + + Building the whole profile rather than one setting matches the blast radius + the file tier already had: resolution validates an adapter's entire table on + first use, so a bad ``concurrency`` in ``[waterdata]`` has always been able + to fail a read of ``retries`` for Water Data -- and must not touch NLDI. + + The sources are walked here and the result handed to ``model_validate``, + rather than going through ``BaseSettings()`` and letting the library drive + them. That is a deliberate departure, and it is about cost, not shape -- + the sources and their order are still pydantic-settings'. ``BaseSettings`` + builds the four *stock* sources on every instantiation before + :meth:`settings_customise_sources` gets to discard them, and two of those + snapshot and case-fold the whole of ``os.environ``. Settings resolve on the + request path here, several times per query, so that is not affordable: + profiling one read put 74% of it in ``_settings_init_sources``, at ~330 us + against the ~5 us the whole resolution costs this way. ``model_validate`` + runs the same field validators, ``extra="forbid"`` and + ``model_post_init`` hook, so nothing about the schema half changes. + """ + cls: type[AdapterSettings] = ( + Settings if adapter is None else _REGISTRY.get(adapter, Settings) + ) + labels: dict[str, str] = {} + recording = _recorder.set(labels) + try: + merged: dict[str, Any] = {} + for source in _CHAIN: + for name, value in source(cls, adapter or "")().items(): + # First writer wins, which is what makes the tuple above a + # precedence order. ``_record`` follows the same rule, so the + # label and the value always come from the same tier. + merged.setdefault(name, value) + # ``cls(**merged)`` rather than ``model_validate``: the custom + # ``__init__`` above is what pydantic dispatches to either way, so + # calling it directly saves a round trip through the validator. + return cls(**merged), labels + finally: + _recorder.reset(recording) + + +def _resolve(name: str, adapter: str | None = None) -> tuple[Any, str]: + """Return the value for *name* and a human-readable source label. + + Precedence is *source-major*: the chain walks block, then environment, then + file, exactly as ADR 0009 defines it -- and *within* each source an + adapter-scoped value outranks a package-wide one. So a variable exported for + one run still beats a stale ``[wqp]`` table in the settings file, which + scope-major ordering would have quietly inverted (ADR 0010). + + Returns + ------- + tuple[Any, str] + The value, and where it came from -- ``None`` with :data:`_BUILT_IN` + when nothing configured it. An explicit ``None`` from a block reads the + same way at the accessors, which is what makes it a scoped reset to + built-in behavior. + """ + # An adapter name nobody recognizes is a typo in *our* source, and its + # failure mode is silence: the file would hold no table under that name and + # the read would fall through to the package-wide value, so a + # ``WaterdataSettings`` would be ignored with nothing raised anywhere. + if adapter is not None and adapter not in ADAPTERS: + raise ConfigurationError( + f"{adapter!r} is not a configurable adapter. The adapters are " + f"{', '.join(ADAPTERS)}." + ) + + # Refused before anything is consulted, not at the environment's turn in the + # chain. The file refuses ``base_url`` whether or not a block also set one, + # and the two surfaces are one rule, so a variable that cannot work must not + # be silently outranked by a block that happens to work. + refused = _REFUSED_ENV_VARS.get(name) + if refused is not None and refused in os.environ: + raise ConfigurationError( + f"{_env_source_label(refused)} is set, but {name!r} may only be set " + "in code, in a configure() block, never from the environment. Unset " + "it and pass the value on the adapter's settings, e.g. " + f"WaterdataSettings({name}=...)." + ) + + # ``None`` unless this adapter actually reads this setting, so a setting + # outside its vocabulary resolves package-wide rather than looking for a + # scope it could never have been written into. + scoped = adapter if adapter is not None and _accepts(adapter, name) else None + instance, labels = _resolved(scoped) + if name not in type(instance).model_fields: + # The package-wide profile has no ``base_url``: there is no one base URL, + # so nothing could have set it and the service's own default stands. + return None, _BUILT_IN + if name not in labels: + return None, _BUILT_IN + return getattr(instance, name), labels[name] + + +def _accepts(adapter: str, name: str) -> bool: + """Whether *adapter* reads the setting *name*. + + An adapter this process has not imported has no vocabulary to consult, so + every setting is assumed to be in scope for it: the file stays valid either + way, and an adapter cannot be misreading a setting it has not loaded. + """ + accepted = settings_for(adapter) + return name in _ALL_SETTINGS if accepted is None else name in accepted + + +def _source_label(name: str, adapter: str | None = None) -> str: + """The provenance label for one setting, for :func:`show_settings`.""" + return _resolve(name, adapter)[1] + + +# --- resolved settings --------------------------------------------------- + + +def api_key() -> str | None: + """The Water Data API key, or ``None`` if none is configured. + + Surrounding whitespace is stripped, so a key read from a file with a + trailing newline works; a blank value resolves to ``None``. + """ + value, _source = _resolve("api_key") + return value.strip() or None if value is not None else None + + +def concurrency( + default: int | None = DEFAULT_CONCURRENCY, *, adapter: str | None = None +) -> int | None: + """Cap on simultaneous chunks; ``None`` means unbounded. + + ``default`` is the caller's own preference for when nothing is configured -- + NWDC ships a lower figure than the OGC getters, because it is only + stress-tested to that level. A value resolved from the chain always wins over + it: a service able to override an explicit setting would make + ``concurrency=1`` a lie. + """ + value, _source = _resolve("concurrency", adapter) + if value is None: + return default + return None if value == CONCURRENCY_UNBOUNDED else int(value) + + +def retries(*, adapter: str | None = None) -> int: + """Retries attempted after the first try; ``0`` disables retrying.""" + value, _source = _resolve("retries", adapter) + return DEFAULT_RETRIES if value is None else int(value) + + +def progress() -> bool | None: + """Explicit progress-line setting, or ``None`` to auto-detect. + + ``None`` means nothing configured it, so the caller applies its own default + (a TTY or Jupyter kernel gets the line, redirected output doesn't). + """ + value, _source = _resolve("progress") + return None if value is None else bool(value) + + +def parallel_chunks(*, adapter: str | None = None) -> int: + """Configured default fan-out for multi-value queries. + + ``1`` (the default) means "chunk only as much as the URL byte limit forces". + This is the *baseline*; :func:`dataretrieval.parallel_chunks` overrides it + for one call. + """ + value, _source = _resolve("parallel_chunks", adapter) + return DEFAULT_PARALLEL_CHUNKS if value is None else int(value) + + +def stall_timeout(*, adapter: str | None = None) -> float: + """Longest a call may go without receiving data before retrying stops. + + Seconds; ``0`` disables the bound. Bounds the wall-clock cost of a dead + connection, which the retry *count* does not: it counts attempts, not + seconds. + """ + value, _source = _resolve("stall_timeout", adapter) + return DEFAULT_STALL_TIMEOUT if value is None else float(value) + + +@overload +def base_url(*, adapter: str | None = ...) -> str | None: ... + + +@overload +def base_url(*, adapter: str | None = ..., default: str) -> str: ... + + +def base_url(*, adapter: str | None = None, default: str | None = None) -> str | None: + """An adapter's configured base URL, falling back to *default*. + + Settable from code only: an adapter's settings may carry it, and both the + file and the environment refuse it -- the file at :func:`_accepted_keys` and + the environment at :data:`_REFUSED_ENV_VARS`, each with an error naming the + block to write instead. A file that silently redirects a data-retrieval + library to another host is a supply-chain-shaped hazard, while a + ``configure`` block keeps the redirect where a reader of the script sees it + (ADR 0011). + + There is no package-wide default, because there is no one base URL: what an + adapter's requests are built on is the adapter's own fact, so the service + passes its own and the URL stays declared beside the service that owns it. + + Parameters + ---------- + adapter : str, optional + Whose base URL to resolve. + default : str, optional + The service's own base, returned when nothing configured one. + """ + value, _source = _resolve("base_url", adapter) + return default if value is None else str(value) + + +# --- the settings file --------------------------------------------------- +# +# pydantic-settings ships a ``TomlConfigSettingsSource``, and it is deliberately +# not used: it reads a whole file into a flat mapping on every instantiation, +# with no notion of adapter tables or named profiles, no cache, and no +# permission check. Settings resolve on the request path here, so re-reading and +# re-parsing per read is not affordable, and the grammar below is most of what +# the file layer *is*. What the library does own -- the merge and the validation +# -- is what the sources above delegate to it. + + +@dataclass(frozen=True) +class _ParsedFile: + """A parsed settings file: package-wide keys plus per-adapter tables. + + ``exists`` distinguishes "the file is there and defines nothing" from "there + is no file", which decides which of the two messages a caller selecting a + profile gets (see :func:`_named_profile`). + """ + + base: dict[str, Any] = field(default_factory=dict) + #: Raw, *unvalidated* ``[]`` tables, keyed by adapter name. Each + #: holds that adapter's default-profile keys and, as sub-tables, its named + #: profiles. Left unvalidated because a bad value in ``[nldi]`` must not fail + #: a Water Data call that never reads it. + adapters: dict[str, dict[str, Any]] = field(default_factory=dict) + exists: bool = False + + +#: Stand-in for "no settings file", which is the common case. Shared rather than +#: rebuilt per read so that callers can memoize on the parsed file's identity; +#: nothing mutates a ``_ParsedFile``. +_NO_FILE = _ParsedFile() + +# Resolved settings-file path, memoized on the raw ``DATARETRIEVAL_CONFIG`` +# value (see :func:`config_path`). +_path_cache: tuple[str | None, object | None, Path] | None = None + +# Parsed settings file, keyed by file identity, change metadata, and raw +# content. POSIX ctime makes metadata hits reliable; Windows ctime is creation +# time, so cache hits there compare content before reusing the parsed result. +_FileStamp = tuple[int, int, int, int, int, int] +_file_cache: tuple[Path, _FileStamp, bytes, _ParsedFile] | None = None + +# Validated ``[]`` tables, keyed by adapter name and memoized on the +# parsed file's identity, because an adapter table is validated only once that +# adapter is actually used. +_adapter_cache: dict[str, tuple[_ParsedFile, Path, Mapping[str, tuple[Any, str]]]] = {} + +# Paths already warned about for loose permissions, so the warning fires once. +_permission_warned: set[Path] = set() + + +def config_path() -> Path: + """Path to the settings file, honoring ``DATARETRIEVAL_CONFIG``. + + Memoized on the raw ``DATARETRIEVAL_CONFIG`` value, because this sits on the + per-request path via :func:`api_key` and building the default costs more than + the ``stat`` it leads to (``Path.home()`` alone dominates the whole + resolution). + + Returns + ------- + pathlib.Path + The explicit path from ``DATARETRIEVAL_CONFIG`` if set, otherwise + ``~/.dataretrieval/config.toml``. The file need not exist. + """ + global _path_cache + override = os.environ.get(CONFIG_PATH_ENV) + + cached = _path_cache + if cached is not None and cached[0] == override: + cached_guard, path = cached[1], cached[2] + # The memo is only valid while whatever the path was *derived from* is + # unchanged, so each branch records its own guard. A relative override is + # anchored to the working directory (a later ``os.chdir`` in a per-job + # notebook or scheduler must not keep reading the previous job's file); + # the default branch is anchored to ``$HOME``. An absolute override + # depends on neither and guards with ``None``. + if cached_guard is None or cached_guard == _path_guard(cached_guard): + return path + + expanded = ( + Path(override.strip()).expanduser() if override and override.strip() else None + ) + guard: object | None + if expanded is None: + path = _default_home_path() + guard = _home_id() + elif expanded.is_absolute(): + path = expanded + guard = None + else: + guard = _cwd_id() + path = _resolve_against_cwd(expanded) + _path_cache = (override, guard, path) + return path + + +def _default_home_path() -> Path: + """The default ``~/.dataretrieval/config.toml``, or an unusable path. + + ``Path.home()`` raises ``RuntimeError`` where no home can be resolved at all + -- a rootless container running as an arbitrary UID with no passwd entry and + no ``HOME``. That is not a misconfiguration to report: such a deployment + simply has no settings file, and before settings were layered it worked fine + on the environment alone. + """ + try: + home = Path.home() + except (RuntimeError, OSError): + return Path("~") / ".dataretrieval" / "config.toml" + return home / ".dataretrieval" / "config.toml" + + +def _resolve_against_cwd(relative: Path) -> Path: + """Resolve a relative override, or report a working directory that is gone.""" + try: + return Path.cwd() / relative + except OSError as exc: + raise ConfigurationError( + f"cannot resolve the relative {CONFIG_PATH_ENV} path {str(relative)!r}: " + f"the working directory is unavailable ({exc})." + ) from exc + + +def _path_guard(previous: object) -> object: + """Re-read whichever guard the cached entry was built with.""" + return _cwd_id() if isinstance(previous, tuple) else _home_id() + + +def _cwd_id() -> tuple[int, int]: + """Identify the working directory without building its path string. + + Only identifies the directory; :func:`_resolve_against_cwd` is what turns a + missing cwd into a :class:`ConfigurationError`. Both are needed, because + ``stat`` on a *deleted* working directory still succeeds -- the process holds + the open handle -- while resolving its path does not. + """ + try: + st = os.stat(".") + except OSError as exc: + raise ConfigurationError( + f"cannot resolve the relative {CONFIG_PATH_ENV} path: the working " + f"directory is unavailable ({exc})." + ) from exc + return (st.st_dev, st.st_ino) + + +def _home_id() -> str: + """The home directory as the environment reports it. + + Which variable that is differs by platform, and the memo has to agree with + the resolver or it watches the wrong thing. ``posixpath.expanduser`` reads + ``HOME``; ``ntpath.expanduser`` reads ``USERPROFILE`` (then + ``HOMEDRIVE``/``HOMEPATH``) and ignores ``HOME`` outright. + """ + if os.name == "nt": + return ( + os.environ.get("USERPROFILE") + or os.environ.get("HOMEDRIVE", "") + os.environ.get("HOMEPATH", "") + or "" + ) + return os.environ.get("HOME") or "" + + +def _toml_parser() -> Any: + """The TOML parser, imported on first use. + + ``import dataretrieval`` imports this module, but the parser is reachable + only once a settings file actually exists -- the minority case. + """ + if sys.version_info >= (3, 11): + import tomllib + else: # pragma: no cover - exercised only on Python 3.10 + import tomli as tomllib + return tomllib + + +def _current_file() -> tuple[Path, _ParsedFile]: + """The settings file as currently loaded: its path and its parsed form.""" + path = config_path() + return path, _load_file(path) + + +def _load_file(path: Path) -> _ParsedFile: + """Parse the settings file at *path*, caching until it changes on disk.""" + global _file_cache + try: + st = path.stat() + except FileNotFoundError: + # No file is the normal case: continue to the built-in default. + return _NO_FILE + except OSError as exc: + raise ConfigurationError(f"could not access {path}: {exc}") from exc + + if stat.S_ISDIR(st.st_mode): + raise ConfigurationError(f"settings path {path} is a directory, not a file.") + + # Only a regular file is parsed. Anything else readable -- a character + # device, a FIFO -- is treated as *empty* settings without being opened, + # which is what ``DATARETRIEVAL_CONFIG=/dev/null`` asks for and the only + # coherent answer for a stream: settings are re-resolved on every request, so + # a FIFO would hand its contents to the first getter and nothing to the rest. + if not stat.S_ISREG(st.st_mode): + return _ParsedFile(exists=True) + + # POSIX ``st_ctime_ns`` advances on any inode change, so the metadata stamp + # catches even a rewrite that restores the original mtime (``cp -p``, rsync + # ``--times``, an editor that preserves timestamps). Windows ctime is + # *creation* time, so there the stamp cannot see that class of edit and the + # content compare below is the only correct check -- worth the re-read, since + # serving a stale API key is the alternative. + # + # Please do not "optimize" this without a Windows-safe change detector; + # ``test_file_edit_is_picked_up`` pins the behavior. + cached = _file_cache + if ( + os.name != "nt" + and cached is not None + and cached[0] is path + and cached[1] == _file_stamp(st) + ): + return cached[3] + + try: + with path.open("rb") as handle: + content = handle.read() + opened_st = os.fstat(handle.fileno()) + except OSError as exc: + raise ConfigurationError(f"could not read {path}: {exc}") from exc + + if cached is not None and cached[0] is path and cached[2] == content: + parsed = cached[3] + else: + tomllib = _toml_parser() + try: + data = tomllib.loads(content.decode("utf-8")) + except UnicodeDecodeError as exc: + raise ConfigurationError(f"{path} is not valid UTF-8: {exc}") from exc + except tomllib.TOMLDecodeError as exc: + raise ConfigurationError(f"{path} is not valid TOML: {exc}") from exc + parsed = _interpret(data, path) + _warn_on_loose_permissions(path, opened_st, parsed) + _file_cache = (path, _file_stamp(opened_st), content, parsed) + return parsed + + +def _file_stamp(st: os.stat_result) -> _FileStamp: + """Metadata that changes with file replacement, content, or permissions.""" + return ( + st.st_dev, + st.st_ino, + st.st_mode, + st.st_size, + st.st_mtime_ns, + st.st_ctime_ns, + ) + + +def _interpret(data: dict[str, Any], path: Path) -> _ParsedFile: + """Validate a parsed TOML document into package-wide keys plus adapter tables. + + Only the top-level table is validated here, because it always applies. An + adapter's table is kept raw and validated when that adapter first resolves a + setting: a bad value in ``[nldi]`` must not fail a Water Data call. + """ + top: dict[str, Any] = {} + adapters: dict[str, dict[str, Any]] = {} + + for key, value in data.items(): + if key in ADAPTERS: + if not isinstance(value, dict): + raise ConfigurationError( + f"{path}: [{key}] must be a table of settings for the " + f"{key} adapter." + ) + adapters[key] = value + continue + if key == _RETIRED_PROFILES_TABLE: + # A file written against the earlier design, where one profile + # switched every service at once. The generic message below would + # send its author hunting for a typo in a table spelled exactly as + # the old docs said, so name the replacement instead. + raise ConfigurationError( + f"{path}: [{_RETIRED_PROFILES_TABLE}] is no longer read. A " + "profile now belongs to one adapter: write [.] " + 'and select it with Settings.load("").' + ) + if isinstance(value, dict): + raise ConfigurationError( + f"{path}: unknown table [{key}]. Per-adapter tables are " + f"{', '.join(f'[{name}]' for name in ADAPTERS)}; a named profile " + f"goes under one of them, as [.{key}]; top-level keys " + "are the package-wide defaults." + ) + top[key] = value + + return _ParsedFile(_checked_table(top, path, _TOP_LEVEL, SETTINGS), adapters, True) + + +def _accepted_keys( + table: dict[str, Any], + path: Path, + where: str, + allowed: frozenset[str] | tuple[str, ...], +) -> dict[str, Any]: + """Filter one table down to the settings it is allowed to name. + + The key policy for every table in the file, in one place, so the default + profile and a named profile cannot come to disagree about what is a typo. An + unrecognized name warns rather than raising, so a file written for a newer + release still works; a name this release *does* know but that table cannot + use raises, because that one can never become meaningful. + """ + out: dict[str, Any] = {} + for key, value in table.items(): + if isinstance(value, dict): + # A named profile -- ``[waterdata.bulk]`` parses as a sub-table of + # ``[waterdata]``. Inert until a caller selects it, so it is neither + # a setting here nor an error. + continue + if key in ADAPTER_ONLY_SETTINGS: + # Rejected from the file wherever it appears. A file that silently + # redirects a data-retrieval library to another host is a + # supply-chain-shaped hazard (ADR 0011). + raise ConfigurationError( + f"{path}: {key!r} at {where} may only be set in code, in a " + "configure() block, never from a file." + ) + if key not in allowed: + if key in SETTINGS: + # A real setting, in a table that does not read it. Unlike an + # unrecognized name -- which may simply belong to a newer release + # -- this cannot become meaningful later, and silently ignoring + # it would leave a caller believing they had tuned something. + raise ConfigurationError( + f"{path}: {key!r} at {where} is not a setting that table " + f"accepts. It accepts: {', '.join(sorted(allowed))}." + ) + warnings.warn( + f"{path}: unknown setting {key!r} at {where} (ignored). " + f"Known settings: {', '.join(SETTINGS)}.", + UserWarning, + stacklevel=_WARN_STACKLEVEL, + ) + continue + out[key] = value + return out + + +def _checked_table( + table: dict[str, Any], + path: Path, + where: str, + allowed: frozenset[str] | tuple[str, ...], +) -> dict[str, Any]: + """Check one table of the file and normalize its values. + + Every table in the file comes through here: the top-level keys, an adapter's + default profile, and a named profile. Written once because the checks are the + interesting part and they must not diverge. + + ``tomllib`` returns typed scalars (``concurrency = 32`` is an ``int``, + ``concurrency = "unbounded"`` a ``str``), and each goes through the same + grammar the other sources use, with a source that names the file and the + table -- a grammar error found on the way *out* of the file should say which + line to fix, not merely which field ended up holding it. + """ + checked: dict[str, Any] = {} + for key, value in _accepted_keys(table, path, where, allowed).items(): + if where == _TOP_LEVEL and key in _WARN_AT_TOP_LEVEL: + warnings.warn( + f"{path}: {_WARN_AT_TOP_LEVEL[key]}", + UserWarning, + stacklevel=_WARN_STACKLEVEL, + ) + checked[key] = _validate(key, value, f"{path}: {key!r} at {where}") + return checked + + +def _adapter_file_settings( + adapter: str, path: Path, parsed: _ParsedFile +) -> Mapping[str, tuple[Any, str]]: + """The ``[]`` table's own keys -- its default profile. + + Layers *above* the file's top-level keys rather than being merged into them: + within the file tier an adapter's own value outranks the package-wide one. + + Validated on first use, not at parse time, so a bad value in ``[nldi]`` + cannot fail a Water Data call -- the blast-radius rule ADR 0010 set. + """ + table = parsed.adapters.get(adapter) + if not table: + return {} + + cached = _adapter_cache.get(adapter) + if cached is not None and cached[0] is parsed and cached[1] == path: + return cached[2] + + where = f"[{adapter}]" + # An adapter this process has not imported declares no vocabulary, so its + # table is checked against the package-wide settings alone: refusing a key + # for want of a schema would make the file's validity depend on which + # optional extras happened to be installed. + accepted = settings_for(adapter) + validated = _checked_table( + table, path, where, SETTINGS if accepted is None else accepted + ) + label = f"{path} {where}" + result: Mapping[str, tuple[Any, str]] = MappingProxyType( + {name: (value, label) for name, value in validated.items()} + ) + _adapter_cache[adapter] = (parsed, path, result) + return result + + +def _named_profiles(parsed: _ParsedFile, adapter: str) -> dict[str, dict[str, Any]]: + """The named profiles the file defines for *adapter*, by name. + + A sub-table of an adapter's table is a named profile: ``[waterdata.bulk]`` + parses as a sub-table of ``[waterdata]``, and everything else in that table + is a setting of the adapter's default profile. The two readers of that rule + -- selecting a profile and reporting which ones exist -- share this one + definition so they cannot come to disagree about what a profile is. + """ + return { + name: table + for name, table in parsed.adapters.get(adapter, {}).items() + if isinstance(table, dict) + } + + +def _named_profile( + adapter: str, profile: str, allowed: frozenset[str] +) -> dict[str, Any]: + """The ``[.]`` table, checked against *allowed*.""" + path, parsed = _current_file() + named = _named_profiles(parsed, adapter) + if profile not in named: + if not parsed.exists: + raise ConfigurationError( + f"profile {profile!r} cannot be selected for {adapter}: there " + f"is no settings file at {path}." + ) + defined = ", ".join(sorted(named)) or "none" + raise ConfigurationError( + f"{path}: no [{adapter}.{profile}] table. Profiles defined for " + f"{adapter}: {defined}." + ) + + where = f"[{adapter}.{profile}]" + table = named[profile] + + # A profile is one flat set of settings for one adapter, so a table inside + # one is a shape the grammar has no reading for -- most likely a file + # migrated from the retired ``[profiles.bulk.ngwmn]``. Dropping it silently + # would leave the author believing they had tuned something. + nested = sorted(key for key, value in table.items() if isinstance(value, dict)) + if nested: + raise ConfigurationError( + f"{path}: {where} contains a table, [{adapter}.{profile}.{nested[0]}]. " + "A profile names settings for one adapter and nothing else; to " + "configure two adapters for one run, give each its own profile and " + "select both in the same configure() block." + ) + + return _checked_table(table, path, where, allowed) + + +def _holds_api_key(parsed: _ParsedFile) -> bool: + """Whether the file names an API key anywhere, including inert tables. + + Inert tables count because the question is what the *file* contains, not what + this run resolves: a key sitting in a profile nobody selected is just as + readable to another user on the machine. + """ + if "api_key" in parsed.base: + return True + return any( + "api_key" in table + or any("api_key" in p for p in table.values() if isinstance(p, dict)) + for table in parsed.adapters.values() + ) + + +def _warn_on_loose_permissions( + path: Path, st: os.stat_result, parsed: _ParsedFile +) -> None: + """Warn once if a file holding an API key is readable by other users. + + Follows the ``~/.ssh`` and ``.netrc`` convention, but warns rather than + refusing -- shared filesystems on HPC clusters have their own conventions, + and refusing to read would strand those users. + """ + if os.name != "posix" or path in _permission_warned: + return + if not _holds_api_key(parsed): + return + if stat.S_IMODE(st.st_mode) & 0o077: + _permission_warned.add(path) + warnings.warn( + f"{path} contains an API key and is readable by other users. " + f"Restrict it with: chmod 600 {path}", + UserWarning, + stacklevel=_WARN_STACKLEVEL, + ) + + +# --- the report ---------------------------------------------------------- + + +def _display_api_key(adapter: str | None = None) -> str: + """Render the key's presence, never its value.""" + return "" if api_key() else "" + + +def _display_concurrency(adapter: str | None = None) -> str: + value = concurrency(adapter=adapter) + return CONCURRENCY_UNBOUNDED if value is None else str(value) + + +def _display_progress(adapter: str | None = None) -> str: + setting = progress() + return "auto" if setting is None else ("on" if setting else "off") + + +#: How each setting renders in :func:`show_settings`. Keyed by the same names as +#: :data:`_ALL_SETTINGS`, and asserted to cover them, so a setting added to one +#: without the other fails loudly instead of silently printing a neighbour's +#: value in the one report whose whole job is to be trustworthy. +_DISPLAYS: dict[str, Callable[[str | None], str]] = { + "api_key": _display_api_key, + "concurrency": _display_concurrency, + "retries": lambda adapter: str(retries(adapter=adapter)), + "progress": _display_progress, + "parallel_chunks": lambda adapter: str(parallel_chunks(adapter=adapter)), + "stall_timeout": lambda adapter: f"{stall_timeout(adapter=adapter):g}s", + "base_url": lambda adapter: base_url(adapter=adapter) or "", +} + +if set(_DISPLAYS) != set(_ALL_SETTINGS): # pragma: no cover - guards a coding error + # Not an ``assert``: ``python -O`` strips those, and this guards the one + # report whose whole job is to be trustworthy about provenance. + raise RuntimeError( + "every setting needs a show_settings renderer; " + f"missing={sorted(set(_ALL_SETTINGS) - set(_DISPLAYS))} " + f"extra={sorted(set(_DISPLAYS) - set(_ALL_SETTINGS))}" + ) + + +def show_settings(*, stream: TextIO | None = None) -> None: + """Print the effective settings and the source of each one. + + A debugging aid for "why is this using my old key?". Every value is reported + with the source that supplied it, named exactly: which variable, which table + of the file, and -- when a caller selected one -- which profile. The API key + is never printed, only whether one is set. + + Parameters + ---------- + stream : file-like, optional + Where to write. Defaults to ``sys.stdout``. + + Examples + -------- + The sample below is generated by running this function, not written by hand; + ``test_show_settings_sample_output_is_current`` re-runs it and fails if the + two drift apart. + + .. code-block:: text + + >>> with dataretrieval.configure(WaterdataSettings.load("bulk")): + ... dataretrieval.show_settings() + settings file /home/u/.dataretrieval/config.toml (found) + api_key /home/u/.dataretrieval/config.toml + concurrency 16 /home/u/.dataretrieval/config.toml + retries 8 $API_USGS_RETRIES + progress auto built-in default + parallel_chunks 1 built-in default + stall_timeout 60s built-in default + + A built-in default is package-wide. An adapter may prefer its own for + its own calls; a value from any source above overrides both. + + adapter overrides + waterdata parallel_chunks 8 configure() block [waterdata.bulk] + ngwmn concurrency 4 /home/u/.dataretrieval/config.toml [ngwmn] + + profiles in the file: [waterdata.bulk] + A profile applies only where a row above names it; select one in + code with Settings.load(""). + + not reported: nldi (not imported, so the settings each accepts are unknown here) + """ + out = sys.stdout if stream is None else stream + try: + path = config_path() + except ConfigurationError as exc: + # Resolution itself can fail (a relative override with the working + # directory removed). That is precisely a configuration a caller would + # run this to understand, so report it as the file row rather than + # raising out of the explainer. + print(f"settings file ", file=out) + return + + # Nothing here raises. This function exists to explain a configuration, and + # the configurations most in need of explaining are the broken ones. Each + # distinct failure is printed once, in the first place it shows up; a repeat + # is collapsed, so one bad file does not bury the rows that did resolve. + reported: str | None = None + + def cell(render: Callable[[], object]) -> str: + nonlocal reported + try: + value = render() + except ConfigurationError as exc: + if str(exc) == reported: + return "" + reported = str(exc) + return f"" + return "" if value is None else str(value) + + # Probing the file once here means a whole-file problem -- unparseable TOML, + # a bad value at the top level -- is reported on the file row rather than + # repeated in every setting's row below. + parsed = _NO_FILE + try: + _, parsed = _current_file() + status = "found" if path.exists() else "not found" + except ConfigurationError as exc: + reported = str(exc) + status = f"ERROR: {exc}" + print(f"settings file {path} ({status})", file=out) + + rows = [ + ( + name, + cell(_display_for(name, None)), + cell(_label_getter(name)), + ) + for name in SETTINGS + ] + name_width = max(len(name) for name, _value, _source in rows) + value_width = max(len(value) for _name, value, _source in rows) + for name, value, source in rows: + print(f"{name:<{name_width}} {value:<{value_width}} {source}", file=out) + + # A built-in default is package-wide, and a service may prefer its own for + # its own calls -- so a row reading "built-in default" is not a promise about + # every service. Saying so is the honest scope of this report. + if any(source == _BUILT_IN for _name, _value, source in rows): + print( + "\nA built-in default is package-wide. An adapter may prefer its own " + "for\nits own calls; a value from any source above overrides both.", + file=out, + ) + + _show_adapter_overrides(out, cell, {name: source for name, _value, source in rows}) + _show_profiles(out, parsed) + _show_unimported_adapters(out) + + +def _display_for(name: str, adapter: str | None) -> Callable[[], object]: + """A thunk rendering one setting, for :func:`show_settings`'s error trap.""" + return lambda: _DISPLAYS[name](adapter) + + +def _label_getter(name: str, adapter: str | None = None) -> Callable[[], object]: + """A thunk resolving one setting's provenance label.""" + return lambda: _source_label(name, adapter) + + +def _show_adapter_overrides( + out: TextIO, + cell: Callable[[Callable[[], object]], str], + package_wide: Mapping[str, str], +) -> None: + """Print the adapter-scoped settings that differ from the rows above. + + Only settings actually overridden, and only adapters that override one: a + full adapter-by-setting grid would be mostly inherited values, burying the + answer to "what will this call use". + """ + overrides: list[tuple[str, str, str, str]] = [] + for adapter in ADAPTERS: + accepted = settings_for(adapter) + if accepted is None: + continue + for name in _ALL_SETTINGS: + if name not in accepted: + continue + scoped = cell(_label_getter(name, adapter)) + # ``package_wide`` is what the rows above already resolved. Asking + # again would repeat the work once per adapter *and* consume the + # shared error-dedupe state. An adapter-only setting has no row + # above, so its baseline is the built-in default. + if scoped == package_wide.get(name, _BUILT_IN): + continue # inherited from the package-wide tier + value = cell(_display_for(name, adapter)) + overrides.append((adapter, name, value, scoped)) + + if overrides: + print("\nadapter overrides", file=out) + a_width = max(len(a) for a, _n, _v, _s in overrides) + n_width = max(len(n) for _a, n, _v, _s in overrides) + v_width = max(len(v) for _a, _n, v, _s in overrides) + for adapter, name, value, source in overrides: + print( + f" {adapter:<{a_width}} {name:<{n_width}} " + f"{value:<{v_width}} {source}", + file=out, + ) + + +def _show_profiles(out: TextIO, parsed: _ParsedFile) -> None: + """Print the named profiles the file defines, selected or not. + + A named profile does nothing until a caller selects it, and that is the thing + readers of a settings file get wrong: adding ``[waterdata.bulk]`` changes no + run on its own. A report that mentioned a profile only when one had been + selected would leave that silence with nothing to explain it. + """ + defined = [ + f"[{adapter}.{name}]" + for adapter in ADAPTERS + for name in sorted(_named_profiles(parsed, adapter)) + ] + if not defined: + return + print(f"\nprofiles in the file: {', '.join(defined)}", file=out) + print( + " A profile applies only where a row above names it; select one in\n" + ' code with Settings.load("").', + file=out, + ) + + +def _show_unimported_adapters(out: TextIO) -> None: + """Name the adapters this process cannot report on, and say why. + + An adapter is only known to accept a setting once the module declaring that + vocabulary has been imported, and NLDI is deliberately imported on demand for + the geopandas extra. Omitting it silently would read as "nothing is + configured for nldi", which is a different claim and the wrong one. + """ + unimported = [a for a in ADAPTERS if settings_for(a) is None] + if unimported: + print( + f"\nnot reported: {', '.join(unimported)} " + "(not imported, so the settings each accepts are unknown here)", + file=out, + ) + + +def _reset_file_cache() -> None: + """Drop the parsed-file cache. For tests that rewrite the file in place.""" + global _file_cache, _path_cache + _file_cache = None + _path_cache = None + _adapter_cache.clear() + _permission_warned.clear() diff --git a/dataretrieval/streamstats.py b/dataretrieval/streamstats.py index 22b3bf45..1c45f15a 100644 --- a/dataretrieval/streamstats.py +++ b/dataretrieval/streamstats.py @@ -7,18 +7,42 @@ from __future__ import annotations import json -from typing import Any, cast +from typing import Any, ClassVar, cast import httpx +from dataretrieval import settings as _settings from dataretrieval._querying import _get_with_retry +from dataretrieval.settings import ( + AdapterSettings, + _Redirectable, + _register, + _Retrying, +) from dataretrieval.transport.http import HTTPX_DEFAULTS -__all__ = ["download_workspace", "get_sample_watershed", "get_watershed", "Watershed"] +__all__ = [ + "StreamstatsSettings", + "Watershed", + "download_workspace", + "get_sample_watershed", + "get_watershed", +] STREAMSTATS_URL = "https://streamstats.usgs.gov/streamstatsservices" +def _service_base() -> str: + """The StreamStats base this call targets: a block's redirect, or its own. + + Both endpoints below hang off this, so a + ``StreamstatsSettings(base_url=...)`` moves the whole service rather + than the one endpoint a caller happened to reach first. Resolved per call, + because a ``configure`` block is scoped to a ``with`` statement. + """ + return _settings.base_url(adapter="streamstats", default=STREAMSTATS_URL) + + def download_workspace(workspaceID: str, format: str = "") -> httpx.Response: """Download a StreamStats workspace. @@ -39,9 +63,9 @@ def download_workspace(workspaceID: str, format: str = "") -> httpx.Response: """ payload = {"workspaceID": workspaceID, "format": format} - url = f"{STREAMSTATS_URL}/download" + url = f"{_service_base()}/download" - r = _get_with_retry(url, params=payload, **HTTPX_DEFAULTS) + r = _get_with_retry(url, params=payload, adapter="streamstats", **HTTPX_DEFAULTS) return r # data = r.raw.read() @@ -142,9 +166,9 @@ def get_watershed( "includefeatures": includefeatures, "simplify": simplify, } - url = f"{STREAMSTATS_URL}/watershed.geojson" + url = f"{_service_base()}/watershed.geojson" - r = _get_with_retry(url, params=payload, **HTTPX_DEFAULTS) + r = _get_with_retry(url, params=payload, adapter="streamstats", **HTTPX_DEFAULTS) if format == "geojson": return r @@ -215,3 +239,36 @@ def _populate(self, streamstats_json: dict[str, Any]) -> None: self.watershed_polygon = streamstats_json["featurecollection"][1]["feature"] self.parameters = streamstats_json["parameters"] self._workspaceID = streamstats_json["workspaceID"] + + +class StreamstatsSettings(_Redirectable, _Retrying, AdapterSettings): + """Settings for StreamStats calls alone. + + No fan-out dials: a StreamStats query is answered by a single + request. + + Lives here rather than in :mod:`dataretrieval.settings` because + *which* settings a service reads is the service's own knowledge (ADR + 0011); what each of them means is shared, so the fields come from the + setting groups declared beside their grammar. + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying + stops. + base_url : str, optional + Services base to send StreamStats requests to, instead of its own + (``STREAMSTATS_URL``). Both endpoints hang off it. Code only: + the file and the environment refuse it. + """ + + # One request per call, so this service reads the retry dials and a + # redirectable base and no fan-out dial. Each setting is declared once, + # in :mod:`dataretrieval.settings`, beside its grammar. + adapter: ClassVar[str] = "streamstats" + + +_register(StreamstatsSettings) diff --git a/dataretrieval/transport/env.py b/dataretrieval/transport/env.py deleted file mode 100644 index 2bfaf472..00000000 --- a/dataretrieval/transport/env.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Environment parsing for the ``API_USGS_*`` numeric knobs. - -A dependency-free leaf: every transport setting read from the environment -shares one grammar and one error voice, and no policy module has to be -imported to get at the parser. -""" - -from __future__ import annotations - -import math -import os -from collections.abc import Callable -from typing import TypeVar - -from dataretrieval.exceptions import ConfigurationError - -_Number = TypeVar("_Number", int, float) - - -def _read_env_number( - name: str, - default: _Number, - cast: Callable[[str], _Number], - expected: str, - *, - minimum: float = 0, - hint: str = "", -) -> _Number: - """Read a bounded number from the environment, or ``default`` if unset. - - The single parser behind every ``API_USGS_*`` numeric knob, so they share - one grammar and one error voice rather than each adapter hand-rolling the - read-cast-validate sequence its own way. - - Raises :class:`~dataretrieval.exceptions.ConfigurationError` -- a - ``DataRetrievalError`` *and* a ``ValueError`` -- for an unusable value, so a - typo in the environment doesn't escape a request path as a bare - ``ValueError`` that ``except DataRetrievalError`` misses. ``hint`` appends - a sentence pointing at the fix when a setting has one (e.g. the keyword - that disables a cap). - """ - raw = os.environ.get(name, "").strip() - if not raw: - return default - try: - value = cast(raw) - except ValueError as exc: - raise ConfigurationError( - f"{name} must be {expected} (got {raw!r}).{hint}" - ) from exc - # ``nan`` passes every ordering test, so a bare ``< minimum`` guard lets it - # through and then silently makes each budget comparison false. - if not math.isfinite(value): - raise ConfigurationError(f"{name} must be {expected} (got {raw!r}).{hint}") - if value < minimum: - raise ConfigurationError(f"{name} must be >= {minimum:g} (got {value}).{hint}") - return value diff --git a/dataretrieval/transport/fanout.py b/dataretrieval/transport/fanout.py index 53b46c8a..74371a57 100644 --- a/dataretrieval/transport/fanout.py +++ b/dataretrieval/transport/fanout.py @@ -27,9 +27,11 @@ ``asyncio.Semaphore`` -- not the client's connection pool, which is merely sized to match -- caps the chunks in flight at ``N``; see :meth:`FanOut._run` for why the gate must be the semaphore rather than the pool. -``API_USGS_CONCURRENT`` resolves ``N``: an integer N > 1 allows N chunks -in flight; ``1`` forces sequential dispatch; the literal ``unbounded`` lifts the -cap. ``N`` bounds only how many of a query's chunks are in flight at once +The ``concurrency`` setting resolves ``N`` -- a ``configure()`` block, then +``API_USGS_CONCURRENT``, then the config file, and per adapter as well as +package-wide: an integer N > 1 allows N chunks in flight; ``1`` forces +sequential dispatch; the literal ``unbounded`` lifts the cap. ``N`` bounds only +how many of a query's chunks are in flight at once -- a client-side trade-off between open connections and fan-out latency. It does not affect the API rate limit: a fanned-out call issues the same number of chunks regardless of ``N``, so ``N`` changes their timing, not the total @@ -41,9 +43,10 @@ Retries: each chunk is retried on a transient failure (429, 5xx, connect/read timeout) with exponential backoff + full jitter, honoring a server -``Retry-After`` when present. ``API_USGS_RETRIES`` sets the cap (default 4; -``0`` disables). A ``Retry-After`` longer than the per-call ceiling escalates to -a resumable interruption. +``Retry-After`` when present. The ``retries`` setting caps them (default 4; +``0`` disables), resolved through the same chain and scopable per adapter. A +``Retry-After`` longer than the per-call ceiling escalates to a resumable +interruption. Interruption: any mid-stream transient failure surfaces as a :class:`~dataretrieval.interruptions.FanOutInterrupted` subclass carrying @@ -56,7 +59,6 @@ import asyncio import functools -import os from collections.abc import Awaitable, Callable, Iterator from typing import Any, Generic, Protocol, TypeVar, cast @@ -65,6 +67,7 @@ from anyio.from_thread import start_blocking_portal from dataretrieval import progress as _progress +from dataretrieval import settings as _settings from dataretrieval._ambient import Ambient from dataretrieval.combining import ( _combine_chunk_frames, @@ -75,7 +78,6 @@ _classify_chunk_error, _walk_causes, ) -from dataretrieval.transport.env import _read_env_number from dataretrieval.transport.http import network_error, open_async_client from dataretrieval.transport.retry import _NO_RETRY, RetryPolicy from dataretrieval.transport.retry import retry_async as _retry @@ -88,56 +90,12 @@ #: supertype, the way ``Iterable`` is covariant for the same reason. _ChunkCo = TypeVar("_ChunkCo", covariant=True) -# Fan-out concurrency cap, read at call time (not import) so test -# ``monkeypatch.setenv`` applies. Value grammar in :func:`_read_concurrency_env`; -# the concurrency model is in the module docstring. -_CONCURRENCY_ENV = "API_USGS_CONCURRENT" -_CONCURRENCY_DEFAULT = 32 -_CONCURRENCY_UNBOUNDED = "unbounded" - - -def _resolve_concurrency(default: int = _CONCURRENCY_DEFAULT) -> int | None: - """ - Resolve the parallelism cap: the general setting, or a module's default. - - ``API_USGS_CONCURRENT`` is the general knob and applies to every fanned-out - call in the package. A module may pass a different ``default`` when its - service warrants one — Water Use ships a lower figure than the OGC getters, - because the NWDC is only stress-tested to that level. - - The ordering is deliberate: an explicitly set environment variable wins over - a module's default, never the reverse. A module that could override the - general setting would make ``API_USGS_CONCURRENT=1`` a lie — the user - dialing concurrency down to be polite to the service would find one adapter - quietly ignoring them, which is precisely the defect this consolidates away. - Module defaults express "absent instruction, this service prefers N"; they - do not express "this service knows better than you". - - Parameters - ---------- - default : int - Cap to use when ``API_USGS_CONCURRENT`` is unset or empty. - - Returns - ------- - int or None - ``1`` for sequential dispatch (one chunk at a time); an - integer >1 for bounded concurrency; ``None`` to disable the - per-call cap entirely (the ``unbounded`` keyword). - """ - # Only the ``unbounded`` keyword is specific to this knob; the rest is the - # same read-cast-validate every ``API_USGS_*`` number gets, so it delegates - # rather than growing a third copy with its own error wording. - if os.environ.get(_CONCURRENCY_ENV, "").strip().lower() == _CONCURRENCY_UNBOUNDED: - return None - return _read_env_number( - _CONCURRENCY_ENV, - default, - int, - f"a positive integer or '{_CONCURRENCY_UNBOUNDED}'", - minimum=1, - hint=f" Use '{_CONCURRENCY_UNBOUNDED}' to disable the cap.", - ) +# The fan-out concurrency cap resolves through +# :func:`dataretrieval.settings.concurrency`, which owns the setting's name, its +# grammar (``1`` sequential, >1 bounded, ``unbounded`` uncapped) and its +# built-in default. Naming any of those here too would let this module and the +# chain disagree about what a value means. The concurrency model -- why the cap +# is a semaphore rather than the connection pool -- is in the module docstring. # --------------------------------------------------------------------------- @@ -281,8 +239,8 @@ class FanOut(Generic[_Chunk]): Extra ``httpx.AsyncClient`` options for the shared client this run opens (e.g. ``{"verify": False}``). default_concurrent : int, optional - This service's preferred in-flight cap when ``API_USGS_CONCURRENT`` - is unset. Defaults to 32. + This adapter's preferred in-flight cap for when nothing is + configured. Any resolved ``concurrency`` outranks it. Defaults to 32. canonical_url : str or None, optional URL identifying the query as a whole, restored onto the combined response so the caller sees the request they made rather than @@ -290,7 +248,7 @@ class FanOut(Generic[_Chunk]): :meth:`resume` labels its progress line with. service : str or None, optional Human-facing name of what is being retrieved (e.g. ``"daily"``, - ``"wateruse"``), used to label the progress line :meth:`resume` + ``"nwdc"``), used to label the progress line :meth:`resume` opens. ``None`` leaves the line unlabelled. Attributes @@ -319,10 +277,11 @@ def __init__( retry_policy: RetryPolicy = _NO_RETRY, finalize: _Finalize = _passthrough_result, client_options: dict[str, Any] | None = None, - default_concurrent: int = _CONCURRENCY_DEFAULT, + default_concurrent: int = _settings.DEFAULT_CONCURRENCY, *, canonical_url: str | None = None, service: str | None = None, + adapter: str | None = None, ) -> None: self.plan = plan self.fetch = fetch @@ -333,10 +292,17 @@ def __init__( # to ``canonical_url``, because this class is what emits the progress # events — see :meth:`resume`. self.service = service - # This service's preferred cap when the user has not set - # ``API_USGS_CONCURRENT``. Resolved at resume time, not here, so a - # test's ``monkeypatch.setenv`` still applies. See - # :func:`_resolve_concurrency` for why the env var outranks it. + # Which adapter's settings this drive resolves, so a ``[ngwmn]`` table + # or an ``NgwmnSettings`` reaches only NGWMN calls. Distinct from + # ``service`` above, which is a *display label* for the progress line + # and is variously a collection name or prose. ``None`` resolves + # package-wide. See ADR 0010. + self.adapter = adapter + # This service's preferred cap for when nothing is configured. Resolved + # at resume time, not here, so a setting that arrives after this call + # was built still applies. Anything the chain resolves outranks it -- + # see :func:`dataretrieval.settings.concurrency` for why a service + # preference must not override an explicit setting. self.default_concurrent = default_concurrent # Extra ``httpx.AsyncClient`` options merged into the shared client this # run opens (``verify`` for the Water Use ``ssl_check`` flag, say). The @@ -543,7 +509,14 @@ def resume(self) -> tuple[pd.DataFrame, Any]: with _progress.progress_context( service=self.service, target_url=self.canonical_url ): - concurrency = _resolve_concurrency(self.default_concurrent) + # Resolve concurrency here, per drive, rather than at construction. + # It is the one dial a caller adjusts precisely *while* retrying -- + # the documented recovery from QuotaExhausted is to wait and + # re-issue more gently -- so a ``configure()`` block entered + # between the interruption and the resume has to win. + concurrency = _settings.concurrency( + self.default_concurrent, adapter=self.adapter + ) with start_blocking_portal() as portal: # ``portal.call`` returns ``Any`` because ``functools.partial`` # erases ``_run``'s return type; restore the declared tuple. diff --git a/dataretrieval/transport/http.py b/dataretrieval/transport/http.py index 1b2e29b3..b14d5bb6 100644 --- a/dataretrieval/transport/http.py +++ b/dataretrieval/transport/http.py @@ -48,16 +48,26 @@ def default_headers(target_url: str | httpx.URL | None = None) -> dict[str, str]: - """Build standard headers, scoping the API key to its authorized host.""" + """Build standard headers, scoping the API key to its authorized host. + + The host is checked *before* the key is resolved, and the key is resolved + only for the authorized host. Order matters now that settings come from a + layered chain: resolution reads the config file and can raise + :class:`~dataretrieval.exceptions.ConfigurationError` for a malformed file or + a profile it no longer defines. Resolving first would let a Water Data + configuration problem break a legacy NWIS, WQP, or NGWMN call that would + never have received the key. + """ headers = { "Accept-Encoding": "compress, gzip", "Accept": "application/json", "User-Agent": USER_AGENT, "lang": "en-US", } - token = api_key() - if token and accepts_api_key(target_url): - headers["X-Api-Key"] = token + if accepts_api_key(target_url): + token = api_key() + if token: + headers["X-Api-Key"] = token return headers diff --git a/dataretrieval/transport/retry.py b/dataretrieval/transport/retry.py index 95dd1b88..456f5c3e 100644 --- a/dataretrieval/transport/retry.py +++ b/dataretrieval/transport/retry.py @@ -12,13 +12,13 @@ import httpx from dataretrieval import progress as _progress +from dataretrieval import settings as _settings from dataretrieval.exceptions import ( ConfigurationError, NetworkError, TransientError, ) from dataretrieval.interruptions import _deterministic_failure -from dataretrieval.transport.env import _read_env_number from dataretrieval.transport.liveness import ( credit_wait, elapsed_since_progress, @@ -38,8 +38,6 @@ # on a request that can never succeed and delays the caller's error. _RETRYABLE_STATUSES = frozenset({429, *range(500, 600)}) _GATEWAY_STATUSES = frozenset({429, 502, 503, 504}) -_RETRIES_ENV = "API_USGS_RETRIES" -_RETRIES_DEFAULT = 4 _RETRY_BASE_BACKOFF = 0.5 _RETRY_MAX_BACKOFF = 30.0 _RETRY_AFTER_CAP = 60.0 @@ -49,8 +47,6 @@ _RETRY_AFTER_JITTER = 1.0 # Attempts the no-progress budget never withholds; see RetryPolicy.allows_wait. _STALL_EXEMPT_ATTEMPTS = 1 -_STALL_TIMEOUT_ENV = "API_USGS_STALL_TIMEOUT" -_STALL_TIMEOUT_DEFAULT = 60.0 _T = TypeVar("_T") @@ -64,8 +60,10 @@ class RetryPolicy: call may go on receiving nothing. """ - #: Attempts after the first. ``0`` disables retry entirely. - max_retries: int = _RETRIES_DEFAULT + #: Attempts after the first. ``0`` disables retry entirely. The default is + #: ``config``'s, not a second copy of it: a directly-constructed policy and + #: one built by :meth:`from_settings` must agree on the retry budget. + max_retries: int = _settings.DEFAULT_RETRIES #: First backoff ceiling; doubles per attempt up to :attr:`max_backoff`. base_backoff: float = _RETRY_BASE_BACKOFF #: Ceiling for our own exponential backoff. @@ -91,7 +89,7 @@ class RetryPolicy: #: productive download is never cut short, and an attempt already in flight #: is never interrupted. ``0`` disables the bound. See :meth:`allows_wait` #: for how it is applied. - stall_timeout: float = _STALL_TIMEOUT_DEFAULT + stall_timeout: float = _settings.DEFAULT_STALL_TIMEOUT def __post_init__(self) -> None: if self.max_retries < 0: @@ -107,25 +105,32 @@ def __post_init__(self) -> None: raise ConfigurationError("retry backoff settings must be non-negative.") @classmethod - def from_env(cls, retryable_statuses: frozenset[int] | None = None) -> RetryPolicy: - """Build a policy from current environment and module defaults.""" + def from_settings( + cls, + retryable_statuses: frozenset[int] | None = None, + *, + adapter: str | None = None, + ) -> RetryPolicy: + """Build a policy from the effective configuration and module defaults. + + ``max_retries`` and ``stall_timeout`` both resolve through + :mod:`dataretrieval.settings` -- a ``configure()`` block, then the + environment variable, then the config file. ``adapter`` names the + adapter this policy is for, so a ``[wqp] retries = 2`` table applies to + WQP calls and nothing else; ``None`` resolves package-wide. The pure + timing knobs stay module constants read at call time so a test's + ``monkeypatch.setattr`` still applies. + """ statuses = ( _RETRYABLE_STATUSES if retryable_statuses is None else retryable_statuses ) return cls( retryable_statuses=statuses, - max_retries=_read_env_number( - _RETRIES_ENV, _RETRIES_DEFAULT, int, "a non-negative integer" - ), + max_retries=_settings.retries(adapter=adapter), base_backoff=_RETRY_BASE_BACKOFF, max_backoff=_RETRY_MAX_BACKOFF, retry_after_cap=_RETRY_AFTER_CAP, - stall_timeout=_read_env_number( - _STALL_TIMEOUT_ENV, - _STALL_TIMEOUT_DEFAULT, - float, - "a non-negative number of seconds", - ), + stall_timeout=_settings.stall_timeout(adapter=adapter), ) def should_retry(self, attempt: int, retry_after: float | None) -> bool: @@ -272,7 +277,7 @@ async def retry_async( silence. A caller that gated its own body would have to rediscover both, and nothing would catch it getting them wrong. """ - policy = RetryPolicy.from_env() if policy is None else policy + policy = RetryPolicy.from_settings() if policy is None else policy attempt = 0 note_progress() @@ -303,7 +308,7 @@ def retry_sync(fn: Callable[[], _T], policy: RetryPolicy | None = None) -> _T: not caught because the loop handles ``Exception`` rather than ``BaseException``. """ - policy = RetryPolicy.from_env() if policy is None else policy + policy = RetryPolicy.from_settings() if policy is None else policy attempt = 0 note_progress() while True: diff --git a/dataretrieval/waterdata/__init__.py b/dataretrieval/waterdata/__init__.py index 48c4d9fb..d76d1534 100644 --- a/dataretrieval/waterdata/__init__.py +++ b/dataretrieval/waterdata/__init__.py @@ -35,6 +35,7 @@ ) from .nearest import get_nearest_continuous from .ratings import get_ratings +from .settings import WaterdataSettings from .types import ( CODE_SERVICES, PROFILE_LOOKUP, @@ -46,6 +47,7 @@ __all__ = [ "CODE_SERVICES", "FILTER_LANG", + "WaterdataSettings", "PROFILES", "PROFILE_LOOKUP", "SERVICES", diff --git a/dataretrieval/waterdata/cql.py b/dataretrieval/waterdata/cql.py index fab39066..4bf73439 100644 --- a/dataretrieval/waterdata/cql.py +++ b/dataretrieval/waterdata/cql.py @@ -21,6 +21,7 @@ _switch_properties_id, ) from dataretrieval.ogc.shaping import _finalize_ogc +from dataretrieval.waterdata.endpoints import redirected from dataretrieval.waterdata.utils import ( _EXTRA_ID_COLS, _OUTPUT_ID_BY_COLLECTION, @@ -165,18 +166,22 @@ def get_cql( # The OGC package names no collection of its own, so this hand-built request # path states the target itself -- to request construction and to the - # empty-result schema lookup in ``_finalize_ogc``. + # empty-result schema lookup in ``_finalize_ogc``. Resolved once so both + # reach the same API: a redirect that moved the query but left the schema + # lookup on the service's own host would describe one API's empty result + # with another's columns. + api_base = redirected(OGC_API_URL) req = _construct_cql_request( collection, body, - base_url=OGC_API_URL, + base_url=api_base, properties=wire_properties, bbox=bbox, limit=limit, skip_geometry=skip_geometry, ) - df, response = fetch_ogc_request(req, collection=collection) + df, response = fetch_ogc_request(req, collection=collection, adapter="waterdata") return _finalize_ogc( df, @@ -187,7 +192,7 @@ def get_cql( collection=collection, extra_id_cols=_EXTRA_ID_COLS, dialect=WATERDATA_DIALECT, - base_url=OGC_API_URL, + base_url=api_base, ) diff --git a/dataretrieval/waterdata/endpoints.py b/dataretrieval/waterdata/endpoints.py index 7611d120..0077e88f 100644 --- a/dataretrieval/waterdata/endpoints.py +++ b/dataretrieval/waterdata/endpoints.py @@ -7,12 +7,14 @@ key -- while the paths below stay here rather than importing OGC policy internals. -This module imports nothing but that leaf, so a family module can name its -endpoint without also taking on an OGC or transport edge. +This module imports only leaves -- the credentials host and the configuration +chain -- so a family module can name its endpoint, and honor a caller's +redirect, without also taking on an OGC or transport edge. """ from __future__ import annotations +from dataretrieval import settings as _settings from dataretrieval.credentials import WATERDATA_BASE_URL #: Root of the modernized Water Data APIs. @@ -31,6 +33,28 @@ #: STAC catalog serving NWIS rating-curve assets. STAC_URL = f"{BASE_URL}/stac/v0" + +def redirected(endpoint: str) -> str: + """Move one endpoint onto the base URL a ``configure`` block set, if any. + + Water Data is one adapter serving four families -- OGC, samples, + statistics, ratings -- so ``WaterdataSettings(base_url=...)`` names the + *root* they all hang off, and every constant above is built from + :data:`BASE_URL` so that swapping that prefix moves all of them together. A + redirect that reached only the family a caller happened to call first would + send the rest to the service the caller was trying not to talk to, which is + the one mistake a redirect must not make. + + Resolved per call rather than at import, because a ``configure`` block is + scoped to a ``with`` statement and delivered through a ``ContextVar``: a + constant computed once could only ever describe the process, not the call. + """ + override = _settings.base_url(adapter="waterdata") + if override is None: + return endpoint + return override + endpoint.removeprefix(BASE_URL) + + __all__ = [ "BASE_URL", "OGC_API_URL", @@ -38,4 +62,5 @@ "STAC_URL", "STATISTICS_API_URL", "STATISTICS_API_VERSION", + "redirected", ] diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index ff852029..3a7c3aab 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -33,7 +33,7 @@ get as _get, ) from dataretrieval.transport.links import resolve_next_url -from dataretrieval.waterdata.endpoints import STAC_URL +from dataretrieval.waterdata.endpoints import STAC_URL, redirected __all__ = ["get_ratings"] @@ -252,7 +252,7 @@ def _search( if bbox is not None: query_params["bbox"] = ",".join(map(str, bbox)) - url: str | None = f"{STAC_URL}/search" + url: str | None = f"{redirected(STAC_URL)}/search" # ``params`` is sent only on the first request; each STAC ``next`` link # already carries the query, so it is reset to None inside the loop. params: dict[str, Any] | None = query_params diff --git a/dataretrieval/waterdata/reference.py b/dataretrieval/waterdata/reference.py index 14a1884b..b5bf0b76 100644 --- a/dataretrieval/waterdata/reference.py +++ b/dataretrieval/waterdata/reference.py @@ -13,6 +13,7 @@ import pandas as pd from dataretrieval.ogc.schema import queryables_frame +from dataretrieval.waterdata.endpoints import redirected from dataretrieval.waterdata.types import ( METADATA_COLLECTIONS, ) @@ -160,8 +161,10 @@ def get_queryables(collection: str) -> tuple[pd.DataFrame, BaseMetadata]: 'string' """ # Reading the queryables document is OGC protocol work; this getter only - # names the API to ask. - return queryables_frame(collection, base_url=OGC_API_URL) + # names the API to ask -- which is the redirected one when a ``configure`` + # block set a base URL, so the queryables describe the API the getters are + # actually querying. + return queryables_frame(collection, base_url=redirected(OGC_API_URL)) __all__ = ["get_reference_table", "get_queryables"] diff --git a/dataretrieval/waterdata/samples.py b/dataretrieval/waterdata/samples.py index 642fd05a..26e72a29 100644 --- a/dataretrieval/waterdata/samples.py +++ b/dataretrieval/waterdata/samples.py @@ -31,6 +31,7 @@ from dataretrieval.transport.http import ( get as _get, ) +from dataretrieval.waterdata.endpoints import redirected from dataretrieval.waterdata.types import ( CODE_SERVICES, PROFILES, @@ -70,7 +71,13 @@ def get_codes(code_service: CODE_SERVICES) -> tuple[pd.DataFrame, BaseMetadata]: f"Valid options are: {valid_code_services}." ) - url = f"{SAMPLES_URL}/codeservice/{code_service}?mimeType=application%2Fjson" + # ``redirected`` applies a ``WaterdataSettings(base_url=...)`` from an + # enclosing block; the Samples database is one of the four families that + # move together when a caller redirects the adapter. + url = ( + f"{redirected(SAMPLES_URL)}/codeservice/{code_service}" + "?mimeType=application%2Fjson" + ) response = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) @@ -366,7 +373,7 @@ def get_samples( if "boundingBox" in params: params["boundingBox"] = to_str(params["boundingBox"]) - url = f"{SAMPLES_URL}/{service}/{profile}" + url = f"{redirected(SAMPLES_URL)}/{service}/{profile}" df, response = _get_samples_csv(url, params, ssl_check) df = _attach_datetime_columns(df) @@ -428,7 +435,7 @@ def get_samples_summary( f"request, got {type(monitoring_location_id).__name__}." ) - url = f"{SAMPLES_URL}/summary/{quote(monitoring_location_id, safe='')}" + url = f"{redirected(SAMPLES_URL)}/summary/{quote(monitoring_location_id, safe='')}" params = {"mimeType": "text/csv"} df, response = _get_samples_csv(url, params, ssl_check) diff --git a/dataretrieval/waterdata/settings.py b/dataretrieval/waterdata/settings.py new file mode 100644 index 00000000..cb5e62d1 --- /dev/null +++ b/dataretrieval/waterdata/settings.py @@ -0,0 +1,67 @@ +"""The settings the Water Data adapter reads -- its configuration profile. + +A file of its own because :mod:`dataretrieval.waterdata` is a package rather +than a single module; every other adapter declares its class in the module a +caller imports. Either way the point is the same: a setting's definition sits +with the code that reads it, so adding one no longer edits a service-neutral +file (ADR 0011). +""" + +from __future__ import annotations + +from typing import ClassVar + +from dataretrieval.settings import ( + AdapterSettings, + _Chunked, + _Concurrent, + _Redirectable, + _register, + _Retrying, +) + +__all__ = ["WaterdataSettings"] + + +class WaterdataSettings( + _Chunked, _Concurrent, _Redirectable, _Retrying, AdapterSettings +): + """Settings for Water Data calls alone. + + Pass one to :func:`dataretrieval.configure` to narrow a setting to this + service, leaving every other adapter on whatever the tiers below it + resolve:: + + with dataretrieval.configure(WaterdataSettings(concurrency=8)): + df, md = waterdata.get_daily(monitoring_location_id=sites) + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying stops. + base_url : str, optional + Root to send Water Data requests to, instead of the service's own. The + package appends its own paths, so one value moves all four families + together -- ``/ogcapi/v0``, ``/samples-data``, ``/statistics/v0`` and + ``/stac/v0``. Code only: the file and the environment refuse it. The + API key is scoped to the host that honors it, so a redirected call + carries no key. + concurrency : int or str, optional + Cap on simultaneous sub-requests, or ``"unbounded"``. + parallel_chunks : int, optional + Baseline fan-out for multi-value queries. Each sub-request spends + rate-limit quota, so raise it only for pulls you know are large. + """ + + # The settings this service reads, named by the groups they come from: + # every adapter's retry dials, a redirectable base, and -- because Water + # Data queries divide along a URL byte budget and are executed concurrently + # -- both fan-out dials. Each group declares the setting itself once, in + # :mod:`dataretrieval.settings`, which is also where its grammar and + # its coercion live. + adapter: ClassVar[str] = "waterdata" + + +_register(WaterdataSettings) diff --git a/dataretrieval/waterdata/stats.py b/dataretrieval/waterdata/stats.py index 88ff751e..16b3add5 100644 --- a/dataretrieval/waterdata/stats.py +++ b/dataretrieval/waterdata/stats.py @@ -30,7 +30,7 @@ from dataretrieval.transport.http import default_headers from dataretrieval.transport.pagination import paginate from dataretrieval.transport.retry import RetryPolicy -from dataretrieval.waterdata.endpoints import STATISTICS_API_URL +from dataretrieval.waterdata.endpoints import STATISTICS_API_URL, redirected __all__ = ["get_data"] @@ -249,7 +249,7 @@ def get_data( :doc:`/userguide/errors`). """ - url = f"{STATISTICS_API_URL}/{service}" + url = f"{redirected(STATISTICS_API_URL)}/{service}" req = httpx.Request( method="GET", url=url, @@ -290,9 +290,10 @@ async def _fetch(request: httpx.Request) -> tuple[pd.DataFrame, httpx.Response]: df, response = FanOut( [req], _fetch, - RetryPolicy.from_env(), + RetryPolicy.from_settings(adapter="waterdata"), canonical_url=str(req.url), service=service, + adapter="waterdata", ).resume() if expand_percentiles: diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index c24624d0..a5a27843 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -24,12 +24,18 @@ import pandas as pd from dataretrieval.codes.states import apply_state +from dataretrieval.credentials import refuse_credential_keywords from dataretrieval.ogc import OgcDialect, prepare_request_args from dataretrieval.ogc import get_ogc_data as _facade_get_ogc_data # Endpoint constants live in one place for the whole collection; they are re-bound # here because ``waterdata.utils.OGC_API_URL`` is a documented path. -from dataretrieval.waterdata.endpoints import BASE_URL, OGC_API_URL, SAMPLES_URL +from dataretrieval.waterdata.endpoints import ( + BASE_URL, + OGC_API_URL, + SAMPLES_URL, + redirected, +) if TYPE_CHECKING: from dataretrieval._response_metadata import BaseMetadata @@ -130,7 +136,14 @@ def _flatten_queryables(local_vars: dict[str, Any]) -> dict[str, Any]: popped, so this is a no-op on getters without the passthrough and idempotent if called twice. """ - local_vars.update(local_vars.pop("queryables", {})) + queryables = local_vars.pop("queryables", {}) + # A credential-shaped name would go out in the query string, which is the + # one thing this passthrough must not forward. The predicate lives in the + # credentials leaf rather than here: what motivates it -- ``api_key=`` being + # a plausible guess now that ``configure()`` takes it -- is package-wide, + # and WQP's ``**kwargs`` search filters read the same list. + refuse_credential_keywords(queryables) + local_vars.update(queryables) return local_vars @@ -216,9 +229,18 @@ def get_ogc_data( collection, output_id, max_rows=max_rows, - base_url=OGC_API_URL, + # ``redirected`` honors a ``WaterdataSettings(base_url=...)`` set + # by an enclosing ``configure`` block, and is a no-op otherwise. Called + # here rather than bound once at import because the block is scoped to + # a ``with`` statement. + base_url=redirected(OGC_API_URL), extra_id_cols=_EXTRA_ID_COLS, dialect=WATERDATA_DIALECT, + # Which settings table these calls read. Declared here, in the one + # wrapper every Water Data getter goes through, rather than derived + # from ``base_url``: NGWMN is served from the same host, so a URL + # cannot tell the two adapters apart (ADR 0010). + adapter="waterdata", ) diff --git a/dataretrieval/wateruse.py b/dataretrieval/wateruse.py index 7fe69e59..53bdc254 100644 --- a/dataretrieval/wateruse.py +++ b/dataretrieval/wateruse.py @@ -1,462 +1,54 @@ -"""Retrieve USGS water-use data from the NWDC web service. - -The National Water Availability Assessment Data Companion (NWDC) web services -provide national-scale, USGS-modeled water-use data that underlie the `USGS -National Water Availability Assessment `_. -Estimates are served on a HUC12 (12-digit hydrologic unit) spatial grid and can -be queried for any county, state, or hydrologic unit. This is the modern -replacement for the defunct legacy NWIS water-use service -(``nwis.get_water_use``). - -Unlike the main Water Data getters (:mod:`dataretrieval.waterdata`) and NGWMN -(:mod:`dataretrieval.ngwmn`), the NWDC is a plain CSV REST service rather than -an OGC API Features collection. This module supplies the NWDC-specific bits — -request building, CSV parsing, the ``Link``-header cursor, and the ``{detail}`` -error envelope. The service-neutral transport layer supplies cursor pagination, -response aggregation, client lifecycle, and sync-from-async dispatch. The module -follows the same conventions: host-scoped request headers, the typed -:class:`~dataretrieval.exceptions.DataRetrievalError` taxonomy, and a -``(DataFrame, BaseMetadata)`` return. - -See https://api.water.usgs.gov/docs/nwaa-data/ for the API reference and -https://water.usgs.gov/nwaa-data/ for the catalog of available models and -variables. - -Examples --------- -.. code-block:: python - - from dataretrieval import wateruse - - # Monthly public-supply withdrawals for Rhode Island, 2020 onward. - df, md = wateruse.get_wateruse( - model="wu-public-supply-wd", - variable=["pswdtot", "pswdgw", "pswdsw"], - state="RI", - start_date="2020-01", - time_resolution="monthly", - ) - +"""Deprecated alias for :mod:`dataretrieval.nwdc`. + +The module was named for one subset of what the service offers. The National +Water Availability Assessment Data Companion serves ten modeled datasets, of +which the water-use models are five; the rest are hydrologic, +atmospheric-forcing, and assessment outputs. Every other adapter in this +package is named for its service -- ``ngwmn``, ``nldi``, ``wqp``, +``streamstats``, ``nwis`` -- so this one is now ``nwdc``. + +Importing this module emits a :class:`DeprecationWarning` and re-exports +:mod:`dataretrieval.nwdc`'s public surface. The re-exported objects are the +*same objects*, not copies -- ``wateruse.get_wateruse is nwdc.get_wateruse`` +-- so calls and identity comparisons behave identically through either +spelling. + +It is an alias for reading, not a second name for the module. This is a +distinct module object holding its own references to the five public names, +so it does not forward *assignment* or private names: rebinding +``wateruse.get_wateruse`` leaves ``nwdc``'s global untouched (and so has no +effect on anything ``nwdc`` does internally), and ``wateruse._WATERUSE_HOST`` +does not exist. Code that monkeypatches, or that reaches for a private, must +name :mod:`dataretrieval.nwdc` directly -- which is the point of the +deprecation. + +``dataretrieval.__init__`` deliberately imports :mod:`dataretrieval.nwdc` +rather than this module, so ``import dataretrieval`` stays silent. The warning +fires only for code that names ``wateruse`` itself. """ from __future__ import annotations -import io -from collections.abc import Callable, Iterable -from typing import Any - -import httpx -import pandas as pd +import warnings -from dataretrieval._querying import _raise_for_status, to_str -from dataretrieval._response_metadata import BaseMetadata -from dataretrieval.codes.states import to_state -from dataretrieval.exceptions import DataRetrievalError -from dataretrieval.transport.fanout import FanOut, active_client -from dataretrieval.transport.http import default_headers, network_error -from dataretrieval.transport.links import resolve_next_url -from dataretrieval.transport.pagination import paginate -from dataretrieval.transport.retry import RetryPolicy +from dataretrieval import nwdc as _nwdc +from dataretrieval.nwdc import * # noqa: F403 (re-export the public surface) -__all__ = [ - "get_wateruse", - "WATERUSE_URL", - "MODELS", - "TIME_RESOLUTIONS", - "DEFAULT_CONCURRENT_REQUESTS", -] +#: When the alias may be deleted. Announced rather than open-ended so callers +#: can plan; matches the dated-removal convention :mod:`dataretrieval.nwis` +#: uses. +NWDC_RENAME_REMOVAL_DATE = "2027-08-11" -WATERUSE_URL = "https://api.water.usgs.gov/nwaa-data/data" -_WATERUSE_HOST = httpx.URL(WATERUSE_URL).host -# Hosts a ``rel="next"`` cursor may name for this same service; each is -# rewritten to :data:`_WATERUSE_HOST` rather than followed as given. -_WATERUSE_HOST_ALIASES = frozenset({_WATERUSE_HOST, "water.usgs.gov"}) +__all__ = list(_nwdc.__all__) -#: Water-use models (categories) served by the NWDC. The catalog at -#: https://water.usgs.gov/nwaa-data/ lists the variables available within each. -MODELS = ( - "wu-public-supply-wd", # public-supply withdrawals - "wu-public-supply-cu", # public-supply consumptive use - "wu-thermoelectric", # thermoelectric-power water use - "wu-irrigation-wd", # irrigation withdrawals - "wu-irrigation-cu", # irrigation consumptive use +warnings.warn( + "`dataretrieval.wateruse` is deprecated and will be removed from " + f"`dataretrieval` on or after {NWDC_RENAME_REMOVAL_DATE}; " + "use `dataretrieval.nwdc` instead. The service is the National Water " + "Availability Assessment Data Companion, and water use is one of the " + "ten datasets it serves.", + DeprecationWarning, + # 2: the line that imported this module, not this module's own import + # machinery -- an import has no deeper user frame to point at. + stacklevel=2, ) - -#: Temporal resolutions: monthly, annual calendar year, annual water year. -TIME_RESOLUTIONS = ("monthly", "annualcy", "annualwy") - -#: This service's preferred in-flight cap when ``API_USGS_CONCURRENT`` is -#: unset. Lower than the package default of 32 because every location retries -#: independently, so a rate-limit episode bursts this number times the retry -#: count; the NWDC tolerates this level without rate-limit errors (verified by -#: stress test) and higher has not been tested. Setting ``API_USGS_CONCURRENT`` -#: overrides it -- see :func:`dataretrieval.transport.fanout._resolve_concurrency` -#: for why the general setting outranks a module's default rather than the -#: reverse. -DEFAULT_CONCURRENT_REQUESTS = 4 - -# Page responses carry the HUC12 identifier in this column; it must stay a -# string so leading zeros (e.g. "010900020502") survive the round trip. -_HUC12_COLUMN = "huc12_id" - - -def get_wateruse( - model: str, - variable: str | Iterable[str] | None = None, - state: str | int | Iterable[str | int] | None = None, - county: str | Iterable[str] | None = None, - huc: str | Iterable[str] | None = None, - time_resolution: str | None = None, - start_date: str | None = None, - end_date: str | None = None, - intersection: str = "overlap", - limit: int = 600, - ssl_check: bool = True, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get USGS water-use data from the NWDC web service. - - Retrieves modeled water-use estimates from the USGS National Water - Availability Assessment Data Companion. The area is given as exactly one of - ``state``, ``county``, or ``huc``; results are always returned on a HUC12 - grid, in a long (tidy) frame with one row per HUC12 and time step. Large - areas (e.g. a whole region or a populous state) are served across multiple - pages; this function follows those pages transparently and concatenates - them into one frame. - - Each selector also accepts a list of values. The NWDC queries one area per - request, so a list is fanned out into one request per value — up to - ``API_USGS_CONCURRENT`` in parallel, defaulting to - :data:`DEFAULT_CONCURRENT_REQUESTS` for this service — and the results are - concatenated in the order given. A fan-out interrupted by a rate limit or an - upstream fault raises a resumable - :class:`~dataretrieval.interruptions.FanOutInterrupted`, whose - ``.call.resume()`` re-issues only the locations that did not complete. - - Parameters - ---------- - model : string - Water-use category to query. See :data:`MODELS` for the available - options (e.g. ``"wu-public-supply-wd"``). The full catalog of models - and their variables is at https://water.usgs.gov/nwaa-data/. - variable : string or iterable of strings, optional - One or more variable IDs within ``model`` (e.g. ``"pswdtot"`` for total - public-supply withdrawals, or ``["pswdgw", "pswdsw"]`` for the - groundwater and surface-water components). Multiple variables are - comma-joined into a single request. The service requires at least one - variable; omitting it returns a 400 listing the model's valid variable - IDs (surfaced as a :class:`~dataretrieval.exceptions.DataRetrievalError`). - state : string, int, or iterable, optional - One or more US states/territories to query. Each accepts a full name - (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit - ANSI/FIPS code (``"55"`` or ``55``), mirroring - :func:`dataretrieval.ngwmn.get_sites`. - county : string or iterable, optional - One or more five-digit county FIPS codes — state FIPS + county FIPS, - e.g. ``"55025"`` for Dane County, Wisconsin. - huc : string or iterable, optional - One or more hydrologic unit codes. Each code's level is taken from its - length: a 2-digit code queries a HUC2 region, 8-digit a HUC8 subbasin, - 12-digit a single HUC12, and so on (even lengths 2-12, e.g. ``"04"``, - ``"07070005"``, ``"010900020502"``). - - Provide exactly one of ``state``, ``county``, or ``huc`` (each may be a - single value or a list). - time_resolution : string, optional - Temporal resolution: ``"monthly"``, ``"annualcy"`` (annual, calendar - year), or ``"annualwy"`` (annual, water year). See - :data:`TIME_RESOLUTIONS`. - start_date : string, optional - Start of the query window, formatted ``"YYYY"`` for annual data or - ``"YYYY-MM"`` for monthly data. - end_date : string, optional - End of the query window, in the same format as ``start_date``. - intersection : string, optional - How to select HUC12s that straddle the queried-area boundary: - ``"overlap"`` (any overlap, the default) or ``"envelop"`` (fully - enclosed). - limit : int, optional - Maximum number of HUC12s returned per page. Queries spanning more than - ``limit`` HUC12s are split across pages and reassembled. Default 600. - ssl_check : bool, optional - If True (default), verify SSL certificates; set False to skip - verification (e.g. behind a TLS-intercepting proxy). - - Returns - ------- - df : ``pandas.DataFrame`` - Water-use estimates in long form: a ``huc12_id`` column (string, - leading zeros preserved), a time column (``year_month`` for monthly - data or ``year`` for annual data), and one value column per requested - variable (suffixed with its unit, e.g. ``pswdtot_mgd`` for million - gallons per day). - md : :class:`dataretrieval.utils.BaseMetadata` - Metadata describing the request (URL, query time, response headers). - - Raises - ------ - ValueError - If not exactly one of ``state``, ``county``, or ``huc`` is given, or a - given selector is malformed (an unrecognized state, a county code that - is not five digits, or a HUC of invalid length). - DataRetrievalError - On an HTTP error response, the typed subclass for the status (see - :func:`dataretrieval.exceptions.error_for_status`). A transient 429, - 5xx, or recoverable connection failure that exhausts inline retries is - raised as a resumable - :class:`~dataretrieval.interruptions.FanOutInterrupted`; a deterministic - connection failure (for example, a permanently unresolvable host) - remains a :class:`~dataretrieval.exceptions.NetworkError`. - - Examples - -------- - .. doctest:: - :skipif: True # network - - >>> from dataretrieval import wateruse - >>> df, md = wateruse.get_wateruse( - ... model="wu-public-supply-wd", - ... variable=["pswdtot", "pswdgw", "pswdsw"], - ... state="RI", - ... start_date="2020-01", - ... time_resolution="monthly", - ... ) - - """ - # The public parameters are idiomatic snake_case (consistent with - # ``waterdata.get_samples``); the NWDC service expects compact lowercase - # query names, so map to those here as the request is built. - base_params: dict[str, Any] = { - "format": "csv", - "model": model, - "variable": to_str(variable), - "timeres": time_resolution, - "startdate": start_date, - "enddate": end_date, - "intersection": intersection, - "limit": limit, - } - # Drop params the caller left unset; the service rejects empty values. - base_params = {k: v for k, v in base_params.items() if v is not None} - - # The NWDC queries one location per request, so fan a multi-value selector - # out into one request per location, each handled by shared transport - # pagination, and concatenate the results. - headers = default_headers(WATERUSE_URL) - requests = [ - httpx.Request( - "GET", - WATERUSE_URL, - params={**base_params, "location": location}, - headers=headers, - ) - for location in _resolve_locations(state, county, huc) - ] - return _fan_out(requests, headers, ssl_check) - - -# Valid HUC code lengths (digits) → the hydrologic-unit level they query. -_HUC_LENGTHS = (2, 4, 6, 8, 10, 12) - -# Maps each selector to the NWDC ``location=:`` value(s) it produces. -# A value may be a single code or a list; ``_as_list`` normalizes both (``state`` -# additionally normalizes to the two-letter postal code, and ``to_state`` may -# itself return a scalar or list, which ``_as_list`` flattens the same way). -# Since NWDC takes one location per request, a list value fans out — one request -# per location (see :func:`_fan_out`). -_LOCATION_BUILDERS: dict[str, Callable[[Any], list[str]]] = { - "state": lambda v: [f"stateCd:{c}" for c in _as_list(to_state(v, to="postal"))], - "county": lambda v: [f"countyCd:{_validate_county(c)}" for c in _as_list(v)], - "huc": lambda v: [f"huc{len(c)}:{c}" for c in map(_validate_huc, _as_list(v))], -} - - -def _resolve_locations( - state: str | int | Iterable[str | int] | None, - county: str | Iterable[str] | None, - huc: str | Iterable[str] | None, -) -> list[str]: - """Build the NWDC ``location=:`` value(s) from the selectors. - - Exactly one of ``state`` / ``county`` / ``huc`` must be given; each may be a - single value or a list. ``state`` is normalized to the two-letter postal - code ``stateCd`` requires; ``county`` is a five-digit FIPS code; and a - ``huc`` code's length selects its level (``huc2`` … ``huc12``). Returns one - location string per value — the caller issues one request per location. - """ - selected = { - name: value - for name, value in (("state", state), ("county", county), ("huc", huc)) - if value is not None - } - if len(selected) != 1: - raise ValueError( - "Specify exactly one of state, county, or huc " - f"(got: {', '.join(selected) or 'none'})." - ) - [(name, value)] = selected.items() - locations = _LOCATION_BUILDERS[name](value) - if not locations: - raise ValueError( - "The chosen location selector is empty; pass at least one value." - ) - return locations - - -def _as_list(value: object) -> list[Any]: - """Normalize a value to a list. - - A scalar becomes a one-element list; any non-string iterable (list, tuple, - Series, ndarray, generator) is materialized to a list. A string is treated - as a scalar so it isn't exploded into characters. - """ - if isinstance(value, Iterable) and not isinstance(value, str): - return list(value) - return [value] - - -def _validate_county(value: object) -> str: - """Validate and normalize a five-digit state+county FIPS code.""" - code = str(value).strip() - if not (code.isdigit() and len(code) == 5): - raise ValueError( - "county must be a five-digit state+county FIPS code " - f"(e.g. '55025'), got {value!r}." - ) - return code - - -def _validate_huc(value: object) -> str: - """Validate a HUC code (even length 2-12 digits; level set by length).""" - code = str(value).strip() - if not (code.isdigit() and len(code) in _HUC_LENGTHS): - raise ValueError( - "huc must be a hydrologic unit code of even length 2-12 digits " - f"(e.g. '04', '07070005', '010900020502'), got {value!r}." - ) - return code - - -def _fan_out( - requests: list[httpx.Request], headers: dict[str, str], ssl_check: bool -) -> tuple[pd.DataFrame, BaseMetadata]: - """Fetch every request (each paginated) over the shared fan-out executor. - - Each request is paginated by :func:`dataretrieval.transport.pagination.paginate` - with NWDC strategies: parse a CSV page and read its ``Link`` header cursor - (``parse``), follow that cursor (``follow``), and raise the typed error - carrying the NWDC ``detail`` (``raise_for_status``). - - Everything else -- bounded concurrency, per-attempt retry, failure - precedence, progress, and resumable interruption -- belongs to - :class:`~dataretrieval.transport.fanout.FanOut`, which Water Data and NGWMN - drive too. This function is now only the NWDC-specific half: what a - chunk is, and how to read one. - - The plan is the request list itself. ``FanOut`` asks a plan only to be - sized and iterable, and the NWDC accepts one ``location=`` per request, so - the caller's locations arrive already separate -- there is nothing to - divide and so nothing for a plan class to hold. - - The broad retry status set is on purpose: NWDC reports a bad query as a 400 - with a ``{"detail": ...}`` envelope, so unlike WQP and StreamStats its 5xx - really is an upstream fault worth re-sending. - """ - - def parse(response: httpx.Response) -> tuple[pd.DataFrame, str | None]: - return _read_csv_page(response), _next_page_url(response) - - async def follow(cursor: str, sess: httpx.AsyncClient) -> httpx.Response: - return await sess.get(cursor, headers=headers) - - def raise_for_status(response: httpx.Response) -> None: - _raise_for_status(response, detail_from=_nwdc_error_detail) - - async def fetch(request: httpx.Request) -> tuple[pd.DataFrame, httpx.Response]: - """One location's full page walk, over the executor's shared client. - - ``active_client()`` is the client :meth:`FanOut._run` published for this - run; borrowing it keeps every location's pages on one connection pool - instead of opening a client per location. - """ - try: - return await paginate( - request, - parse_response=parse, - follow_up=follow, - client=active_client(), - raise_for_status=raise_for_status, - ) - except httpx.TransportError as exc: - raise network_error(request.url, exc) from exc - - def finalize( - frame: pd.DataFrame, response: httpx.Response - ) -> tuple[pd.DataFrame, BaseMetadata]: - return frame, BaseMetadata(response) - - return FanOut( - requests, - fetch, - RetryPolicy.from_env(), - finalize=finalize, - client_options={"verify": ssl_check}, - default_concurrent=DEFAULT_CONCURRENT_REQUESTS, - # No single URL expresses "all of these locations" -- the service - # has no such request -- so the aggregate reports the first, - # matching what an un-fanned single-location call would show. - canonical_url=str(requests[0].url) if requests else None, - # Labels the progress line the executor opens for this drive. - service="wateruse", - ).resume() - - -def _read_csv_page(response: httpx.Response) -> pd.DataFrame: - """Parse one CSV page; ``huc12_id`` stays a string to keep leading zeros.""" - try: - return pd.read_csv(io.BytesIO(response.content), dtype={_HUC12_COLUMN: str}) - except pd.errors.EmptyDataError as exc: - # NWDC normally signals "no data" with a 400 (handled above) or rows of - # zeros, never an empty body — but keep the typed-error contract if it - # ever returns one rather than leaking a bare pandas exception. - raise DataRetrievalError( - f"NWDC returned an empty response body (URL: {response.url})." - ) from exc - - -def _next_page_url(response: httpx.Response) -> str | None: - """Return the absolute URL of the next page, or None if this is the last. - - Reads the standard ``Link: <...>; rel="next"`` header (parsed by httpx into - ``response.links``). The cursor is normalized before it is trusted, because - the service spells it inconsistently. A relative reference is resolved - against the page it came from, and the bare ``water.usgs.gov`` host is - rewritten to the public ``api.water.usgs.gov`` gateway (over https, whatever - scheme the link used) so the follow-up request reaches the API. Only a - cursor that still points somewhere else after that is refused -- following - it would send Water Use requests, and any credentials on them, to a host the - caller never asked for. - """ - url = response.links.get("next", {}).get("url") - if not url: - return None - return resolve_next_url( - url, - response, - service="Water Use", - allowed_hosts=_WATERUSE_HOST_ALIASES, - rewrite_host=_WATERUSE_HOST, - ) - - -def _nwdc_error_detail(response: httpx.Response) -> str | None: - """Pull the ``detail`` message out of an NWDC JSON error envelope, if any. - - The NWDC reports errors as ``{"detail": "Invalid model name: ..."}``. Passed - to :func:`~dataretrieval.utils._raise_for_status` as ``detail_from`` so the - service's wording surfaces in the typed error message. - """ - try: - body = response.json() - except ValueError: - return None - return body.get("detail") if isinstance(body, dict) else None diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 2a41fd96..fd05e5b2 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -12,16 +12,25 @@ import warnings from io import StringIO -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import pandas as pd +from dataretrieval import settings as _settings from dataretrieval._response_metadata import BaseMetadata +from dataretrieval.credentials import refuse_credential_keywords +from dataretrieval.settings import ( + AdapterSettings, + _Redirectable, + _register, + _Retrying, +) from ._querying import _query_with_retry from ._wqx import _attach_datetime_columns __all__ = [ + "WqpSettings", "get_results", "what_sites", "what_organizations", @@ -42,6 +51,11 @@ from pandas import DataFrame +#: Root the Water Quality Portal serves both its interfaces from. Private +#: because the two builders below are the documented way to name a WQP URL; +#: this is only the piece they share, and the piece a redirect replaces. +_WQP_BASE_URL = "https://www.waterqualitydata.us" + result_profiles_wqx3 = ["basicPhysChem", "fullPhysChem", "narrow"] result_profiles_legacy = ["biological", "narrowResult", "resultPhysChem"] activity_profiles_legacy = ["activityAll"] @@ -199,7 +213,9 @@ def get_results( if legacy is not True and profile is None: kwargs["dataProfile"] = "fullPhysChem" - response = _query_with_retry(url, kwargs, delimiter=";", ssl_check=ssl_check) + response = _query_with_retry( + url, kwargs, delimiter=";", ssl_check=ssl_check, adapter="wqp" + ) df = _read_wqp_csv(response.text) df = _attach_datetime_columns(df) @@ -229,7 +245,7 @@ def _what( url = _legacy_only_url(service, legacy=legacy) response = _query_with_retry( - url, payload=kwargs, delimiter=";", ssl_check=ssl_check + url, payload=kwargs, delimiter=";", ssl_check=ssl_check, adapter="wqp" ) df = _read_wqp_csv(response.text) return df, WQP_Metadata(response, **kwargs) @@ -630,22 +646,33 @@ def _validate_service(service: str, valid_services: list[str], profile: str) -> ) +def _service_base() -> str: + """The WQP root this call targets: a block's redirect, or the portal's own. + + The portal serves the legacy and WQX3 interfaces from one root under + different paths, so a ``WqpSettings(base_url=...)`` names that root and + both follow it. Redirecting only the interface a caller happened to use + first would leave the other pointed at the service they were trying not to + talk to. Resolved per call, because a ``configure`` block is scoped to a + ``with`` statement. + """ + return _settings.base_url(adapter="wqp", default=_WQP_BASE_URL) + + def wqp_url(service: str) -> str: """Construct the WQP URL for a given service.""" - base_url = "https://www.waterqualitydata.us/data/" _warn_legacy_use() _validate_service(service, services_legacy, "Legacy") - return f"{base_url}{service}/Search?" + return f"{_service_base()}/data/{service}/Search?" def wqx3_url(service: str) -> str: """Construct the WQP URL for a given WQX 3.0 service.""" - base_url = "https://www.waterqualitydata.us/wqx3/" _warn_wqx3_use() _validate_service(service, services_wqx3, "WQX3.0") - return f"{base_url}{service}/search?" + return f"{_service_base()}/wqx3/{service}/search?" class WQP_Metadata(BaseMetadata): @@ -704,7 +731,18 @@ def site_info(self) -> tuple[DataFrame, WQP_Metadata] | None: def _check_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]: - """Check kwargs for unsupported parameters.""" + """Check kwargs for unsupported parameters. + + Every WQP getter's ``**kwargs`` funnels through here on its way to the + query payload, so this is the choke point where a credential-shaped name is + refused. The predicate is the credentials leaf's, shared with Water Data's + ``**queryables`` passthrough: ``api_key=`` is a plausible guess on any + getter now that ``configure(Settings(api_key=...))`` is the spelling, + and this is the adapter with the widest passthrough -- ten getters, whose + filter names the portal rather than this package defines. + """ + refuse_credential_keywords(kwargs) + mimetype = kwargs.get("mimeType") if mimetype == "geojson": raise NotImplementedError("GeoJSON not yet supported. Set 'mimeType=csv'.") @@ -759,3 +797,37 @@ def _legacy_only_url(service: str, legacy: bool) -> str: _warn_wqx3_unavailable() warnings.simplefilter("ignore", DeprecationWarning) return wqp_url(service) + + +class WqpSettings(_Redirectable, _Retrying, AdapterSettings): + """Settings for Water Quality Portal calls alone. + + No fan-out dials: a WQP query is answered by a single request, so a + concurrency cap could only report a number nothing honours. + + Lives here rather than in :mod:`dataretrieval.settings` because + *which* settings a service reads is the service's own knowledge (ADR + 0011); what each of them means is shared, so the fields come from the + setting groups declared beside their grammar. + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying + stops. + base_url : str, optional + Root to send WQP requests to, instead of the portal's own. Both + interfaces hang off it, so one value moves the legacy ``/data/`` + and the WQX3 ``/wqx3/`` paths together. Code only: the file and + the environment refuse it. + """ + + # One request per call, so this service reads the retry dials and a + # redirectable base and no fan-out dial. Each setting is declared once, + # in :mod:`dataretrieval.settings`, beside its grammar. + adapter: ClassVar[str] = "wqp" + + +_register(WqpSettings) diff --git a/demos/USGS_WaterUse_Examples.ipynb b/demos/USGS_WaterUse_Examples.ipynb index 0017f048..418457d6 100644 --- a/demos/USGS_WaterUse_Examples.ipynb +++ b/demos/USGS_WaterUse_Examples.ipynb @@ -14,7 +14,7 @@ "the modern replacement for the retired legacy NWIS water-use service.\n", "\n", "`dataretrieval` exposes the service through a single function,\n", - "`wateruse.get_wateruse`, which returns a tidy `pandas.DataFrame` plus a\n", + "`nwdc.get_wateruse`, which returns a tidy `pandas.DataFrame` plus a\n", "metadata object. Available **models** (categories) include:\n", "\n", "| model | description |\n", @@ -50,7 +50,7 @@ "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "\n", - "from dataretrieval import wateruse" + "from dataretrieval import nwdc" ] }, { @@ -74,7 +74,7 @@ "metadata": {}, "outputs": [], "source": [ - "df, md = wateruse.get_wateruse(\n", + "df, md = nwdc.get_wateruse(\n", " model=\"wu-public-supply-wd\",\n", " variable=[\"pswdtot\", \"pswdgw\", \"pswdsw\"],\n", " state=\"WI\",\n", diff --git a/docs/source/architecture/decisions/0008-fan-out-execution.rst b/docs/source/architecture/decisions/0008-fan-out-execution.rst index d5431f1e..44b324fc 100644 --- a/docs/source/architecture/decisions/0008-fan-out-execution.rst +++ b/docs/source/architecture/decisions/0008-fan-out-execution.rst @@ -130,8 +130,8 @@ Consequences Compliance ---------- -``tests/architecture_test.py`` asserts three things. That ``wateruse`` -contains no ``asyncio.gather``, ``Semaphore``, or ``TaskGroup``, so the +``tests/architecture_test.py`` asserts three things. That ``nwdc`` +(named ``wateruse`` when this decision was taken) contains no ``asyncio.gather``, ``Semaphore``, or ``TaskGroup``, so the duplication cannot return. That both plan types are sized and *repeatably* iterable -- resume keys completed work by position, so a generator mistaken for a collection would re-issue the wrong chunks. And that an interruption diff --git a/docs/source/architecture/decisions/0009-layered-configuration.rst b/docs/source/architecture/decisions/0009-layered-configuration.rst new file mode 100644 index 00000000..df1bb711 --- /dev/null +++ b/docs/source/architecture/decisions/0009-layered-configuration.rst @@ -0,0 +1,209 @@ +ADR 0009: Layered configuration resolution +========================================== + +Status +------ + +Accepted, with clauses superseded three times. + +:doc:`0012-pydantic-settings` supersedes the *third-party* half of the +"``dataretrieval.settings`` is a lightweight leaf" clause below, and renames +the vocabulary this ADR introduced -- ``Configuration`` is ``Settings``, +``show_configuration()`` is ``show_settings()``, and the module is +``dataretrieval.settings``. The chain itself is unchanged; it is now expressed +as an ordered tuple of ``PydanticBaseSettingsSource`` implementations rather +than as branches of a hand-written ``_resolve``. The clause's *first-party* +half -- that the module imports no adapter and nothing but the exceptions leaf +-- stands, and is still asserted. + +The prose below has been re-spelled in the new vocabulary, so a reader arriving +from a cross-reference does not have to translate; the decisions themselves are +as they were taken. Where it still says "configuration" as an English word +rather than as a class name, that is deliberate. + +:doc:`0010-adapter-scoped-settings` supersedes "One flat set of setting names" +and "Per-service overrides are deferred" below, having found the premise of the +first -- that every service accepts the same settings -- to be false. + +:doc:`0011-configuration-profiles` supersedes three more: + +- **The** ``[profiles.]`` **table** in step 3 of the chain, and the + recommendation in "``parallel_chunks`` at the top level of the file warns" to + put the setting in one. A profile is now named under the adapter it + configures (``[.]``); the global table and + ``DATARETRIEVAL_PROFILE`` are retired, since a table that switched every + service at once could not carry per-service detail. +- **"The environment ranks above the file"**, inverted for -- and only for -- a + profile selected in code. Everything the caller did not name in code still + follows the rule as written here. +- **The refusal of a configuration object**, stated in the leaf clause ("a + scoped action, not a ``Settings`` dataclass") and in "A configuration + object would have no way to reach the call". ``configure()`` now takes + exactly such objects. The grounds were that an instance had no way to reach a + free function; the ``ContextVar`` this ADR established is that way, and ADR + 0010 had already narrowed the objection to a payload-shape preference. + +The chain itself, the ``ContextVar`` delivery, host-scoped credentials, and the +leaf constraint stand. + +Context +------- + +Settings reached the library through one mechanism: process-global environment +variables (``API_USGS_PAT``, ``API_USGS_CONCURRENT``, ``API_USGS_RETRIES``, +``API_USGS_PROGRESS``), each with its own hand-rolled parser at its point of +use. Nothing could report the effective configuration, and the grammars were +free to drift apart. + +That mechanism cannot express a per-call credential. An application holding +keys in a secret store, a notebook pulling for two accounts, or a server +handling concurrent users must assign to ``os.environ`` — which is +process-global, so it races across threads and tasks (issue #352). + +The obvious fix, an ``api_key=`` parameter on the public getters, is unsafe +here. Every Water Data getter ends in ``_get_args(locals())`` with a +``**queryables`` catch-all that forwards unrecognized keywords to the API as +query parameters. A credential parameter missed in one of ~20 signatures would +be serialized into a URL. The maintainers also object to an ``api_key=`` +parameter on the separate ground that it invites keys pasted into shared +scripts. + +Decision +-------- + +Every setting resolves through one ordered chain, owned by a new +``dataretrieval.settings`` module: + +1. An active ``dataretrieval.configure(...)`` block (a ``ContextVar``). +2. The setting's environment variable. +3. The configuration file: ``~/.dataretrieval/config.toml``, or the path in + ``DATARETRIEVAL_CONFIG``. Top-level keys are the defaults; a + ``[profiles.]`` table layers over them per setting when selected. +4. The built-in default. + +Supporting decisions: + +- **Precedence is per setting, not per source.** An environment that sets only + ``API_USGS_PAT`` leaves a file-provided ``concurrency`` in effect. A + *blank* environment variable does not count as set, so it cannot shadow the + file: container and CI tooling routinely materializes one. The exception is + ``progress``, where a blank ``API_USGS_PROGRESS`` has always meant "off" -- + so "does blank count as a value?" is a property of the setting + (``settings._BLANK_MEANS_SET``) rather than an extra tier in the chain. +- **The environment ranks above the file.** This follows the established + precedence used by `pip + `_ + and `AWS + `_, + supports deployment-time overrides without editing mounted files, and keeps + the pre-existing ``API_USGS_*`` interface authoritative. +- **Omitted and explicitly cleared values differ.** An omitted + ``configure()`` argument inherits from lower sources. Explicit ``None`` is a + scoped reset to built-in behavior, so a server can guarantee an anonymous + call rather than accidentally falling through to its process credential. +- **No public getter grows a credential parameter.** ``configure`` is the only + programmatic path, and a fitness function asserts no getter accepts + ``api_key`` / ``session`` / ``token``. The generic ``**queryables`` path also + rejects those names before request construction so they cannot enter a URL. +- **The module owns each setting's parser.** ``unbounded``, bounds, and + rejection messages live in one place. ``tomllib`` returns typed scalars, so + the file and Python API validate source-level types before normalized values + pass through the shared parsers. Legacy environment-only forms, including a + blank numeric value and arbitrary non-empty progress value, remain compatible + without making the new surfaces equally permissive. +- **TOML, read with** ``tomllib``. Stdlib from Python 3.11; the ``tomli`` + backport is a marker-scoped dependency that disappears when + ``requires-python`` moves to ``>=3.11``. YAML was rejected because PyYAML is + a dependency at every Python version and the settings are flat. +- **Not every setting gets an environment variable.** ``parallel_chunks`` + spends rate-limit quota, and ADR-adjacent documentation on + ``dataretrieval.parallel_chunks`` argues it must stay a deliberate choice. + It does not add a new exported process-global knob; the file and ``configure`` + block are its only sources, with a scoped block as the recommended use. +- **Names distinguish execution capacity from planning granularity.** + ``concurrency`` is the noun for the maximum in-flight subrequests and maps to + the established ``API_USGS_CONCURRENT`` variable. ``parallel_chunks`` asks + the planner for optional extra chunks; it does not promise that many requests + execute simultaneously. The name is retained because the context manager is + already public. ``parallelism`` and ``chunk_parallelism`` were rejected + because they would conflate this planning hint with ``concurrency``. +- **Settings errors are in the error taxonomy.** ``ConfigurationError`` is a + ``DataRetrievalError`` *and* a ``ValueError``. Settings resolves lazily + on the request path, so a broken file surfaces from inside whichever getter + runs first; ``except DataRetrievalError`` around a call has to catch it like + any other failure of that call, while the ``ValueError`` base keeps the + handlers that predate the file layer working. +- **``parallel_chunks`` at the top level of the file warns.** It is the one + setting that spends rate-limit quota, so a value left there applies to every + splittable query in every process that reads the file. A + ``[profiles.]`` table is opt-in per run, which is the shape this + setting wants; the top-level form still works but says so. +- **``dataretrieval.settings`` is a lightweight leaf.** It uses only the standard + library, the ``tomli`` backport on Python 3.10, and + ``dataretrieval.exceptions`` -- itself a dependency-free leaf, so this adds + no weight and cannot cycle. It is read by ``utils`` + (headers), ``ogc.chunking``, ``ogc.retry``, and ``ogc.progress``, so under ADR + 0003 it must import none of them. The public callable is named ``configure`` + rather than ``config`` so it does not shadow the module. It is a scoped + action, not a ``Settings`` dataclass: a value object would imply + snapshot, equality, serialization, and representation contracts while + risking disclosure of the API key through generated helpers. + +- **One flat set of setting names, shared by every service.** ``concurrency`` + means the same thing to every adapter, so the chain resolves one name rather + than one per service. Services differ in the *value* they want, not the + vocabulary, and that difference is expressed as a caller-supplied default: + ``wateruse`` passes its ``DEFAULT_CONCURRENT_REQUESTS`` of 4 to + ``configuration.concurrency()`` where the OGC getters take the package default of + 32, and the single-shot adapters pass ``_GATEWAY_STATUSES`` to + ``RetryPolicy.from_settings()`` because WQP and StreamStats report a rejected + query as a 500. A value resolved from the chain always outranks a caller + default -- a service able to override an explicit setting would make + ``concurrency=1`` a lie. + +- **Per-service overrides are deferred, not refused.** One ``configure()`` + block cannot currently ask for a gentler Water Use than Water Data. Every + known service difference is a default, which the caller already supplies, so + nothing needs it yet. If something does, the shape is a namespace inside this + chain -- a ``[wateruse]`` table beside the top-level keys, read as + ``configuration.concurrency(default, service=...)``. It costs a second dimension in + resolution, which ``show_settings()`` must then render as a matrix rather than + a list, and that cost should buy a real requirement before it is paid. + +- **A configuration object would have no way to reach the call.** The public + surface is free functions -- ``waterdata.get_daily(...)``, not a client with + methods. An instance would therefore arrive either as a parameter on every + getter, which is the threading the ``ContextVar`` exists to remove and which + the ``**queryables`` catch-all makes unsafe, or through a module-level + global, which restores the cross-thread and cross-task leakage this ADR + exists to end. A library entered through a constructed client can hold + settings on that client; one entered through free functions cannot, and the + scoped block follows from that. + +Consequences +------------ + +- A credential can be supplied per thread or per task without touching + ``os.environ``, which is what issue #352 asked for. +- Host scoping is unchanged and unconditional: a key from any source is sent + only to ``api.waterdata.usgs.gov`` and is stripped on cross-host redirects. +- ``show_settings()`` reports the effective value and provenance of each setting + without ever printing the key. +- Behavior is unchanged when no file exists and no block is active, so + existing environment-variable users are unaffected. +- A configuration file becomes a supported artifact whose format is now a + compatibility surface. +- The Python floor and the file format are coupled: raising + ``requires-python`` to ``>=3.11`` drops the ``tomli`` dependency with no + other change. + +Compliance +---------- + +``tests/architecture_test.py::test_config_is_a_standard_library_only_leaf`` +asserts the module imports nothing from ``dataretrieval`` other than the +``exceptions`` taxonomy leaf, and no third-party package other than the +``tomli`` backport. +``tests/settings_test.py`` covers the precedence chain, per-setting merging, +thread and asyncio isolation, host scoping for file-sourced keys, redaction in +``show_settings``, and rejection of credential parameters on public getters. diff --git a/docs/source/architecture/decisions/0010-adapter-scoped-settings.rst b/docs/source/architecture/decisions/0010-adapter-scoped-settings.rst new file mode 100644 index 00000000..4e12d251 --- /dev/null +++ b/docs/source/architecture/decisions/0010-adapter-scoped-settings.rst @@ -0,0 +1,293 @@ +ADR 0010: Adapter-scoped settings +================================= + +Status +------ + +Accepted, except for two clauses, and re-spelled by +:doc:`0012-pydantic-settings` -- which also settles decision 5's worry directly: +the schemas are pydantic models, so their annotations are checked rather than +decorative, and an adapter that drifted to ``retries: str | None`` now rejects +an integer at construction instead of type-checking clean and failing when a +value reaches the chain. + +Supersedes the "One flat set of setting names" and "Per-service overrides are +deferred" clauses of +:doc:`0009-layered-configuration`; the rest of ADR 0009 stands, subject to what +ADR 0011 supersedes there. + +:doc:`0011-configuration-profiles` supersedes decisions 5 and 8 below -- +adapter schemas held centrally as ``TypedDict``, and each adapter a named +keyword on ``configure()``. Each adapter now declares a ``AdapterSettings`` +subclass in the module that *reads* those settings, and ``configure()`` takes +instances of them positionally, which is what removes the adapter roster from +the call site. The spelling shown in decision 1 goes with decision 8. Decisions +2, 3, 4, 6 and 7 stand: the tiers, source-major precedence, package-wide +environment variables, the host-scoped key, and the adapter names. + +Context +------- + +ADR 0009 resolved every setting through one flat namespace, on the premise that +"a setting means the same thing to every service; services differ in the value +they want, not the vocabulary." Surveying the seven APIs this package retrieves +from shows the premise is false. The settings themselves differ: + +.. list-table:: + :header-rows: 1 + + * - Adapter + - Host / path + - Fan-out + - Retryable statuses + - ``ssl_check`` + * - ``waterdata`` + - ``api.waterdata.usgs.gov/ogcapi`` + - yes (OGC chunking) + - all 5xx + 429 + - yes, on 3 of its getters + * - ``ngwmn`` + - ``api.waterdata.usgs.gov/ngwmn/ogcapi`` + - yes (OGC chunking) + - all 5xx + 429 + - -- + * - ``nwdc`` + - ``api.water.usgs.gov/nwaa-data`` + - yes (fan-out) + - all 5xx + 429 + - yes + * - ``nldi`` + - ``api.water.usgs.gov/nldi/linked-data`` + - no + - gateway only + - no + * - ``wqp`` + - ``www.waterqualitydata.us`` + - no + - gateway only + - yes + * - ``streamstats`` + - ``streamstats.usgs.gov`` + - no + - gateway only + - no + * - ``nwis`` (deprecated, not an adapter key) + - ``waterservices.usgs.gov`` + - no + - fixed, no retry + - yes + +``concurrency`` and ``parallel_chunks`` are meaningless for the four +single-shot adapters -- there is nothing to fan out. ``ssl_check`` applies to +four adapters (``waterdata``, ``nwdc``, ``nwis``, ``wqp``) and is currently a +per-call keyword outside the chain entirely; it reaches ``httpx``'s ``verify``, +verified by spying on the client. A flat namespace accepts +``configure(streamstats={"parallel_chunks": 8})`` without complaint, which is +the typo class ADR 0009 exists to catch. + +The credential is a separate axis, and measurement settled it. Probing the live +APIs with and without a key: + +* NGWMN and Water Data are served from the *same host* + (``ngwmn.py`` derives its base URL from ``credentials.WATERDATA_BASE_URL``). +* Both return ``200`` with no key, and both return ``x-ratelimit-limit: 1000`` + with one. +* Alternating authenticated calls decrement a *single* counter + (997, 996, 996, 994, 993, 992), so the two adapters share one quota pool. +* Water Data's OpenAPI declares ``ApiKeyHeader``/``ApiKeyQuery``; NGWMN's + declares no security scheme at all across 34 paths -- yet the gateway meters + it regardless. Every response carries ``via: ... api-umbrella``. + +The key is therefore a credential of the **gateway fronting the host**, not of +either adapter. It cannot meaningfully vary per adapter: two keys against one +quota pool is not a state the gateway can be in. + +Decision +-------- + +Settings are scoped to the **adapter**, not the service, and not the host. + +1. **The configuration file gains one table per adapter**, beside the existing + top-level keys:: + + concurrency = 16 # every adapter + + [ngwmn] + concurrency = 4 # this adapter only + + ``configure()`` takes the same shape, so one block configures several + adapters at once:: + + with dataretrieval.configure(ngwmn={"concurrency": 4}, + wqp={"retries": 2}): + ... + + .. note:: + + The file table stands; the ``configure()`` spelling above is superseded + by :doc:`0011-configuration-profiles` along with decision 8. One block + still configures several adapters at once, now as + ``configure(NgwmnSettings(concurrency=4), + WqpSettings(retries=2))``. + +2. **The top-level tier survives.** An adapter table *overrides* it per key; it + does not replace it. Every setting still has a package-wide spelling, and + the shipped ``API_USGS_*`` variables are package-wide by construction. + ``retries`` and ``stall_timeout`` are additionally adapter-scopable, because + a service that answers slowly or refuses often warrants its own budget + without changing anyone else's. ``progress`` is not: it describes the + caller's terminal, and there is one progress line per call, so scoping it + per adapter could only produce a contradiction. + +3. **Precedence stays source-major.** Resolution walks block, then environment, + then file, as ADR 0009 defines; *within* each source an adapter-scoped value + outranks a top-level one. The environment therefore still outranks the file, + so a stale adapter table cannot quietly beat a variable exported for one run. + +4. **Adapter-scoped settings get no environment variables.** Every entry in + ``ENV_VARS`` stays package-wide, for the reason ``parallel_chunks`` already + has none: an exported variable is inherited by every subprocess and + invisible at the call site. Six adapters times four settings would be a + namespace nobody could hold in mind. + +5. **Each adapter's schema is a** ``TypedDict``. Its ``__annotations__`` *are* + the schema -- there is no second table to maintain, ``mypy --strict`` checks + literal dicts at call sites, and the file path validates against the same + annotations. A key an adapter does not accept raises ``ConfigurationError`` + at block entry, the way an unknown profile already does. + + *Superseded by* :doc:`0011-configuration-profiles`. The schema is now a + frozen dataclass owned by the adapter, for the same "the annotations are the + schema" reason -- what changed is where it lives. A ``TypedDict`` had to be + declared centrally to annotate a central keyword, which put a Water Data + setting's definition in a module that knows nothing about Water Data. + +6. **The API key stays host-scoped and is not an adapter setting.** + ``credentials`` keeps sole ownership of which host honors the key. There is + no ``[ngwmn] api_key``. + +7. **Adapters are keyed by their service's name**, matching the module: + ``waterdata``, ``ngwmn``, ``nwdc``, ``wqp``, ``nldi``, ``streamstats``. + The deprecated ``nwis`` is deliberately absent: its calls pin + ``max_retries=0``, so a ``[nwis]`` table could only be reported as live and + then ignored -- the failure this decision exists to prevent. + +8. **Each adapter is a named, typed parameter on** ``configure()``, annotated + with its own ``TypedDict``, so a type checker rejects a setting the adapter + does not read before the code runs. A ``**unknown`` catch-all remains, and + exists to turn a misspelled *setting* into a message naming the settings -- + ``configure(concurrancy=8)`` would otherwise be a bare ``TypeError``. + + *Superseded by* :doc:`0011-configuration-profiles`. ``configure()`` takes + configuration objects positionally instead, so the adapter is named by the + class rather than by a keyword. The type checking survives -- a setting an + adapter does not read is not a field of its class -- and the catch-all is + no longer needed for a misspelling, because + ``WaterdataSettings(concurrancy=8)`` is already a ``TypeError`` naming + the keyword that does not exist. What the change buys is that ``configure()`` + no longer enumerates the adapters at all: that enumeration was the roster + this ADR left spelled in four places. + +Consequences +------------ + +- **A caller can be gentle with one adapter without throttling the rest** -- + the requirement ADR 0009 deferred. Because NGWMN and Water Data share a quota + pool, throttling NGWMN now measurably preserves quota for Water Data. + +- **The schema stops being a separate mechanism.** Choosing ``TypedDict`` over + a hand-maintained table removes the failure mode where a new adapter setting + is added and the validation table is not, and over a dataclass per adapter it + keeps the payload a plain mapping, so the file and block paths share one + validator and ``configuration`` grows no runtime classes. *Superseded with + decision 5*: the classes exist, and live with their adapters rather than in + the leaf. + +- **A configuration object is still refused, but on narrower grounds than ADR + 0009 stated.** That ADR rejected an object because it had no way to *reach* + the call. A per-adapter payload type does not have that problem -- the + ``ContextVar`` remains the delivery mechanism and the type is only the + payload's shape. ``TypedDict`` is chosen over a dataclass for the reason + above, not because an object could not be delivered. + + *Withdrawn by* :doc:`0011-configuration-profiles`, which took the remaining + step. Narrowing the objection to a payload-shape preference is what left it + open, and a dataclass turned out to buy the thing a mapping could not: an + instance knows which adapter it targets, so the caller stops naming one and + the roster stops being duplicated. + +- **``show_settings()`` grows a second section, not a matrix.** It prints + the top-level tier as today, then only those adapter overrides actually set. + A seven-by-eight grid of mostly-inherited values would bury the answer to + "what will this call use". + +- **The shared quota pool is not modelled.** ``[waterdata]`` and ``[ngwmn]`` + read as independent dials but draw on one 1000/hour allowance. A host or + gateway tier would express it; that is deferred until someone is confused by + it, since the pool is a property of the credential, which is already + host-scoped. + +- **``stall_timeout`` joins the chain.** ``API_USGS_STALL_TIMEOUT`` was read + directly from ``os.environ``, so it could not be set by a block or the file + and never appeared in ``show_settings()`` -- a gap in ADR 0009's own + claim that every setting resolves through one chain. It is package-wide by + default and adapter-scopable. ``dataretrieval/transport/env.py`` existed only + to parse it and is deleted, so ``configuration`` is now the only module in + the package that reads ``os.environ`` for a setting. + +- **``ssl_check`` stays a per-call argument and does not become a setting.** + It is a defaulted keyword on 23 shipped getters across four adapters -- + ``wqp`` (9), ``nwis`` (10), ``waterdata`` (3) and ``nwdc`` (1) -- and it does + reach ``httpx``'s ``verify``. It was added in 2023 to what were then the only + modules; the OGC getters arrived later and never adopted it, so its + distribution records the package's history rather than a boundary. + + Three reasons not to promote it. It disables certificate verification, so as + a per-call keyword it is a visible, scoped decision, while a config-file key + or environment variable would make a security downgrade process-wide and + invisible at the call site -- the opposite of the direction this chain + narrows everything else. It does not respect adapter boundaries: within + ``waterdata`` it applies only to the getters that bypass the OGC engine, so + ``[waterdata] ssl_check`` would be honored by three getters and silently + ignored by the rest, exactly the shape this ADR refuses elsewhere. And the + need it serves is already met better: the legitimate case is a + TLS-intercepting corporate proxy, and ``httpx`` natively honors + ``SSL_CERT_FILE`` and ``SSL_CERT_DIR`` on both its sync and async clients -- + so that mechanism already covers *every* getter, including the OGC ones that + have no ``ssl_check``, and it trusts the corporate CA rather than trusting + nothing. The ``bool`` type cannot even carry a CA bundle path, which is the + value a caller actually wants. + + The settings guide documents ``SSL_CERT_FILE`` for that case. Whether + ``ssl_check`` should be deprecated outright is a public-API question left to + its own change. + +- ``tests/settings_test.py`` covers adapter-table resolution, top-level + inheritance per setting, source-major precedence (the environment still + outranks an adapter table), an adapter block outranking a package-wide one, + and rejection of a setting an adapter does not read -- from both the file and + ``configure()``. +- ``test_api_key_is_never_adapter_scoped`` asserts no adapter configuration + accepts ``api_key``. +- ``test_adapter_roster_names_real_modules_that_register_themselves`` imports + every name in the roster, so a renamed adapter cannot leave a configuration + pointing at nothing. +- ``lint-imports`` continues to place ``configuration`` between ``credentials`` + and ``exceptions``. + +The two entries covering decision 8's ``**adapters`` catch-all +(``test_a_misspelled_setting_is_not_taken_for_an_adapter``) and the central +``TypedDict`` registry (``test_adapter_schema_names_a_real_module``) went with +the clauses ADR 0011 superseded; the checks they stood for are named above in +their current form. + +Notes +----- + +- Supersedes two clauses of :doc:`0009-layered-configuration`; the chain, the + ``ContextVar`` delivery, and the leaf constraint are unchanged. +- Live-API measurements behind the credential decision were taken 2026-08-11 + against ``api.waterdata.usgs.gov`` and ``api.water.usgs.gov``. +- The ``wateruse`` module is renamed ``nwdc`` under separate cover; the service + names itself "National Water Availability Assessment Data Companion" and + serves ten models, only five of which are water use. diff --git a/docs/source/architecture/decisions/0011-configuration-profiles.rst b/docs/source/architecture/decisions/0011-configuration-profiles.rst new file mode 100644 index 00000000..6dcdad4d --- /dev/null +++ b/docs/source/architecture/decisions/0011-configuration-profiles.rst @@ -0,0 +1,250 @@ +ADR 0011: Settings profiles, scoped to one adapter +======================================================== + +Status +------ + +Accepted, and re-spelled by :doc:`0012-pydantic-settings`. Every decision below +stands; the classes named in them are pydantic-settings models rather than +frozen dataclasses, and carry the library's names -- +``Settings`` for ``Configuration``, ``AdapterSettings`` for +``BaseConfiguration``, ``validate_settings()`` for ``validate()``. Two details +follow from the move: the "annotations are the schema" argument in decision 5 +is now enforced rather than aspirational, and the ``_UNSET`` sentinel that +distinguished an omitted setting from an explicit ``None`` is replaced by +``model_fields_set``. + +Supersedes two clauses of :doc:`0010-adapter-scoped-settings` -- +decision 5 (adapter schemas held centrally as ``TypedDict``) and decision 8 +(each adapter a named keyword on ``configure``) -- and three of +:doc:`0009-layered-configuration`: the global ``[profiles.]`` table; the +environment-above-file rule, inverted for a profile selected in code; and the +refusal of a configuration object, which ADR 0010 had already narrowed to a +preference about the payload's shape. The chain, the ``ContextVar`` delivery, +host-scoped credentials and the leaf constraint stand. + +Context +------- + +ADR 0010 gave each adapter its own slice of the chain, so ``[ngwmn]`` narrows a +setting to NGWMN. That covers "tune one service" but not the case a +multi-service caller actually has: + +- **Several named configurations per adapter.** A caller with an overnight + bulk shape and a polite daytime shape for Water Data cannot store both. The + only named construct is ``[profiles.]``, which switches *every* + service at once. +- **Composing them.** The two mechanisms do not compose: + ``[profiles.bulk.ngwmn]`` raises, so a profile cannot carry per-service + detail. That refusal was recorded in ADR 0010 on the grounds that layering + them needed a fourth precedence rule nobody had asked for. Someone has now + asked for it, and it is the primary use case. + +Two further problems ADR 0010 left open feed into the same decision. The +adapter roster is spelled in four places, only one of which is derived -- +adding an adapter needs coordinated edits, and forgetting one leaves a schema +no call site can reach, which happened to three adapters and shipped +undetected until a fitness test was written. And a setting's definition lives +in ``config`` rather than in the module that reads it, so adding a Water Data +setting edits a file that knows nothing about Water Data. + +Decision +-------- + +**A configuration profile is a named set of settings for one adapter.** The +file gains named profiles beside each adapter's default profile:: + + concurrency = 16 # package-wide defaults + + [waterdata] # waterdata's DEFAULT profile: always active + concurrency = 32 + + [waterdata.bulk] # a NAMED profile: only when selected + parallel_chunks = 8 + + [ngwmn.gentle] + concurrency = 4 + +A named profile never enters the chain unless a caller selects it. The global +``[profiles.]`` table and ``DATARETRIEVAL_PROFILE`` are retired; nothing +has shipped, so nothing is deprecated. + +**``configure()`` takes configuration objects.** Positionally, one per +adapter, and nothing else:: + + with dataretrieval.configure( + Settings(api_key=vault.read("usgs/pat")), + WaterdataSettings.load("bulk"), + NgwmnSettings(concurrency=4), + ): + ... + +The adapter an instance targets is a property of its class, so the caller +never restates it -- which is what removes the roster duplication. Naming two +configurations for one adapter raises: they would be the one pairing with no +defined order. + +Keyword settings are removed, so ``configure(api_key=...)`` no longer works. +This is the most-typed line the feature exists to enable, and making it wordier +is a real cost, accepted deliberately for one shape everywhere. + +**Schemas live with their adapter; names live centrally.** ``configuration`` +is a standard-library-only leaf every adapter may import, so it cannot import +adapters. It holds the tuple of adapter *names*, which is what parsing a file +needs (is ``[ngwmn]`` a table or a typo?). Each adapter package owns its +subclass, which is what a setting's definition needs to be local to the +service that reads it. + +Registration at import alone would not do: ``dataretrieval`` imports six of +seven adapters eagerly, but NLDI is deliberately on demand for the geopandas +extra, so a registry built from imports would reject a valid ``[nldi]`` table +until something imported it, and the report would vary by what a caller had +touched. + +**Precedence**, highest first: + +1. A configuration instance passed to ``configure()`` +2. A profile selected in code, ``WaterdataSettings.load("bulk")`` +3. The setting's environment variable (package-wide settings only) +4. The adapter's default profile in the file +5. Package-wide defaults in the file +6. The adapter's built-in preference in code +7. The package built-in default + +Each level overrides the one below **per key**, so a named profile still +inherits its adapter's default profile and the package-wide keys. Positions 1 +and 2 are both code and both target one adapter, so the same-adapter rule +means they cannot tie. + +Position 2 above 3 inverts ADR 0009's environment-above-file rule for this one +case. A profile named in code is a more deliberate act than a variable +inherited from a shell, and losing to that variable is the behaviour a caller +would file a bug about. Everything the caller did *not* name in code still +follows the original rule. + +**Validation is lazy.** A file's structure is checked when it is parsed; a +table's keys are checked when that adapter first resolves a setting. This +keeps the blast-radius rule ADR 0010 established -- a malformed ``[nldi]`` +table must not fail a Water Data call -- and it is what allows the schema to +live in a module the parser cannot import. + +**Base URLs may be configured, from code only.** An adapter's configuration +may carry its base URL, settable in a ``configure()`` block and rejected from +the file and the environment. A file that silently redirects a data-retrieval +library to another host is a supply-chain-shaped hazard; an in-code block +keeps the redirect where a reader sees it. + +**The module is renamed** ``dataretrieval.config`` to +``dataretrieval.settings``, +and ADR 0009's rule reserving ``config`` as an abbreviation for the module and +the file is withdrawn. The path has never been released, so no alias is +needed. + +**Credentials are unchanged, and measurement settled why.** The API key stays +one package-wide setting scoped to the single host that honours it. Probing +the live services: + +.. list-table:: + :header-rows: 1 + + * - Host + - No key + - With key + - Bad key + * - ``api.waterdata.usgs.gov`` (waterdata, ngwmn) + - no limit header + - ``x-ratelimit-limit: 4000`` + - 403 + * - ``api.water.usgs.gov`` (nwdc) + - ``1000`` + - ``1000`` + - 403 + * - ``api.water.usgs.gov`` (nldi) + - ``3600`` + - ``3600`` + - 403 + +NWDC and NLDI meter by address and report the *same* limit with or without a +key; the gateway validates one only if present. Sending the key there would +gain nothing and would turn a stale key into 403s on calls that work +anonymously today. The three hosts also keep independent counters, so ADR +0010's "one key, one quota pool" is true of waterdata and ngwmn only. + +Consequences +------------ + +- **The multi-service case gets a spelling**, which is the point. One block, + several adapters, at most one configuration each, any of them from the file + or from code. +- **The roster stops being duplicated.** An adapter declares itself once. The + failure mode where a schema exists that nothing passes becomes impossible by + construction rather than caught by a fitness test. +- **A setting's definition moves next to the code that reads it.** Adding a + Water Data setting no longer edits a service-neutral module. +- **``configure(api_key=...)`` breaks.** The README, the settings guide, + the PR description and ADR 0009's examples all use it and all must change in + the same commit. +- **``show_settings()`` can only resolve the settings an adapter accepts + once that adapter has been imported.** It names the adapters it could not + check rather than omitting them silently, which is the honest cost of lazy + validation. The *profile list* is not import-limited: what a profile is + called is a fact about the file, so every ``[.]`` table it + defines is listed, imported or not -- withholding one would make the + section's answer depend on which optional extras happened to be installed. +- **Two names differ only by case** -- the ``configuration`` module and the + ``Settings`` class. The module stays out of the package's public + exports so the confusing import line cannot arise. +- **Separate quota pools are still not modelled.** Three exist. Nothing in the + library needs to know yet. +- **``ssl_check`` is unaffected** and remains a per-call argument, for the + reasons in ADR 0010. + +Compliance +---------- + +Satisfied. In ``tests/settings_test.py``: + +- ``test_several_named_profiles_are_selected_independently`` -- one block, + a different profile per adapter. +- ``test_a_named_profile_layers_per_key_over_the_tiers_below`` -- a profile + inherits its adapter's default profile and the package-wide keys per key. +- ``test_adding_a_named_profile_changes_nothing_until_it_is_selected`` -- a + named profile is inert until something selects it. +- ``test_two_configurations_for_one_adapter_raise``. +- ``test_a_code_selected_profile_outranks_the_environment``, plus a case per + rung of the seven-rung ladder above, each written against one file that + populates every rung with a distinct value. +- ``test_inner_block_can_lower_a_setting_an_outer_block_scoped`` -- the + innermost block wins, including over an adapter-scoped outer one. +- ``test_a_table_for_an_unimported_adapter_stays_valid`` and + ``test_a_malformed_table_does_not_fail_another_adapters_call`` -- the + blast-radius rule under lazy validation. +- ``test_base_url_applies_from_code_and_is_refused_from_the_file``, with + ``test_base_url_is_refused_from_the_environment`` for the other source, and + ``test_every_water_data_endpoint_use_goes_through_redirected`` -- an AST scan + over ``dataretrieval/waterdata`` for the one adapter that cannot resolve its + base at a single choke point, so a family module cannot quietly keep sending + traffic to the host a caller redirected away from. +- ``test_adapter_roster_names_real_modules_that_register_themselves`` and + ``test_every_adapter_is_actually_wired_to_a_read_site`` -- the roster + resolves, and no configuration exists that nothing reads. An adapter name + the code does not recognize now raises out of ``_resolve`` rather than + falling through to the package-wide value, so the grep is a backstop rather + than the only guard. + +``tests/architecture_test.py::test_config_is_a_standard_library_only_leaf`` +asserts the module imports no adapter -- ``dataretrieval.exceptions`` is its +only first-party import -- and ``lint-imports`` keeps ``configuration`` below +``credentials``. + +Notes +----- + +- Live measurements taken 2026-08-11 against ``api.waterdata.usgs.gov`` and + ``api.water.usgs.gov``. +- Open, not decided here: whether ``parallel_chunks`` is renamed. ``fan_out`` + was suggested and conflicts with the glossary, where fan-out is *executing* + chunks concurrently -- which ``concurrency`` already governs -- while + ``parallel_chunks`` asks the planner to *divide* more finely. ADR 0009 + rejected ``parallelism`` and ``chunk_parallelism`` for the same conflation. + ``chunk_count`` or ``target_chunks`` would stay on the correct side of it. diff --git a/docs/source/architecture/decisions/0012-pydantic-settings.rst b/docs/source/architecture/decisions/0012-pydantic-settings.rst new file mode 100644 index 00000000..d4d6a172 --- /dev/null +++ b/docs/source/architecture/decisions/0012-pydantic-settings.rst @@ -0,0 +1,166 @@ +ADR 0012: Settings resolution is built on pydantic-settings +============================================================ + +Status +------ + +Accepted. Supersedes the standard-library-only clause of +:doc:`0009-layered-configuration` ("``dataretrieval.configuration`` is a +lightweight leaf") in its *third-party* half only, and renames the vocabulary +that ADRs 0009 through 0011 established. The chain, the ``ContextVar`` +delivery, host-scoped credentials, adapter scoping and per-adapter profiles all +stand unchanged; what changes is who implements them. + +Context +------- + +ADR 0009 built the resolution chain from scratch: a hand-written per-setting +type check (``_coerce_typed``), a hand-written grammar per setting +(``_VALIDATORS`` and the ``_parse_*`` family), a hand-written merge across +tiers, and a frozen dataclass per adapter whose field annotations were, as ADR +0010 admitted, "decorative" -- an adapter that drifted to ``retries: str | None`` +would type-check clean under ``mypy --strict`` and fail only when a value +reached the chain. + +That is a settings library. One already exists, is widely deployed, and is +maintained by people whose job this is. + +Three candidates were considered. + +**dynaconf** was rejected on its two central abstractions. It is schemaless -- +settings are a dynamic object and ``Validator`` objects are runtime assertions, +not annotations -- so the per-adapter vocabularies ADR 0011 is built around +would go back to being a hand-maintained table, which is the failure mode that +shipped undetected for three adapters. And its *environments* switch every +service at once, which is exactly the global ``[profiles.]`` table ADR +0011 retired for being unable to carry per-service detail. + +**typed-settings** was the near miss. Its loader chain is arguably a cleaner +statement of a tiered resolution than ``settings_customise_sources``, and its +attrs/dataclass backend would have left the adapter classes as the frozen +dataclasses they already were. It was rejected because the custom work does not +shrink -- adapter tables, named profiles, the ``ContextVar`` tier, the +``base_url`` refusal and the environment's legacy grammars are custom loaders +either way -- while it is a substantially smaller project, has no +``extra="forbid"`` equivalent for rejecting unknown keys in an arbitrary TOML +table, and exposes no stable per-value provenance. Provenance is load-bearing +here: ``show_settings()`` exists to report it. + +Decision +-------- + +**Settings are pydantic-settings models.** Each adapter's class is a +``BaseSettings`` subclass; the shared setting groups are model mixins. The +annotations are now enforced rather than decorative, which is what ADR 0011 +wanted from them. + +**Each tier of the chain is a** ``PydanticBaseSettingsSource``. The four +sources -- the ``configure()`` block, the ``API_USGS_*`` variables, the +``[]`` table, the file's top-level keys -- are listed in ``_CHAIN``, +highest first, and resolution keeps the first value it sees for each key. ADR +0009's "precedence is per setting, not per source" is therefore an ordering +rather than a stack of hand-written fallbacks. + +**The vocabulary follows the library's.** ``Configuration`` is ``Settings``, +``BaseConfiguration`` is ``AdapterSettings``, ``Configuration`` is +``Settings``, ``show_configuration()`` is ``show_settings()``, and the +module ``dataretrieval.configuration`` is ``dataretrieval.settings``. The file +keeps the name ``config.toml`` and the variable keeps the name +``DATARETRIEVAL_CONFIG``: both are compatibility surfaces, and ``config`` is +the conventional name for a file on disk. + +The glossary follows: a *configuration profile* is a **settings profile**, and +*effective configuration* is the **effective settings**. ``configure()`` keeps +its name -- it is the verb, and pydantic-settings has no competing spelling. + +**It is a required dependency, not an extra.** Settings resolve on the request +path -- every call reads the API key -- so an optional dependency would mean +shipping a second, standard-library implementation of the same chain and +testing both. That is more hand-rolled settings code than this ADR exists to +delete. + +**Resolution does not go through** ``BaseSettings.__init__``. This is the one +place the library's shape is set aside, and it is a cost decision rather than a +design one. ``BaseSettings.__init__`` builds the four stock sources on every +instantiation -- two of which snapshot and case-fold the whole of ``os.environ`` +-- before ``settings_customise_sources`` can discard them. That is the right +trade for a settings object built once at start-up and the wrong one for a +package that resolves lazily per read: profiling put 74% of a single read +inside ``_settings_init_sources``. So ``_resolved`` walks ``_CHAIN`` itself and +validates the merged mapping, and ``__init__`` validates the caller's keywords +directly. The sources, their order, the field schema, ``extra="forbid"`` and +``model_post_init`` are all still the library's. + +**Two behaviors change**, both narrowing an inconsistency rather than adding +one: + +- A setting an adapter does not read, or a misspelled one, raises + ``ConfigurationError`` from ``extra="forbid"`` rather than the dataclass's + bare ``TypeError``. The same mistake written into the file has always raised + ``ConfigurationError``, so the two surfaces now agree, and the message lists + the settings the adapter *does* accept. +- The "unset" sentinel is gone. ``_UNSET`` existed to distinguish an omitted + setting from an explicit ``None``; ``model_fields_set`` is pydantic's record + of exactly that, so every field simply defaults to ``None``. + +Consequences +------------ + +- **Roughly half the module is deleted.** ``_coerce_typed``, ``_validated_raw``, + the ``_UNSET`` sentinel, the memoized ``_settings_of``, and the hand-written + merge in ``_resolve`` all go. What remains is what pydantic-settings has no + opinion about: the TOML grammar of adapter tables and named profiles, the file + cache, the provenance labels, and the ``ContextVar``. + +- **A read costs about twice as much.** Measured on one Windows machine, per + read with no settings file: 26--37 us before, 68--82 us after. With a settings + file both are dominated by the file open, which each re-reads on Windows for + the reason ADR 0009 gives, and which on-access virus scanning inflates to + 0.8--1.1 ms and 1.3--2.5 ms respectively. At the eight reads a one-chunk query + performs, that is ~0.3 ms against ~0.6 ms, on a 100--500 ms round trip: a + fraction of a percent either way. + +- **pydantic enters the dependency tree**, at about 9 MB installed -- 5.6 MB of + it the compiled ``pydantic_core`` -- against the ~100 MB of pandas and numpy + the package already requires. It is also the most widely installed of the + three candidates, so for many users it is already present. + +- **The settings module is no longer standard-library-only.** It is still a + leaf: it imports no adapter and nothing first-party but + ``dataretrieval.exceptions``, so it cannot cycle, and the fitness function + still asserts that. What it no longer asserts is the absence of third-party + imports; it pins an allowed roster instead, so a leaf that quietly grew + ``httpx`` or ``pandas`` still fails. + +- **Two names still differ only by case** -- the ``settings`` module and the + ``Settings`` class -- as they did for ``configuration``/``Configuration``. The + module stays out of the package's public exports for the same reason, so the + confusing import line cannot arise. + +- **A validation hook had to be renamed.** ``BaseConfiguration.validate()`` + is ``AdapterSettings.validate_settings()``: pydantic's ``BaseModel`` already + owns ``validate``. + +Compliance +---------- + +``tests/architecture_test.py::test_settings_is_a_first_party_leaf`` asserts the +module imports nothing first-party but the taxonomy leaf, and no third-party +package outside the settings stack. + +``tests/settings_test.py`` is ADR 0009--0011's suite, carried over intact: the +144 cases covering the precedence ladder, per-setting merging, thread and +asyncio isolation, named profiles, host scoping, redaction, lazy validation and +the ``base_url`` refusal all pass unchanged against the new implementation, +which is what "the behavior is the same, the implementation is not" means here. + +Notes +----- + +- Benchmarks taken 2026-08-12 on Windows 11 / CPython 3.12, comparing this + branch against PR #353 in isolated environments. Absolute figures are + machine-specific; the ratio is the point. +- Open, not decided here: raising ``requires-python`` to ``>=3.11`` would drop + the ``tomli`` backport and let ``typing.Self`` replace the ``_S`` TypeVar. + Both are consequences of the floor, not of this ADR, so they belong to + whichever change moves it. diff --git a/docs/source/architecture/decisions/index.rst b/docs/source/architecture/decisions/index.rst index fb131528..8b02bb10 100644 --- a/docs/source/architecture/decisions/index.rst +++ b/docs/source/architecture/decisions/index.rst @@ -25,4 +25,8 @@ records sequentially. 0006-service-neutral-transport 0007-adapter-facades 0008-fan-out-execution + 0009-layered-configuration + 0010-adapter-scoped-settings + 0011-configuration-profiles + 0012-pydantic-settings template diff --git a/docs/source/architecture/index.rst b/docs/source/architecture/index.rst index 72fa1e57..fa88b717 100644 --- a/docs/source/architecture/index.rst +++ b/docs/source/architecture/index.rst @@ -80,7 +80,7 @@ Public service facades configures with an NGWMN-specific base URL, output identifiers, state translation, and :class:`OgcDialect`. -``dataretrieval.wateruse`` +``dataretrieval.nwdc`` NWDC Water Use facade. Builds CSV requests, follows ``Link`` headers, and uses service-neutral transport for bounded fan-out, retry, pagination, response aggregation, and synchronous dispatch. It does not depend on OGC modules. @@ -96,6 +96,16 @@ Public service facades Shared components ^^^^^^^^^^^^^^^^^ +``dataretrieval.settings`` + Settings leaf, built on ``pydantic-settings`` (ADR 0012): it imports no + adapter and nothing first-party but the exceptions taxonomy, so any module + may depend on it without an import cycle. It resolves scoped overrides, + environment variables, a TOML file with optional profiles, and built-in + defaults in + that order. Service and protocol modules may depend on it; it must not + depend back on them. Scoped overrides use ``ContextVar`` so concurrent + threads and asyncio tasks can carry distinct credentials. + ``dataretrieval.ogc`` Protocol subsystem for Water Data and NGWMN. A small facade (``__init__.py``) exposes the service-adapter seam: ``OgcDialect``, @@ -161,7 +171,7 @@ Shared components ``dataretrieval._querying`` The one-shot HTTP query path the single-request adapters (``nwis``, - ``wqp``, ``nldi``, ``streamstats``, ``wateruse``) use: compose the URL, send + ``wqp``, ``nldi``, ``streamstats``, ``nwdc``) use: compose the URL, send it, map the status, retry a transient. It left ``utils`` because the two halves shared only a filename -- this one depends on ``exceptions`` and ``transport``, the shaping half on ``codes`` and pandas, and no caller @@ -268,17 +278,19 @@ used by Water Use because the NWDC accepts only one location per request. Resource and configuration view ------------------------------- -``API_USGS_PAT`` - Optional USGS API token. It is attached only to requests for - ``api.waterdata.usgs.gov``. Shared synchronous and asynchronous clients - re-check every redirected request and strip the token before following a - link to any other host, including external rating assets. +Every setting resolves per key through an active ``configure()`` block, its +environment variable when one exists, the adapter's own table and the top-level +values in ``~/.dataretrieval/config.toml``, then its built-in default. +``configure()`` takes settings profiles -- a package-wide ``Settings`` +and at most one per adapter -- and each adapter's class is defined in the module +that reads those settings, so ``settings`` stays a leaf holding only the +adapter roster. ``show_settings()`` reports the effective source while +redacting credentials. -``API_USGS_CONCURRENT`` - Fan-out concurrency cap; defaults to 32 for OGC and 4 for Water Use when - unset. An explicit value applies to every service, ``1`` is sequential, and - ``unbounded`` removes the explicit cap. A semaphore, not pool waiting, is - the execution throttle. +The settings themselves -- names, defaults, environment variables, and the +config-file format -- are catalogued once in the +:doc:`settings guide `. What matters +architecturally is the behavior around them: ``API_USGS_RETRIES`` Number of retries after the first attempt on supported active request paths; @@ -305,6 +317,18 @@ Resource and configuration view ``API_USGS_PROGRESS`` Controls best-effort progress display. Reporting failures must never change retrieval results. +* The API token is attached only to requests for ``api.waterdata.usgs.gov``. + Shared synchronous and asynchronous clients re-check every redirected request + and strip the token before following a link to any other host, including + external rating assets. +* A semaphore, not connection-pool waiting, is the execution throttle for + sub-request concurrency. +* Retry backoff is exponential with full jitter and honors bounded + ``Retry-After`` values. +* Progress reporting is best-effort: a reporting failure must never change + retrieval results. +* ``dataretrieval.settings`` imports no adapter and nothing first-party but the + exceptions taxonomy, so any module may depend on it without an import cycle. ``dataretrieval.transport`` centralizes HTTP timeout, redirect, and authentication policy. OGC chunk fan-out and Water Use location diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst index 959d2675..23905cca 100644 --- a/docs/source/reference/index.rst +++ b/docs/source/reference/index.rst @@ -7,6 +7,7 @@ API reference .. toctree:: :maxdepth: 1 + settings exceptions ngwmn nldi @@ -14,5 +15,5 @@ API reference streamstats utils waterdata - wateruse + nwdc wqp diff --git a/docs/source/reference/nwdc.rst b/docs/source/reference/nwdc.rst new file mode 100644 index 00000000..52087b4f --- /dev/null +++ b/docs/source/reference/nwdc.rst @@ -0,0 +1,12 @@ +.. _nwdc: +.. _wateruse: + +dataretrieval.nwdc +------------------ + +The National Water Availability Assessment Data Companion. Water use is one of +the datasets it serves; the module was named ``wateruse`` until that became +misleading, and the old name remains as a deprecated alias. + +.. automodule:: dataretrieval.nwdc + :members: diff --git a/docs/source/reference/settings.rst b/docs/source/reference/settings.rst new file mode 100644 index 00000000..c2ead496 --- /dev/null +++ b/docs/source/reference/settings.rst @@ -0,0 +1,18 @@ +.. _config: + +dataretrieval.settings +--------------------------- + +Layered configuration: a ``dataretrieval.configure(...)`` block holding one +configuration per adapter, then the ``API_USGS_*`` environment variables, then +the adapter's table and the package-wide keys in +``~/.dataretrieval/config.toml``, then built-in defaults. A +``[.]`` table is a named profile, selected in code with +``Settings.load("")``. See the +:doc:`settings guide ` for the settings and +worked examples. + +.. automodule:: dataretrieval.settings + :members: configure, Settings, AdapterSettings, show_settings, + config_path, settings_for, ConfigurationError + :show-inheritance: diff --git a/docs/source/reference/wateruse.rst b/docs/source/reference/wateruse.rst deleted file mode 100644 index db4e4962..00000000 --- a/docs/source/reference/wateruse.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. _wateruse: - -dataretrieval.wateruse ----------------------- - -.. automodule:: dataretrieval.wateruse - :members: diff --git a/docs/source/userguide/errors.rst b/docs/source/userguide/errors.rst index 046f7a5d..0708c1be 100644 --- a/docs/source/userguide/errors.rst +++ b/docs/source/userguide/errors.rst @@ -102,7 +102,7 @@ mid-stream, the work already completed is preserved: catch except FanOutInterrupted as again: exc = again -The same loop works for ``wateruse.get_wateruse`` with a list of states, +The same loop works for ``nwdc.get_wateruse`` with a list of states, counties, or HUCs. Chunk a large request more finely diff --git a/docs/source/userguide/index.rst b/docs/source/userguide/index.rst index 96ca88fc..b61b7372 100644 --- a/docs/source/userguide/index.rst +++ b/docs/source/userguide/index.rst @@ -13,6 +13,7 @@ Contents .. toctree:: :maxdepth: 1 + settings errors timeconventions dataportals diff --git a/docs/source/userguide/settings.rst b/docs/source/userguide/settings.rst new file mode 100644 index 00000000..dddb23fc --- /dev/null +++ b/docs/source/userguide/settings.rst @@ -0,0 +1,598 @@ +.. _settings: + +============= +Settings +============= + +``dataretrieval`` retrieves from several services, and most of what you would +want to adjust — a concurrency cap, a retry budget, where requests go — belongs +to *one* of them. So a **settings profile** is a named set of settings for +one adapter, written in code or stored in your settings file, and a +``configure`` block puts one profile per adapter into effect for the calls +inside it. The Water Data API key is the exception that proves the rule: it +authenticates to a gateway rather than to an adapter, so it stays package-wide. + +.. contents:: + :local: + :depth: 1 + + +.. _configuration-one-block: + +One block, several services +--------------------------- + +This is the case the mechanism exists for. Say the file holds what you would +write once and keep — the key, a retry budget, and Water Data's everyday +concurrency — plus two named profiles for the shapes you only sometimes want: + +.. code-block:: toml + + api_key = "your_api_key_here" # package-wide: every adapter that reads it + retries = 6 + + [waterdata] + concurrency = 16 # waterdata's default profile: always active + + [waterdata.overnight] # a named profile: only when selected + concurrency = "unbounded" + parallel_chunks = 8 + + [ngwmn.gentle] + concurrency = 2 + +Then one block configures three services, taking two of them from the file by +name and building the third on the spot: + +.. code-block:: python + + import dataretrieval + from dataretrieval import ngwmn, waterdata, wqp + from dataretrieval.ngwmn import NgwmnSettings + from dataretrieval.waterdata import WaterdataSettings + from dataretrieval.wqp import WqpSettings + + with dataretrieval.configure( + WaterdataSettings.load("overnight"), # from the file, by name + NgwmnSettings.load("gentle"), # from the file, by name + WqpSettings(retries=2), # built here + ): + flow, _ = waterdata.get_daily(monitoring_location_id=sites, time="P30D") + levels, _ = ngwmn.get_water_level(monitoring_location_id=wells) + samples, _ = wqp.get_results(siteid=sites) + +Inside the block Water Data runs unbounded and asks the planner for eight +chunks, NGWMN runs two requests at a time, and WQP retries twice. Everything a +configuration does *not* name still comes from below it, per setting: Water +Data and NGWMN both retry six times and both send the ``api_key``, written once +at the top of the file, because a configuration contributes what it names and +inherits the rest. Only WQP named ``retries``, so only WQP departs from the +file's six. + +Outside the block nothing has changed, and putting those two profiles in the +file changed nothing on its own — a named profile is inert until a caller +selects it, which is what makes one safe to add to a file other people's jobs +also read. + +Two rules keep a block like that unambiguous. A configuration knows which +adapter it targets — that is a property of its class — so you never restate it, +and ``Settings`` targets none of them, which is what makes it +package-wide. And there is at most one configuration per adapter: naming two +raises rather than picking one, because there would be no defined order between +them — combine them into one instead. + + +Settings +-------- + +.. list-table:: + :header-rows: 1 + :widths: 18 12 26 44 + + * - Setting + - Default + - Environment variable + - What it does + * - ``api_key`` + - none + - ``API_USGS_PAT`` + - Water Data API key. Raises your hourly request quota substantially; + `register for one `_. + * - ``concurrency`` + - ``32`` + - ``API_USGS_CONCURRENT`` + - Cap on sub-requests in flight at once for a chunked query. A positive + integer, ``1`` to run them one at a time, or ``"unbounded"`` to remove + the cap. Does not change how many requests are made, only how many run + simultaneously. + * - ``retries`` + - ``4`` + - ``API_USGS_RETRIES`` + - Retries after a transient failure (429, 5xx, timeout). ``0`` disables. + * - ``progress`` + - auto + - ``API_USGS_PROGRESS`` + - Whether to draw the status line. Auto means on for a terminal or + Jupyter kernel, off for redirected output and CI. + * - ``parallel_chunks`` + - ``1`` + - *(none — see below)* + - Default fan-out for multi-value queries. ``1`` means split only as far + as the URL byte limit forces. + * - ``stall_timeout`` + - ``60`` + - ``API_USGS_STALL_TIMEOUT`` + - Seconds a call may go without receiving *any* data before retrying + stops and the failure surfaces. Bounds the wall-clock cost of a dead + connection, which ``retries`` alone does not — it counts attempts, not + seconds. Progress resets the clock; ``0`` disables the bound. + * - ``base_url`` + - the service's own + - *(none — code only)* + - Where to send one service's requests. Per adapter, and settable only in + a ``configure`` block: a file that silently redirected the library to + another host would be a supply-chain hazard. See + :ref:`configuration-redirect`. + + +Where settings come from +------------------------ + +Highest precedence first: + +1. A configuration passed to an active ``dataretrieval.configure(...)`` block. +2. A named profile you selected in that block — + ``WaterdataSettings.load("bulk")``. +3. The environment variable for that setting. +4. The adapter's default profile in the settings file: the + ``[]`` table. +5. The package-wide keys at the top of the settings file — + ``~/.dataretrieval/config.toml``, or the path in ``DATARETRIEVAL_CONFIG``. +6. The adapter's own built-in preference, where it has one — NWDC asks for a + ``concurrency`` of 4, because that is as far as the service is + stress-tested. It is a default, not a cap: anything you set above outranks + it. +7. The package built-in default, which for ``concurrency`` is 32. + +The top two rungs both name a single adapter, and naming two configurations +for one adapter raises, so they cannot disagree inside one block. Between +nested blocks the innermost decides, as it does for everything else. + +Precedence applies **per setting**. An environment that sets only +``API_USGS_PAT`` leaves a file-provided ``concurrency`` fully in effect — +sources are merged, not replaced. + +A variable that is *set but empty* (``export API_USGS_PAT=``, or a CI secret +that resolves to nothing) does not count as configured, so an empty variable +your tooling happened to create cannot silently discard the key in your config +file. The one exception is ``API_USGS_PROGRESS``, where blank has always meant +"off" and so is treated as a real value. + +.. note:: + + The environment ranks above the file, matching common deployment tools and + preserving the existing ``API_USGS_*`` variables as authoritative runtime + overrides. The reasoning is in :doc:`ADR 0009 + `. + + The one exception is rung 2 above rung 3 — a profile you name in code. That + is a more deliberate act than a variable inherited from whatever started + your process, and having it lose to that variable is the kind of thing you + would file a bug about. The inversion covers what the profile names and + nothing else: every setting you did *not* name still follows the + environment-above-file rule, in the same block. See :doc:`ADR 0011 + `. + + +An environment variable +----------------------- + +Still fully supported, and the simplest option for a single key on one +machine: + +.. code-block:: bash + + export API_USGS_PAT="your_api_key_here" + +This is also the mechanism the `R dataRetrieval package +`_ uses, under the same variable +name, so one export serves both. + + +A configuration file +-------------------- + +Better when you would rather not have a credential in your shell environment, +where it is inherited by every process you start. Create +``~/.dataretrieval/config.toml``: + +.. code-block:: toml + + api_key = "your_api_key_here" + +Restrict it so other users on the machine cannot read it — ``dataretrieval`` +warns once if a file containing a key is group- or world-readable: + +.. code-block:: bash + + chmod 600 ~/.dataretrieval/config.toml + +Any setting can go in the file: + +.. code-block:: toml + + api_key = "your_api_key_here" + concurrency = 16 + retries = 8 + +Point ``DATARETRIEVAL_CONFIG`` at a different path to override the location — +useful for a container or a job scheduler that mounts secrets elsewhere. + + +Per-adapter settings +~~~~~~~~~~~~~~~~~~~~ + +To tune one service and leave the rest alone, name the adapter — the same name +you import: + +.. code-block:: toml + + concurrency = 16 # every adapter + + [ngwmn] + concurrency = 4 # NGWMN only + + [wqp] + retries = 2 + +.. code-block:: python + + from dataretrieval.ngwmn import NgwmnSettings + from dataretrieval.wqp import WqpSettings + + with dataretrieval.configure( + NgwmnSettings(concurrency=4), WqpSettings(retries=2) + ): + ... + +An adapter table *overrides* the top-level one per setting, so ``[ngwmn]`` +above still inherits ``retries`` and the ``api_key``. Precedence is unchanged +otherwise: an adapter-scoped value outranks a package-wide one only within the +same source, so ``API_USGS_CONCURRENT`` exported for one run still beats a +``[ngwmn] concurrency`` in the file. + +Between ``configure`` blocks that tie-break applies per block: an adapter +configuration beats a package-wide value set by the *same* block, while +anything set by a block nested inside it wins over both. So a +``configure(Settings(concurrency=1))`` can still throttle a call an +enclosing block had scoped to one adapter, and the innermost block decides. + +Each adapter accepts only the settings it reads, and they are the fields of its +configuration class — ``concurrency`` and ``parallel_chunks`` are meaningless to +an adapter that issues a single request, so ``StreamstatsSettings`` has no +such field and ``[streamstats] parallel_chunks = 8`` is an error rather than a +line that quietly does nothing: + +==================================== ====================================== ======================================== +Adapter Settings profile Accepts +==================================== ====================================== ======================================== +``waterdata`` ``waterdata.WaterdataSettings`` ``concurrency``, ``parallel_chunks``, + ``retries``, ``stall_timeout``, + ``base_url`` +``ngwmn`` ``ngwmn.NgwmnSettings`` the same five +``nwdc`` ``nwdc.NwdcSettings`` ``concurrency``, ``retries``, + ``stall_timeout``, ``base_url`` +``wqp``, ``nldi``, ``streamstats`` ``wqp.WqpSettings`` and so on ``retries``, ``stall_timeout``, + ``base_url`` +==================================== ====================================== ======================================== + +Each class lives in the module whose code reads those settings, so a setting's +definition sits next to its use rather than in a service-neutral file. + +``api_key`` is deliberately not per-adapter. It authenticates to the *gateway* +in front of a host, and Water Data and NGWMN are served from the same host — +one key, one hourly quota shared between them — so a per-adapter key would +describe a distinction the service does not have. ``progress`` is likewise +package-wide: there is one progress line per call. + + +Named profiles +~~~~~~~~~~~~~~ + +An adapter can hold more than one shape at a time. The ``[]`` table is +that adapter's **default profile** — always in effect, as above — while a +``[.]`` table is a **named profile**, inert until you select it: + +.. code-block:: toml + + [waterdata] + concurrency = 16 # the default profile: always in effect + + [waterdata.bulk-pull] + concurrency = "unbounded" # only when selected + parallel_chunks = 8 + +So one file can hold an overnight bulk shape beside a polite daytime one, and +name as many of each as an adapter has uses for. + +A named profile states only what differs: everything it does not name still +comes from the adapter's default profile, the package-wide keys, and the tiers +below — per setting. + +``load`` reads the table and hands you a configuration object, so a name the +file does not define raises there and then, listing the names it does define — +a profile you just typed is more likely a typo than a request to fall through +to settings you did not ask for. What comes back is inert until you pass it to +``configure``; that is what puts a selected profile above the environment, +since selecting one is something your code did. + +A profile holds settings and nothing else: ``[waterdata.bulk-pull.ngwmn]`` is +not a Water Data profile carrying NGWMN detail, and selecting it says so rather +than quietly ignoring the nested table. Two adapters means two profiles, +selected in the same block, as in :ref:`the example above +`. + + +A ``configure`` block +--------------------- + +The highest-precedence source, and the one to use when a setting must apply to +*this* call and no other: + +.. code-block:: python + + import dataretrieval + from dataretrieval import Settings, waterdata + + with dataretrieval.configure(Settings(api_key=secrets["usgs"])): + df, md = waterdata.get_daily( + monitoring_location_id="USGS-05114000", + parameter_code="00060", + time="P7D", + ) + +``configure`` takes configuration objects positionally, and nothing else. The +adapter a configuration targets is a property of its class, so you never +restate it — and ``Settings`` targets none of them in particular, which is +what makes it package-wide. + +.. note:: + + Settings are not keywords on ``configure``. ``configure(api_key=...)`` and + the per-adapter mappings ``configure(ngwmn={"concurrency": 4})`` were an + earlier spelling and are gone; write ``Settings(api_key=...)`` and + ``NgwmnSettings(concurrency=4)`` instead. Passing anything that is not + a configuration raises and names the replacement, so an old script says what + to write rather than failing obscurely. + +Because it is backed by a :class:`~contextvars.ContextVar`, the value applies +to the current thread and to asyncio tasks started inside the block, and +cannot leak into another thread or task. That is what makes it usable from a +web service or a notebook working with more than one account: + +.. code-block:: python + + # each thread keeps its own key; no os.environ mutation, no race + def fetch_for(user): + with dataretrieval.configure(Settings(api_key=vault.read(user.key_path))): + return waterdata.get_daily(monitoring_location_id=user.sites) + +Blocks nest and merge per setting, so an inner block that tunes one thing +keeps the rest: + +.. code-block:: python + + with dataretrieval.configure(Settings(api_key=key, concurrency=8)): + ... + # api_key still applies + with dataretrieval.configure(Settings(concurrency=1)): + ... + +Values are validated when the configuration is *constructed*, so a typo raises +on the line you wrote it on rather than deep inside a later request. + +Omitted settings inherit from an outer block or a lower-precedence source. +Passing ``None`` explicitly suppresses those sources and restores built-in +behavior for that block. ``Settings(api_key=None)``, for example, makes an +anonymous call even if ``API_USGS_PAT`` is set. + +.. tip:: + + Prefer reading the key from a secret store, environment, or config file + over writing a literal into a script — a literal is what ends up committed + or pasted into a shared notebook. + + +Checking what is in effect +-------------------------- + +``show_settings()`` reports each setting's effective value and where it came +from. It never prints the key itself. The report below is what a file holding a +key, a package-wide ``concurrency``, an ``[ngwmn]`` table and a +``[waterdata.bulk]`` profile produces, with ``API_USGS_RETRIES`` exported and +the ``bulk`` profile selected for the block: + +.. code-block:: python + + >>> with dataretrieval.configure(WaterdataSettings.load("bulk")): + ... dataretrieval.show_settings() + settings file /home/u/.dataretrieval/config.toml (found) + api_key /home/u/.dataretrieval/config.toml + concurrency 16 /home/u/.dataretrieval/config.toml + retries 8 $API_USGS_RETRIES + progress auto built-in default + parallel_chunks 1 built-in default + stall_timeout 60s built-in default + + A built-in default is package-wide. An adapter may prefer its own for + its own calls; a value from any source above overrides both. + + adapter overrides + waterdata parallel_chunks 8 configure() block [waterdata.bulk] + ngwmn concurrency 4 /home/u/.dataretrieval/config.toml [ngwmn] + + profiles in the file: [waterdata.bulk] + A profile applies only where a row above names it; select one in + code with Settings.load(""). + + not reported: nldi (not imported, so the settings each accepts are unknown here) + +Each line names the exact source, including which table inside the file, which +is usually enough to answer "why is it still using my old key?". A value that +came from a profile names the profile — ``configure() block +[waterdata.bulk]``, not merely "a block" — so a report taken from inside a +``with`` block says which selection produced it. Only settings actually +overridden for an adapter get a row in the second section; everything else is +inherited from the rows above it. + +The profile section lists what the *file* defines, whether or not this run +selected any of it. A named profile does nothing until a caller selects it, so +seeing ``[waterdata.bulk]`` there while no row above mentions it is the answer +to "I added a profile and nothing changed". + +The last line is the honest cost of validating an adapter's settings lazily: +``dataretrieval`` cannot say what ``nldi`` accepts until something imports it, +so it says that rather than quietly omitting the service. It is named rather +than left out, because an omitted service would read as "nothing is configured +for it", which is a different claim. + +It never raises. A malformed file or a value that fails its grammar is reported +in place — on the ``settings file`` line for a whole-file problem, or in that +setting's own row — because a broken configuration is exactly when you reach +for this. + + +Why ``parallel_chunks`` has no environment variable +--------------------------------------------------- + +Every other setting can be set from the environment. ``parallel_chunks`` +cannot, on purpose. + +Raising it splits a query into more sub-requests, and *each sub-request spends +rate-limit quota*. Whether that is a good trade depends on the size of the +query — which the library cannot know in advance. The setting therefore does +not add another process-global environment knob that could be exported once +and inherited by every subprocess. + +Set it per call, which is almost always what you want: + +.. code-block:: python + + with waterdata.parallel_chunks(8): + df, md = waterdata.get_daily(monitoring_location_id=many_sites) + +or as a baseline in the config file — deliberately written, and visible in +``show_settings()``. Put it in a ``[.]`` table rather than +at the top level: a named profile applies only to runs that select it, while a +top-level value applies to every query in every process that reads the file, +which is how a setting added for one bulk pull quietly exhausts an hourly quota +months later. ``dataretrieval`` warns if it finds one at the top level. + +The value limits optional refinement only. URL-byte safety can require more +sub-requests than the configured value, and an input with nothing to split +stays a single request. + +``parallel_chunks(n)`` is sugar for +``configure(Settings(parallel_chunks=n))``: one scoping mechanism, so the +innermost block wins whichever spelling set it, and ``show_settings()`` +always reports the value the chunker will actually use. + + +.. _configuration-redirect: + +Pointing an adapter at another host +----------------------------------- + +``base_url`` sends one adapter's requests somewhere else — a staging instance, +a mirror, or a recording proxy — for the duration of a block: + +.. code-block:: python + + import dataretrieval + from dataretrieval import waterdata + from dataretrieval.waterdata import WaterdataSettings + + with dataretrieval.configure( + WaterdataSettings(base_url="https://staging.example/waterdata") + ): + df, md = waterdata.get_daily(monitoring_location_id="USGS-05114000") + +It names one adapter, so nothing else moves: NGWMN is served from the same host +as Water Data, and a ``WaterdataSettings`` still leaves it alone. What the +value replaces is that adapter's own base, and the package appends its usual +paths to it — for Water Data that is the root all four of its APIs hang off, so +one value moves the OGC collections, the Samples database, the statistics +service and the STAC catalog together. + +**Code only.** The configuration file and the environment both refuse it. A +``base_url`` key anywhere in the file, and an exported ``API_USGS_BASE_URL``, +each raise a ``ConfigurationError`` saying the setting *may only be set in +code, in a configure() block* and naming the configuration to pass it on +instead. + +A file or a shell export that silently redirected a data-retrieval library to +another host would be a supply-chain hazard: nothing at the call site would +show it, and a script that reads correctly would be talking to someone else's +service. A ``with`` block keeps the redirect where a reader of the script sees +it. The refusal is loud rather than silent for the same reason — a variable +that was quietly ignored would leave you believing you had redirected +something. + +**The API key does not follow.** It is scoped to the one host that honors it +(:ref:`below `), so a redirected call goes out +without it. That is deliberate: the host you redirected to is not the host you +gave a credential to. If the mirror needs its own credential, it needs its own +mechanism. + + +.. _configuration-secret-store: + +Keeping a key out of your environment entirely +---------------------------------------------- + +If your credentials live in a secret manager, nothing needs to touch +``os.environ``: + +.. code-block:: python + + import dataretrieval + import boto3 + from dataretrieval import Settings, waterdata + + secrets = boto3.client("secretsmanager") + key = secrets.get_secret_value(SecretId="usgs-pat")["SecretString"] + + with dataretrieval.configure(Settings(api_key=key)): + df, md = waterdata.get_continuous(monitoring_location_id="USGS-05114000") + +Wherever the key comes from, it is sent only to ``api.waterdata.usgs.gov`` and +is stripped from any cross-host redirect, so it cannot leak to another host. + + +Behind a TLS-intercepting proxy +------------------------------- + +On a corporate network that re-signs HTTPS traffic, requests fail with a +certificate-verification error. Point the standard OpenSSL variables at your +organization's CA bundle: + +.. code-block:: bash + + export SSL_CERT_FILE=/path/to/corporate-ca.pem + # or, for a directory of hashed certificates: + export SSL_CERT_DIR=/etc/ssl/certs + +``httpx`` honors these natively, so they apply to **every** getter in the +package — including the OGC collection getters (``get_daily``, +``get_continuous``, and the rest), which take no SSL parameter of their own. + +Prefer this to ``ssl_check=False``. That argument exists on some of the older +getters and switches certificate verification *off* rather than trusting your +CA, so it accepts any certificate a network path offers — and it is not +available on the OGC getters at all. A CA bundle keeps verification on and +works everywhere. + +.. note:: + + ``SSL_CERT_FILE`` is read by OpenSSL, not by ``dataretrieval``, so it does + not appear in :func:`~dataretrieval.show_settings`. diff --git a/pyproject.toml b/pyproject.toml index 5a184141..aa7578d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,17 @@ dependencies = [ # Directly imported by ``waterdata`` (``anyio.from_thread.start_blocking_portal``), # so declared here rather than relied on transitively via httpx. "anyio>=4.0", + # ``dataretrieval.settings`` declares each adapter's settings as a + # ``BaseSettings`` subclass and resolves them through a chain of + # ``PydanticBaseSettingsSource`` implementations (ADR 0012). A required + # dependency rather than an extra: settings resolve on the request path -- + # every call reads the API key -- so an optional one would mean shipping a + # second, standard-library implementation of the same chain and testing both. + "pydantic-settings>=2.6", + # ``dataretrieval.settings`` reads a TOML settings file. ``tomllib`` is + # stdlib from 3.11, so this marker installs the backport only on 3.10 and + # drops itself when ``requires-python`` moves to >=3.11. + "tomli>=1.1.0; python_version < '3.11'", ] dynamic = ["version"] diff --git a/tests/architecture_test.py b/tests/architecture_test.py index 0fff59ef..a8dbb71c 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -163,6 +163,45 @@ def test_runtime_import_graph_is_acyclic() -> None: raise AssertionError(f"Runtime import cycle: {cycle}") from exc +def test_settings_is_a_first_party_leaf() -> None: + """Settings resolution must stay importable from anywhere. + + ``dataretrieval.settings`` is read by ``utils`` (headers), ``ogc.chunking`` + (concurrency), ``ogc.retry``, and ``ogc.progress``. If it imported any of + them it would create a cycle, so ``dataretrieval.exceptions`` is its one + allowed first-party import: the taxonomy leaf, itself free of first-party + and runtime third-party imports (asserted above), so depending on it adds no + weight and cannot cycle. ``ConfigurationError`` lives there because settings + resolve on the request path, so a broken settings file must be catchable as + ``except DataRetrievalError`` like any other failure of that call. + + The *third-party* half of this contract was withdrawn by ADR 0012: the + module is built on ``pydantic-settings``, which is a runtime dependency of + the package. What the roster below still buys is that the list stays + deliberate -- a leaf that quietly grew ``httpx`` or ``pandas`` would make the + cheapest module in the package expensive, and every adapter imports this one. + """ + imports = _runtime_imports(PACKAGE_ROOT / "settings.py") + first_party = { + name + for name in imports + if name.startswith("dataretrieval") and name != "dataretrieval.exceptions" + } + assert not first_party, ( + "dataretrieval.settings may only import dataretrieval.exceptions: " + f"{sorted(first_party)}" + ) + roots = {module.partition(".")[0] for module in imports} + # Static analysis sees both sides of the version guard. Python 3.10's stdlib + # inventory does not yet include the unreachable ``tomllib`` branch. + allowed = {"dataretrieval", "pydantic", "pydantic_settings", "tomli", "tomllib"} + third_party = roots - sys.stdlib_module_names - allowed + assert not third_party, ( + "dataretrieval.settings gained third-party dependencies beyond the " + f"settings stack: {sorted(third_party)}" + ) + + def test_engine_request_import_surface_does_not_grow() -> None: """Engine imports only request names it uses and may not grow a new hub. @@ -357,11 +396,11 @@ def test_credential_policy_has_one_definition() -> None: def _waterdata_family_paths() -> tuple[str, ...]: """Read the collection-family inventory from its dependency contract.""" - config = configparser.ConfigParser() - config.read(PACKAGE_ROOT.parent / ".importlinter") + parser = configparser.ConfigParser() + parser.read(PACKAGE_ROOT.parent / ".importlinter") return tuple( module.removeprefix("dataretrieval.").replace(".", "/") + ".py" - for module in config["importlinter:contract:waterdata-families"][ + for module in parser["importlinter:contract:waterdata-families"][ "modules" ].split() ) @@ -377,7 +416,7 @@ def _waterdata_family_paths() -> tuple[str, ...]: "ngwmn.py", "nldi.py", "streamstats.py", - "wateruse.py", + "nwdc.py", "wqp.py", "waterdata/nearest.py", "waterdata/ratings.py", @@ -513,7 +552,7 @@ def test_empty_result_shaping_consults_the_schema_endpoint() -> None: ) -def test_wateruse_does_not_reimplement_fan_out_orchestration() -> None: +def test_nwdc_does_not_reimplement_fan_out_orchestration() -> None: """Water Use must drive its locations through the shared fan-out executor. It previously ran its own ``asyncio.gather`` with a private semaphore and a @@ -522,7 +561,7 @@ def test_wateruse_does_not_reimplement_fan_out_orchestration() -> None: resume, progress, and the shared concurrency setting. Assert the duplication cannot quietly return. """ - source = (PACKAGE_ROOT / "wateruse.py").read_text(encoding="utf-8") + source = (PACKAGE_ROOT / "nwdc.py").read_text(encoding="utf-8") tree = ast.parse(source) offenders = { f"{node.value.id}.{node.attr}" diff --git a/tests/conftest.py b/tests/conftest.py index de5cfd36..897f4b85 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,8 @@ import pytest +from dataretrieval import settings + def pytest_collection_modifyitems(config, items): """Apply relaxed ``pytest-httpx`` strict-mode settings to every test @@ -34,7 +36,7 @@ def non_mocked_hosts() -> list[str]: @pytest.fixture(autouse=True) -def _pin_chunker_env(monkeypatch): +def _pin_chunker_env(monkeypatch, tmp_path): """Pin every test to one connection, no retries, and no stall budget. Production defaults ``API_USGS_CONCURRENT`` to 32, @@ -56,3 +58,9 @@ def _pin_chunker_env(monkeypatch): monkeypatch.setenv("API_USGS_CONCURRENT", "1") monkeypatch.setenv("API_USGS_RETRIES", "0") monkeypatch.setenv("API_USGS_STALL_TIMEOUT", "0") + # Point DATARETRIEVAL_CONFIG at a path that does not exist, so a developer's + # real ~/.dataretrieval/config.toml -- which may hold an API key or a raised + # concurrency -- can never influence a test run. Config tests opt in by + # pointing the variable at a file they wrote. + monkeypatch.setenv("DATARETRIEVAL_CONFIG", str(tmp_path / "no-such-config.toml")) + settings._reset_file_cache() diff --git a/tests/contracts/README.md b/tests/contracts/README.md index d05bed14..72886e9f 100644 --- a/tests/contracts/README.md +++ b/tests/contracts/README.md @@ -5,7 +5,7 @@ The suite uses four dependency-oriented layers without moving established tests: - **Public contract** (`tests/contracts/`): imports, exports, signatures, return annotations, metadata/error promises, and compatibility paths. These tests use public modules and no live services. -- **Adapter contract** (`waterdata_test.py`, `ngwmn_test.py`, `wateruse_test.py`, +- **Adapter contract** (`waterdata_test.py`, `ngwmn_test.py`, `nwdc_test.py`, `wqp_test.py`, `nldi_test.py`, `streamstats_test.py`): service request wiring, response parsing, and documented protocol behavior. - **Component** (`transport_test.py`, `waterdata_chunking_test.py`, diff --git a/tests/contracts/public_api_test.py b/tests/contracts/public_api_test.py index 53e28aeb..c5345222 100644 --- a/tests/contracts/public_api_test.py +++ b/tests/contracts/public_api_test.py @@ -20,6 +20,7 @@ _EXPECTED_WATERDATA_ALL = [ "CODE_SERVICES", "FILTER_LANG", + "WaterdataSettings", "PROFILES", "PROFILE_LOOKUP", "SERVICES", diff --git a/tests/ngwmn_test.py b/tests/ngwmn_test.py index 5921aa1e..2658df86 100644 --- a/tests/ngwmn_test.py +++ b/tests/ngwmn_test.py @@ -18,7 +18,8 @@ import pytest from pandas import DataFrame -from dataretrieval import ngwmn +import dataretrieval +from dataretrieval import ngwmn, settings from dataretrieval.utils import BaseMetadata # Agency-qualified ids in the multi-agency form NGWMN uses (not all ``USGS-``). @@ -519,6 +520,37 @@ def test_empty_result_returns_typed_empty_frame(httpx_mock): assert "monitoring_location_id" in df.columns +def test_a_configured_base_url_redirects_ngwmn_alone(httpx_mock): + """Two adapters share this host, and a redirect must still name only one. + + NGWMN and Water Data are served from ``api.waterdata.usgs.gov``, so a URL + cannot tell them apart -- which is why the settings table an OGC call reads + is declared by the adapter rather than derived from its base. Redirecting + NGWMN therefore has to leave Water Data where it was, and the Water Data + mock here is never requested: the assertion is on the whole request list. + """ + mirror = "https://mirror.example/ngwmn" + httpx_mock.add_response( + method="GET", + url=re.compile(rf"^{re.escape(mirror)}/collections/sites/items"), + json=_SITES, + ) + _mock(httpx_mock, "sites", _SITES) + + with dataretrieval.configure(ngwmn.NgwmnSettings(base_url=mirror)): + df, md = ngwmn.get_sites(state="Wisconsin", limit=10) + + assert len(df) == 2 + assert str(md.url).startswith(f"{mirror}/collections/sites/items") + assert [urlsplit(str(r.url)).netloc for r in httpx_mock.get_requests()] == [ + "mirror.example" + ] + + # And Water Data, the other adapter on the real host, was never named by it. + with dataretrieval.configure(ngwmn.NgwmnSettings(base_url=mirror)): + assert settings.base_url(adapter="waterdata") is None + + # --- live upstream monitor --------------------------------------------------- diff --git a/tests/nldi_test.py b/tests/nldi_test.py index d60c3886..81bc530d 100644 --- a/tests/nldi_test.py +++ b/tests/nldi_test.py @@ -3,6 +3,7 @@ import pytest from geopandas import GeoDataFrame +import dataretrieval import dataretrieval.nldi as nldi from dataretrieval.nldi import ( NLDI_API_BASE_URL, @@ -56,7 +57,9 @@ def test_query_nldi_opts_into_retry(monkeypatch): monkeypatch.setattr(nldi, "_query_with_retry", query) assert nldi._query_nldi("https://example.test", {}) == {} - query.assert_called_once_with("https://example.test", payload={}) + # ``adapter`` names whose settings the retry resolves, so a ``[nldi]`` + # table reaches these calls and no others. + query.assert_called_once_with("https://example.test", payload={}, adapter="nldi") def mock_request(httpx_mock, request_url, file_path): @@ -440,3 +443,33 @@ def test_query_504_raises_service_unavailable(httpx_mock): # legacy query path renders verbatim as "HTTP 504 (URL: ...)". with pytest.raises(ServiceUnavailable, match="504"): query(url, {"a": "1"}) + + +def test_a_configured_base_url_redirects_every_nldi_request(httpx_mock): + """The block moves the catalog probe and the query alike. + + NLDI validates a feature source against a catalog it fetches itself, so a + redirect that reached only the getter's own URL would leave the library + asking the real service whether the mirror's sources exist -- and the mirror + exists precisely because the caller cannot or should not reach the service. + Both mocks are on the mirror, so either one straying fails this. + """ + mirror = "https://mirror.example/nldi" + httpx_mock.add_response( + method="GET", url=f"{mirror}/", json=[{"source": "WQP"}, {"source": "comid"}] + ) + with open("tests/data/nldi_get_basin.json") as body: + httpx_mock.add_response( + method="GET", + url=( + f"{mirror}/WQP/USGS-054279485/basin" + "?simplified=true&splitCatchment=false" + ), + text=body.read(), + ) + + with dataretrieval.configure(nldi.NldiSettings(base_url=mirror)): + gdf = get_basin(feature_source="WQP", feature_id="USGS-054279485") + + assert isinstance(gdf, GeoDataFrame) + assert {str(r.url).startswith(mirror) for r in httpx_mock.get_requests()} == {True} diff --git a/tests/wateruse_test.py b/tests/nwdc_test.py similarity index 83% rename from tests/wateruse_test.py rename to tests/nwdc_test.py index 5e88f006..46563e26 100644 --- a/tests/wateruse_test.py +++ b/tests/nwdc_test.py @@ -1,10 +1,11 @@ -"""Offline tests for :mod:`dataretrieval.wateruse`. +"""Offline tests for :mod:`dataretrieval.nwdc`. All HTTP is mocked with ``pytest-httpx``; no live calls (per AGENTS.md). """ import re import socket +import warnings from urllib.parse import parse_qs, urlsplit import httpx @@ -12,11 +13,12 @@ import pytest import dataretrieval +from dataretrieval import nwdc, settings from dataretrieval import progress as _progress -from dataretrieval import wateruse +from dataretrieval.exceptions import DataRetrievalError +from dataretrieval.nwdc import _next_page_url, _resolve_locations, get_wateruse from dataretrieval.transport import fanout as _fanout from dataretrieval.utils import BaseMetadata -from dataretrieval.wateruse import _next_page_url, _resolve_locations, get_wateruse # Match the NWDC endpoint regardless of query string, so assertions can drill # into the captured params without coupling registration to param order. @@ -514,9 +516,70 @@ def capture(request: httpx.Request) -> httpx.Response: assert sent["auth"] is None +def test_a_configured_base_url_redirects_the_request(httpx_mock): + """The whole call moves, page walk included, or the redirect is a half-truth. + + The page-two mock is served from the mirror and its cursor names the mirror: + if either the request or the ``rel="next"`` walk had stayed on the NWDC's + host, one of them would go unmocked and this would fail rather than quietly + talk to the service the block redirected away from. + """ + mirror = re.compile(r"^https://mirror\.example/data") + httpx_mock.add_response( + method="GET", + url=mirror, + text=_CSV_P1, + headers={"link": '; rel="next"'}, + ) + httpx_mock.add_response(method="GET", url=mirror, text=_CSV_P2) + + with dataretrieval.configure( + nwdc.NwdcSettings(base_url="https://mirror.example/data") + ): + df, _ = get_wateruse(model="wu-public-supply-wd", state="RI") + + assert len(df) == 3 + assert [urlsplit(str(r.url)).netloc for r in httpx_mock.get_requests()] == [ + "mirror.example", + "mirror.example", + ] + + +def test_next_page_url_drops_the_service_rewrite_when_redirected(): + """The alias list and the rewrite are facts about the NWDC, not about URLs. + + Nothing but the NWDC answers for ``water.usgs.gov``, so a call an + ``NwdcSettings(base_url=...)`` pointed elsewhere gets the general rule + instead: follow a link only back to the host that served the page. Keeping + the rewrite would send page two of a mirrored query to the USGS -- and + refusing the mirror's own cursor would throw away page one. + """ + mirrored = httpx.Response( + 200, + text="", + headers={"link": '; rel="next"'}, + request=httpx.Request("GET", "https://mirror.example/data"), + ) + + assert _next_page_url(mirrored, host="mirror.example") == ( + "https://mirror.example/data?skip=600" + ) + + # A cursor back to the real service is now the cross-host case, refused for + # the same reason a foreign link is refused on an ordinary call. + strayed = httpx.Response( + 200, + text="", + headers={"link": '; rel="next"'}, + request=httpx.Request("GET", "https://mirror.example/data"), + ) + with pytest.raises(DataRetrievalError, match="cross-host"): + _next_page_url(strayed, host="mirror.example") + + def test_module_exposes_catalog_constants(): - assert "wu-public-supply-wd" in wateruse.MODELS - assert set(wateruse.TIME_RESOLUTIONS) == {"monthly", "annualcy", "annualwy"} + assert "wu-public-supply-wd" in nwdc.MODELS + assert set(nwdc.TIME_RESOLUTIONS) == {"monthly", "annualcy", "annualwy"} def test_initial_transient_is_retried(httpx_mock, monkeypatch): @@ -584,12 +647,12 @@ async def open_mock_client(**overrides): monkeypatch.setattr(_fanout, "open_async_client", open_mock_client) requests = [ - httpx.Request("GET", wateruse.WATERUSE_URL, params={"location": location}) + httpx.Request("GET", nwdc.WATERUSE_URL, params={"location": location}) for location in ("stateCd:AA", "stateCd:BB") ] with pytest.raises(dataretrieval.DataRetrievalError, match="Invalid model"): - wateruse._fan_out(requests, {}, True) + nwdc._fan_out(requests, {}, True) assert pages["n"] == 2, "the sibling finished its walk rather than being abandoned" @@ -655,15 +718,15 @@ def test_fan_out_honors_the_general_concurrency_setting(monkeypatch): quietly ignoring them -- the defect that motivated consolidating the knob. """ monkeypatch.setenv("API_USGS_CONCURRENT", "7") - assert _fanout._resolve_concurrency(wateruse.DEFAULT_CONCURRENT_REQUESTS) == 7 + assert settings.concurrency(nwdc.DEFAULT_CONCURRENT_REQUESTS) == 7 monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) assert ( - _fanout._resolve_concurrency(wateruse.DEFAULT_CONCURRENT_REQUESTS) - == wateruse.DEFAULT_CONCURRENT_REQUESTS + settings.concurrency(nwdc.DEFAULT_CONCURRENT_REQUESTS) + == nwdc.DEFAULT_CONCURRENT_REQUESTS ) # The service default is deliberately below the package-wide 32. - assert wateruse.DEFAULT_CONCURRENT_REQUESTS < _fanout._CONCURRENCY_DEFAULT + assert nwdc.DEFAULT_CONCURRENT_REQUESTS < settings.DEFAULT_CONCURRENCY def test_fan_out_reports_progress(httpx_mock, monkeypatch): @@ -762,7 +825,7 @@ async def fail(*_args, **_kwargs): failure.__context__ = resolution raise failure - monkeypatch.setattr(wateruse, "paginate", fail) + monkeypatch.setattr(nwdc, "paginate", fail) with pytest.raises(dataretrieval.NetworkError) as excinfo: get_wateruse(model="wu-public-supply-wd", state="RI") @@ -829,3 +892,82 @@ def test_mid_page_walk_transient_is_still_resumable(httpx_mock): assert excinfo.value.call is not None assert excinfo.value.completed_chunks == 1 assert excinfo.value.total_chunks == 2 + + +# --------------------------------------------------------------------------- +# Deprecated ``wateruse`` alias +# --------------------------------------------------------------------------- + + +def _reimport_wateruse(): + """Import the alias fresh, so its module-level warning fires again.""" + import importlib + import sys + + sys.modules.pop("dataretrieval.wateruse", None) + return importlib.import_module("dataretrieval.wateruse") + + +def test_wateruse_alias_warns_and_names_the_replacement(): + """Importing the old name is deprecated, dated, and points at ``nwdc``.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _reimport_wateruse() + + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert len(deprecations) == 1, [str(w.message) for w in caught] + message = str(deprecations[0].message) + assert "`dataretrieval.wateruse` is deprecated" in message + assert "`dataretrieval.nwdc`" in message + # Dated removal, per the convention nwis follows. + assert nwdc_alias_removal_date() in message + + +def nwdc_alias_removal_date() -> str: + from dataretrieval.wateruse import NWDC_RENAME_REMOVAL_DATE + + return NWDC_RENAME_REMOVAL_DATE + + +def test_wateruse_alias_re_exports_the_same_objects(): + """The alias forwards, it does not copy: identity must survive it. + + A caller monkeypatching through one spelling and asserting through the + other would otherwise see two different objects. + """ + alias = _reimport_wateruse() + + assert alias.get_wateruse is nwdc.get_wateruse + assert alias.MODELS is nwdc.MODELS + assert alias.TIME_RESOLUTIONS is nwdc.TIME_RESOLUTIONS + assert alias.DEFAULT_CONCURRENT_REQUESTS == nwdc.DEFAULT_CONCURRENT_REQUESTS + assert alias.__all__ == nwdc.__all__ + + +def test_importing_dataretrieval_does_not_warn(): + """``import dataretrieval`` must stay silent. + + The package imports ``nwdc`` directly; only code naming ``wateruse`` + itself should see the warning. If ``__init__`` ever imports the alias, + every user of the library gets a DeprecationWarning they cannot act on. + + Runs in a subprocess: a fresh interpreter is the only honest way to test + an import side effect, and clearing ``sys.modules`` in-process would hand + every later test a second copy of the package. + """ + import subprocess + import sys + + result = subprocess.run( + [ + sys.executable, + "-W", + "error::DeprecationWarning", + "-c", + "import dataretrieval", + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr diff --git a/tests/settings_test.py b/tests/settings_test.py new file mode 100644 index 00000000..fbc02dd0 --- /dev/null +++ b/tests/settings_test.py @@ -0,0 +1,2086 @@ +"""Tests for layered settings resolution (``dataretrieval.settings``).""" + +from __future__ import annotations + +import asyncio +import inspect +import io +import json +import os +import pathlib +import re +import textwrap +import threading +from dataclasses import dataclass +from typing import ClassVar + +import pytest + +import dataretrieval +from dataretrieval import settings, streamstats, waterdata +from dataretrieval.ngwmn import NgwmnSettings +from dataretrieval.nwdc import DEFAULT_CONCURRENT_REQUESTS, NwdcSettings +from dataretrieval.settings import Settings +from dataretrieval.streamstats import StreamstatsSettings +from dataretrieval.utils import _default_headers +from dataretrieval.waterdata import WaterdataSettings, endpoints +from dataretrieval.wqp import WqpSettings + +WATERDATA_URL = "https://api.waterdata.usgs.gov/ogcapi/v0/collections/daily/items" + +# Where the base-URL tests redirect to. A host the suite can never reach, so a +# redirect that failed to apply shows up as an unmocked request rather than as +# a real one. +_MIRROR = "https://mirror.example/waterdata" +_MIRROR_RE = re.compile(r"^https://mirror\.example/") +_WATERDATA_RE = re.compile(r"^https://api\.waterdata\.usgs\.gov/") + +# One committed page of the ``daily`` collection, shared with the Water Data +# suite. Real response shape rather than a hand-made stub, so a redirect is +# exercised through the same shaping the getters normally do; it carries no +# ``links``, so nothing paginates. +_DAILY_PAGE = json.loads( + (pathlib.Path(__file__).parent / "data" / "waterdata_ogc_fixtures.json").read_text() +)["daily"] + + +@pytest.fixture +def config_file(tmp_path, monkeypatch): + """Write a config file and point ``DATARETRIEVAL_CONFIG`` at it.""" + + def write(text: str): + path = tmp_path / "config.toml" + path.write_text(text) + path.chmod(0o600) # keep the loose-permission warning out of the way + for env in settings.ENV_VARS.values(): + monkeypatch.delenv(env, raising=False) + monkeypatch.setenv(settings.CONFIG_PATH_ENV, str(path)) + settings._reset_file_cache() + return path + + return write + + +# --- precedence ---------------------------------------------------------- + + +def test_default_when_nothing_is_configured(monkeypatch): + for env in settings.ENV_VARS.values(): + monkeypatch.delenv(env, raising=False) + assert settings.api_key() is None + assert settings.concurrency() == settings.DEFAULT_CONCURRENCY + assert settings.retries() == settings.DEFAULT_RETRIES + assert settings.parallel_chunks() == settings.DEFAULT_PARALLEL_CHUNKS + assert settings.progress() is None + + +def test_env_is_used_when_no_file_or_block(monkeypatch): + monkeypatch.setenv("API_USGS_PAT", "env-key") + monkeypatch.setenv("API_USGS_CONCURRENT", "4") + assert settings.api_key() == "env-key" + assert settings.concurrency() == 4 + + +def test_env_outranks_file(config_file, monkeypatch): + config_file('api_key = "file-key"\n') + monkeypatch.setenv("API_USGS_PAT", "env-key") + assert settings.api_key() == "env-key" + + +def test_block_outranks_file_and_env(config_file, monkeypatch): + config_file('api_key = "file-key"\n') + monkeypatch.setenv("API_USGS_PAT", "env-key") + with dataretrieval.configure(Settings(api_key="block-key")): + assert settings.api_key() == "block-key" + assert settings.api_key() == "env-key" + + +def test_precedence_is_per_setting_not_per_source(config_file, monkeypatch): + """An environment key must not blank out file-provided settings.""" + config_file("concurrency = 16\n") + monkeypatch.setenv("API_USGS_PAT", "env-key") + monkeypatch.setenv("API_USGS_RETRIES", "9") + assert settings.concurrency() == 16 # from the file + assert settings.api_key() == "env-key" # still from the env + assert settings.retries() == 9 # still from the env + + +# --- the configure() block ----------------------------------------------- + + +def test_blocks_nest_and_merge_per_setting(): + with dataretrieval.configure(Settings(api_key="outer", concurrency=4)): + with dataretrieval.configure(Settings(concurrency=8)): + assert settings.concurrency() == 8 + assert settings.api_key() == "outer" # inherited from the outer block + assert settings.concurrency() == 4 # inner block restored on exit + + +def test_omitted_setting_inherits_lower_source(monkeypatch): + monkeypatch.setenv("API_USGS_PAT", "env-key") + with dataretrieval.configure(Settings(concurrency=2)): + assert settings.api_key() == "env-key" + + +def test_explicit_none_suppresses_lower_sources(monkeypatch): + monkeypatch.setenv("API_USGS_PAT", "env-key") + monkeypatch.setenv("API_USGS_CONCURRENT", "4") + monkeypatch.setenv("API_USGS_PROGRESS", "true") + with dataretrieval.configure( + Settings(api_key=None, concurrency=None, progress=None) + ): + assert settings.api_key() is None + assert settings.concurrency() == settings.DEFAULT_CONCURRENCY + assert settings.progress() is None + assert settings.api_key() == "env-key" + assert settings.concurrency() == 4 + assert settings.progress() is True + + +@pytest.mark.parametrize( + "values", + [ + {"concurrency": 0}, + {"retries": -1}, + {"parallel_chunks": 0}, + {"progress": "flase"}, + ], +) +def test_a_settings_profile_validates_its_own_settings(values): + """A bad value raises where it was written, not inside a later request. + + Construction is earlier than the ``with``, which is earlier than the + request the value would otherwise have broken. + """ + with pytest.raises(settings.ConfigurationError): + Settings(**values) + + +@pytest.mark.parametrize( + ("values", "expected"), + [ + ({"api_key": 123}, "string"), + ({"concurrency": 1.5}, "integer"), + ({"concurrency": "8"}, "integer"), + ({"retries": "2"}, "integer"), + ({"progress": []}, "bool"), + ({"parallel_chunks": True}, "integer"), + ], +) +def test_settings_reject_values_outside_annotated_types(values, expected): + with pytest.raises(settings.ConfigurationError, match=expected): + Settings(**values) + + +def test_block_accepts_ints_and_strings(): + with dataretrieval.configure(Settings(concurrency="unbounded")): + assert settings.concurrency() is None + with dataretrieval.configure(Settings(concurrency=8)): + assert settings.concurrency() == 8 + with dataretrieval.configure(Settings(progress=False)): + assert settings.progress() is False + with dataretrieval.configure(Settings(progress=True)): + assert settings.progress() is True + + +def test_configure_takes_configurations_and_nothing_else(): + """The argument is an object, so a stray mapping or keyword cannot pass. + + ``configure(ngwmn={"concurrency": 2})`` was the earlier spelling, and it is + exactly what a reader of an old script will try. Naming the replacement in + the error is the difference between a two-minute fix and a search. + """ + with pytest.raises(settings.ConfigurationError, match="settings profiles"): + with dataretrieval.configure({"concurrency": 2}): + pass + with pytest.raises(settings.ConfigurationError, match="settings profiles"): + with dataretrieval.configure("waterdata"): + pass + # Settings are no longer keywords on ``configure`` at all. + with pytest.raises(TypeError): + with dataretrieval.configure(api_key="k"): + pass + + +def test_two_configurations_for_one_adapter_raise(): + """They are the one pairing with no defined order between them. + + Silently letting the last win would make a block's meaning depend on + argument order, which nothing in the surrounding chain does. + """ + with pytest.raises(settings.ConfigurationError, match="two settings profiles"): + with dataretrieval.configure( + WaterdataSettings(concurrency=2), + WaterdataSettings(retries=1), + ): + pass + + # Same rule for the package-wide configuration, which targets no adapter. + with pytest.raises(settings.ConfigurationError, match="package-wide"): + with dataretrieval.configure(Settings(retries=1), Settings(retries=2)): + pass + + # Two *different* adapters in one block is the whole point of the feature. + with dataretrieval.configure( + WaterdataSettings(concurrency=2), NgwmnSettings(concurrency=8) + ): + assert settings.concurrency(adapter="waterdata") == 2 + assert settings.concurrency(adapter="ngwmn") == 8 + + +def test_a_configuration_resolves_end_to_end(config_file, monkeypatch): + """Every tier below a passed configuration still applies, per setting.""" + config_file('api_key = "file-key"\nstall_timeout = 15\n') + monkeypatch.setenv("API_USGS_RETRIES", "9") + + with dataretrieval.configure(Settings(concurrency=3)): + assert settings.concurrency() == 3 # from the configuration + assert settings.retries() == 9 # still from the environment + assert settings.api_key() == "file-key" # still from the file + assert settings.stall_timeout() == 15 # still from the file + assert ( + settings.parallel_chunks() == settings.DEFAULT_PARALLEL_CHUNKS + ) # still the built-in default + + assert settings.concurrency() == settings.DEFAULT_CONCURRENCY + + +def test_an_adapter_configuration_narrows_to_one_adapter(monkeypatch): + """The adapter is a property of the class, so nothing else moves.""" + monkeypatch.delenv("API_USGS_RETRIES") # pinned by the autouse fixture + + with dataretrieval.configure(NgwmnSettings(retries=1)): + assert settings.retries(adapter="ngwmn") == 1 + # Every other adapter, and the package-wide read, are untouched -- + # including waterdata, which shares NGWMN's host and its API key. + for other in ("waterdata", "nwdc", "wqp", "streamstats"): + assert settings.retries(adapter=other) == settings.DEFAULT_RETRIES + assert settings.retries() == settings.DEFAULT_RETRIES + + +# --- isolation (the point of issue #352) --------------------------------- + + +def test_threads_do_not_leak_credentials_into_each_other(): + """Two threads in different blocks see different keys. + + This is the concurrency complaint in #352: ``os.environ`` is + process-global, so it cannot express this. + """ + seen: dict[str, str | None] = {} + started = threading.Barrier(2) + + def worker(name: str, key: str) -> None: + with dataretrieval.configure(Settings(api_key=key)): + started.wait(timeout=5) # force the blocks to overlap in time + seen[name] = settings.api_key() + + threads = [ + threading.Thread(target=worker, args=("a", "key-a")), + threading.Thread(target=worker, args=("b", "key-b")), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + assert seen == {"a": "key-a", "b": "key-b"} + + +def test_asyncio_tasks_do_not_leak_credentials_into_each_other(): + """Concurrent asyncio tasks each keep their own key.""" + + async def worker(key: str) -> str | None: + with dataretrieval.configure(Settings(api_key=key)): + await asyncio.sleep(0) # yield, letting the other task interleave + return settings.api_key() + + async def main() -> list[str | None]: + return list(await asyncio.gather(worker("key-a"), worker("key-b"))) + + assert asyncio.run(main()) == ["key-a", "key-b"] + + +# --- the file ------------------------------------------------------------ + + +def test_named_profile_is_selected_in_code(config_file): + """``[.]`` reaches the chain only when a caller loads it.""" + config_file( + 'api_key = "shared"\nconcurrency = 4\n\n' + "[waterdata]\nretries = 2\n\n" + '[waterdata.bulk]\nconcurrency = "unbounded"\n' + ) + + # Inert until selected: the file alone changes nothing about concurrency. + assert settings.concurrency(adapter="waterdata") == 4 + + with dataretrieval.configure(WaterdataSettings.load("bulk")): + assert settings.concurrency(adapter="waterdata") is None # the profile + assert settings.retries(adapter="waterdata") == 2 # default profile + assert settings.api_key() == "shared" # package-wide, from the file + # It narrows to one adapter, so a sibling on the same host is untouched. + assert settings.concurrency(adapter="ngwmn") == 4 + + assert settings.concurrency(adapter="waterdata") == 4 + + +def test_a_code_selected_profile_outranks_the_environment(config_file, monkeypatch): + """ADR 0011 inverts ADR 0009's environment-above-file rule for this case. + + A profile named in code is a more deliberate act than a variable inherited + from a shell, and losing to that variable is what a caller would file a bug + about. + """ + config_file("[waterdata.gentle]\nconcurrency = 2\n") + monkeypatch.setenv("API_USGS_CONCURRENT", "16") + + assert settings.concurrency(adapter="waterdata") == 16 + with dataretrieval.configure(WaterdataSettings.load("gentle")): + assert settings.concurrency(adapter="waterdata") == 2 + + +def test_several_named_profiles_are_selected_independently(config_file): + """One block, two adapters, a different named profile for each.""" + config_file( + "[waterdata.bulk]\nconcurrency = 32\n\n" + "[waterdata.polite]\nconcurrency = 2\n\n" + "[ngwmn.gentle]\nconcurrency = 4\n" + ) + + with dataretrieval.configure( + WaterdataSettings.load("polite"), NgwmnSettings.load("gentle") + ): + assert settings.concurrency(adapter="waterdata") == 2 + assert settings.concurrency(adapter="ngwmn") == 4 + + +def test_a_named_profile_layers_per_key_over_the_tiers_below(config_file): + """Selecting a profile replaces keys, never whole tiers. + + Every level of the file overrides the one below it *per key* (ADR 0011), so + one adapter-scoped read here draws each of its four settings from a + different table. + """ + config_file( + "concurrency = 16\nretries = 3\nstall_timeout = 30\n\n" + "[waterdata]\nretries = 2\n\n" + '[waterdata.bulk]\nconcurrency = "unbounded"\nparallel_chunks = 8\n' + ) + + with dataretrieval.configure(WaterdataSettings.load("bulk")): + # the profile, over a package-wide key it names... + assert settings.concurrency(adapter="waterdata") is None + # ...the default profile, over a package-wide key the profile is silent + # about... + assert settings.retries(adapter="waterdata") == 2 + # ...the package-wide key, which neither table touched... + assert settings.stall_timeout(adapter="waterdata") == 30 + # ...and a setting only the profile names. + assert settings.parallel_chunks(adapter="waterdata") == 8 + + +def _resolved_settings() -> dict[object, object]: + """Every setting this process can resolve, package-wide and per adapter. + + A snapshot rather than a handful of assertions, because the claim under + test is about what a file does *not* change -- and naming the settings + individually would only prove it for the ones the author thought of. + """ + snapshot: dict[object, object] = { + "api_key": settings.api_key(), + "progress": settings.progress(), + } + for adapter in (None, *settings.ADAPTERS): + snapshot[(adapter, "concurrency")] = settings.concurrency(adapter=adapter) + snapshot[(adapter, "retries")] = settings.retries(adapter=adapter) + snapshot[(adapter, "parallel_chunks")] = settings.parallel_chunks( + adapter=adapter + ) + snapshot[(adapter, "stall_timeout")] = settings.stall_timeout(adapter=adapter) + snapshot[(adapter, "base_url")] = settings.base_url(adapter=adapter) + return snapshot + + +def test_adding_a_named_profile_changes_nothing_until_it_is_selected(config_file): + """Inertness is what makes a profile safe to add to a file others share. + + A named profile that could shift a setting on its own would make every + addition to a shared ``config.toml`` a change to every script reading it, + which is the failure the retired global ``[profiles.]`` table had. + """ + shared = 'api_key = "shared"\nconcurrency = 4\n\n[waterdata]\nretries = 2\n' + config_file(shared) + before = _resolved_settings() + + config_file( + shared + '\n[waterdata.bulk]\nconcurrency = "unbounded"\n' + "retries = 9\nparallel_chunks = 8\nstall_timeout = 5\n" + ) + assert _resolved_settings() == before + + # ...and the profile does reach the chain once it is named in code, so the + # comparison above is inertness rather than a profile nothing can select. + with dataretrieval.configure(WaterdataSettings.load("bulk")): + assert settings.parallel_chunks(adapter="waterdata") == 8 + + +def test_a_named_profile_cannot_hold_a_nested_table(config_file): + """``[waterdata.bulk.ngwmn]`` is the retired shape, not a deeper profile. + + A profile carries settings for the one adapter it belongs to, so a table + inside one has no reading. Refused rather than skipped: silently dropping + it would leave the author believing they had tuned NGWMN. + """ + config_file( + "[waterdata.bulk]\nparallel_chunks = 8\n\n" + "[waterdata.bulk.ngwmn]\nconcurrency = 2\n" + ) + + # Still inert, like every other problem inside an unselected profile: an + # unrelated call resolves without ever reading it. + assert settings.retries(adapter="ngwmn") == settings.DEFAULT_RETRIES + assert settings.parallel_chunks(adapter="waterdata") == ( + settings.DEFAULT_PARALLEL_CHUNKS + ) + + with pytest.raises( + settings.ConfigurationError, match=r"\[waterdata\.bulk\.ngwmn\]" + ): + WaterdataSettings.load("bulk") + + +def test_loading_an_undefined_profile_raises(config_file): + """A name the caller just typed is a typo, not a silent fall-through. + + The message lists what the file *does* define, because a misspelling is + only obvious next to the spelling that was meant -- and only for this + adapter, since selecting a profile is per adapter and another service's + profile names are not candidates for what the caller meant to type. + """ + config_file( + "[waterdata]\nconcurrency = 4\n\n" + "[waterdata.bulk]\nretries = 8\n\n" + "[waterdata.polite]\nretries = 1\n\n" + "[ngwmn.gentle]\nconcurrency = 2\n" + ) + with pytest.raises(settings.ConfigurationError) as excinfo: + WaterdataSettings.load("bluk") + message = str(excinfo.value) + assert "no [waterdata.bluk]" in message + assert "bulk, polite" in message + assert "gentle" not in message + + # An adapter with no profiles at all says so rather than trailing off after + # the colon, which would read as a truncated message. + config_file("[waterdata]\nconcurrency = 4\n") + with pytest.raises(settings.ConfigurationError, match="waterdata: none"): + WaterdataSettings.load("bulk") + + +def test_loading_a_profile_with_no_file_says_so(tmp_path, monkeypatch): + monkeypatch.delenv("API_USGS_CONCURRENT") # pinned by the autouse fixture + monkeypatch.setenv(settings.CONFIG_PATH_ENV, str(tmp_path / "absent.toml")) + settings._reset_file_cache() + + with pytest.raises(settings.ConfigurationError, match="no settings file"): + WaterdataSettings.load("also-gone") + + +def test_the_package_wide_configuration_has_no_profiles(config_file): + """A profile belongs to one adapter, so ``Settings`` cannot name one.""" + config_file("[waterdata.bulk]\nconcurrency = 8\n") + with pytest.raises(settings.ConfigurationError, match="package-wide"): + Settings.load("bulk") + + +def test_missing_file_is_not_an_error(tmp_path, monkeypatch): + monkeypatch.delenv("API_USGS_CONCURRENT") # pinned by the autouse fixture + monkeypatch.setenv(settings.CONFIG_PATH_ENV, str(tmp_path / "absent.toml")) + settings._reset_file_cache() + assert settings.concurrency() == settings.DEFAULT_CONCURRENCY + + +def test_malformed_file_raises_pointing_at_the_file(config_file): + path = config_file("api_key = \n") + with pytest.raises(settings.ConfigurationError) as excinfo: + settings.api_key() + assert "not valid TOML" in str(excinfo.value) + assert str(path) in str(excinfo.value) + + +def test_non_utf8_file_raises_config_error(config_file): + path = config_file("") + path.write_bytes(b'api_key = "\xff"\n') + with pytest.raises(settings.ConfigurationError, match="not valid UTF-8"): + settings.api_key() + + +def test_config_path_must_not_be_a_directory(tmp_path, monkeypatch): + monkeypatch.setenv(settings.CONFIG_PATH_ENV, str(tmp_path)) + monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) + settings._reset_file_cache() + with pytest.raises(settings.ConfigurationError, match="directory"): + settings.concurrency() + + +@pytest.mark.skipif(os.name != "posix", reason="needs /dev/null") +def test_dev_null_config_path_means_no_configuration(monkeypatch): + """``DATARETRIEVAL_CONFIG=/dev/null`` is how a run isolates itself. + + A character device (or the FIFO from process substitution) reads as empty, + which is exactly "no configuration". Rejecting it would raise from + ``_default_headers`` on every request -- the opposite of what the caller + asked for. + """ + monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) + monkeypatch.delenv("API_USGS_PAT", raising=False) + monkeypatch.setenv(settings.CONFIG_PATH_ENV, "/dev/null") + settings._reset_file_cache() + assert settings.api_key() is None + assert settings.concurrency() == settings.DEFAULT_CONCURRENCY + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX directory permissions") +def test_inaccessible_config_path_raises(tmp_path, monkeypatch): + parent = tmp_path / "blocked" + parent.mkdir() + path = parent / "config.toml" + path.write_text("concurrency = 4\n") + parent.chmod(0) + monkeypatch.setenv(settings.CONFIG_PATH_ENV, str(path)) + monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) + settings._reset_file_cache() + try: + try: + path.stat() + except PermissionError: + pass + else: # pragma: no cover - root or a filesystem that ignores mode bits + pytest.skip("filesystem does not enforce directory mode bits") + with pytest.raises(settings.ConfigurationError, match="could not access"): + settings.concurrency() + finally: + parent.chmod(0o700) + + +def test_unknown_setting_warns_but_is_ignored(config_file): + config_file('concurrency = 4\napi_kye = "typo"\n') + with pytest.warns(UserWarning, match="unknown setting"): + assert settings.concurrency() == 4 + + +def test_unknown_table_raises(config_file): + """A profile written as ``[bulk]`` instead of ``[waterdata.bulk]``.""" + config_file("[bulk]\nconcurrency = 4\n") + with pytest.raises(settings.ConfigurationError, match="unknown table"): + settings.concurrency() + + +def test_the_retired_profiles_table_names_its_replacement(config_file): + """Nothing shipped with ``[profiles.]``, but the docs described it. + + The generic "unknown table" message would send its author hunting for a + typo in a table spelled exactly as they had been told to spell it. + """ + config_file("[profiles.bulk]\nconcurrency = 4\n") + with pytest.raises(settings.ConfigurationError, match=r"\[\.\]"): + settings.concurrency() + + +def test_the_retired_profile_environment_variable_is_ignored(config_file, monkeypatch): + """``DATARETRIEVAL_PROFILE`` went with the table it selected (ADR 0011). + + A profile is now named in code. A variable exported once in a shell profile + and inherited by every subprocess is the opposite shape: invisible at the + call site, and able to switch every service at once. Honoring it under the + new grammar would restore exactly what the grammar removed. + """ + config_file('concurrency = 4\n\n[waterdata.bulk]\nconcurrency = "unbounded"\n') + monkeypatch.setenv("DATARETRIEVAL_PROFILE", "bulk") + + assert settings.concurrency(adapter="waterdata") == 4 + assert "DATARETRIEVAL_PROFILE" not in settings.ENV_VARS.values() + + +def test_typed_toml_values_are_normalized(config_file): + """``tomllib`` returns typed values that normalize into shared parsers.""" + config_file("concurrency = 16\nretries = 0\nprogress = true\n") + assert settings.concurrency() == 16 + assert settings.retries() == 0 + assert settings.progress() is True + + +@pytest.mark.parametrize( + "text", + [ + "api_key = true\n", + 'concurrency = "8"\n', + 'retries = "2"\n', + "progress = 17\n", + "parallel_chunks = true\n", + ], +) +def test_toml_rejects_wrong_scalar_types(config_file, text): + config_file(text) + with pytest.raises(settings.ConfigurationError): + settings.parallel_chunks() + + +def test_file_edit_is_picked_up(config_file, monkeypatch): + path = config_file("concurrency = 4\n") + assert settings.concurrency() == 4 + original = path.stat() + path.write_text("concurrency = 8\n") + os.utime(path, ns=(original.st_atime_ns, original.st_mtime_ns)) + # Windows ctime is creation time, so unchanged metadata must fall back to + # comparing raw content before the parsed cache is reused. + monkeypatch.setattr(settings.os, "name", "nt") + assert settings.concurrency() == 8 + + +def test_explicit_config_path_is_expanded(monkeypatch): + monkeypatch.setenv(settings.CONFIG_PATH_ENV, "~/somewhere/config.toml") + assert str(settings.config_path()).startswith(os.path.expanduser("~")) + assert "~" not in str(settings.config_path()) + + +def test_relative_config_path_follows_the_working_directory(tmp_path, monkeypatch): + """A relative ``DATARETRIEVAL_CONFIG`` is resolved against the *current* cwd. + + The path memo keys on the working directory for exactly this reason: a + scheduler or notebook that sets a relative path and chdirs per job would + otherwise keep serving the first job's credentials for the life of the + process, with ``show_settings()`` reporting the stale path as current. + """ + first = tmp_path / "first" + second = tmp_path / "second" + first.mkdir() + second.mkdir() + (first / "config.toml").write_text("concurrency = 4\n") + (second / "config.toml").write_text("concurrency = 9\n") + monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) + monkeypatch.setenv(settings.CONFIG_PATH_ENV, "config.toml") + + monkeypatch.chdir(first) + settings._reset_file_cache() + assert settings.config_path() == first / "config.toml" + assert settings.concurrency() == 4 + + monkeypatch.chdir(second) + assert settings.config_path() == second / "config.toml" + assert settings.concurrency() == 9 + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX file modes") +def test_world_readable_file_with_a_key_warns(tmp_path, monkeypatch): + path = tmp_path / "config.toml" + path.write_text('api_key = "secret"\n') + path.chmod(0o644) + monkeypatch.setenv(settings.CONFIG_PATH_ENV, str(path)) + monkeypatch.delenv("API_USGS_PAT", raising=False) + settings._reset_file_cache() + with pytest.warns(UserWarning, match="readable by other users"): + assert settings.api_key() == "secret" + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX file modes") +def test_permission_change_is_checked_on_cached_file(config_file): + path = config_file('api_key = "secret"\n') + assert settings.api_key() == "secret" + path.chmod(0o644) + with pytest.warns(UserWarning, match="readable by other users"): + assert settings.api_key() == "secret" + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX file modes") +def test_no_permission_warning_without_a_key(tmp_path, monkeypatch, recwarn): + path = tmp_path / "config.toml" + path.write_text("concurrency = 4\n") + path.chmod(0o644) + monkeypatch.setenv(settings.CONFIG_PATH_ENV, str(path)) + monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) + settings._reset_file_cache() + assert settings.concurrency() == 4 + assert not [w for w in recwarn if "readable by other users" in str(w.message)] + + +# --- value grammar ------------------------------------------------------- + + +def test_api_key_is_stripped_and_blank_means_none(monkeypatch): + monkeypatch.setenv("API_USGS_PAT", " key-with-newline\n") + assert settings.api_key() == "key-with-newline" + monkeypatch.setenv("API_USGS_PAT", " ") + assert settings.api_key() is None + + +def test_blank_numeric_env_falls_back_to_the_default(monkeypatch): + monkeypatch.setenv("API_USGS_CONCURRENT", "") + monkeypatch.setenv("API_USGS_RETRIES", "") + assert settings.concurrency() == settings.DEFAULT_CONCURRENCY + assert settings.retries() == settings.DEFAULT_RETRIES + + +def test_blank_progress_env_means_off_not_unset(monkeypatch): + """Preserved from the pre-config behavior: blank disables the line.""" + monkeypatch.setenv("API_USGS_PROGRESS", "") + assert settings.progress() is False + + +@pytest.mark.parametrize("value", ["0", "false", "no", "off", "FALSE"]) +def test_progress_falsey_values(monkeypatch, value): + monkeypatch.setenv("API_USGS_PROGRESS", value) + assert settings.progress() is False + + +@pytest.mark.parametrize("value", ["1", "true", "yes", "on"]) +def test_progress_truthy_values(monkeypatch, value): + monkeypatch.setenv("API_USGS_PROGRESS", value) + assert settings.progress() is True + + +def test_legacy_unknown_progress_env_still_means_on(monkeypatch): + monkeypatch.setenv("API_USGS_PROGRESS", "legacy-nonempty-value") + assert settings.progress() is True + + +@pytest.mark.parametrize("value", ["nope", "-1", "0"]) +def test_invalid_concurrency_raises(monkeypatch, value): + monkeypatch.setenv("API_USGS_CONCURRENT", value) + with pytest.raises(ValueError): # ConfigurationError is a ValueError + settings.concurrency() + + +def test_unbounded_concurrency(monkeypatch): + monkeypatch.setenv("API_USGS_CONCURRENT", "unbounded") + assert settings.concurrency() is None + + +def test_error_message_names_the_source(config_file, monkeypatch): + monkeypatch.setenv("API_USGS_CONCURRENT", "nope") + with pytest.raises(settings.ConfigurationError, match=r"\$?API_USGS_CONCURRENT"): + settings.concurrency() + monkeypatch.delenv("API_USGS_CONCURRENT") + path = config_file('concurrency = "nope"\n') + # ``match`` is a regex, and a Windows path is mostly escapes: + # ``C:\\Users\\...`` makes ``\\U`` an invalid escape. + with pytest.raises(settings.ConfigurationError, match=re.escape(str(path))): + settings.concurrency() + + +# --- security ------------------------------------------------------------ + + +def test_show_config_never_prints_the_key(monkeypatch): + monkeypatch.setenv("API_USGS_PAT", "super-secret-value") + out = io.StringIO() + dataretrieval.show_settings(stream=out) + text = out.getvalue() + assert "super-secret-value" not in text + assert "" in text + assert "$API_USGS_PAT" in text # provenance is still reported + + +def test_show_config_reports_absent_key(monkeypatch): + monkeypatch.delenv("API_USGS_PAT", raising=False) + out = io.StringIO() + dataretrieval.show_settings(stream=out) + assert "" in out.getvalue() + + +def test_file_sourced_key_is_still_host_scoped(config_file): + """A key from a file gets the same host scoping as one from the env.""" + config_file('api_key = "file-key"\n') + assert _default_headers(WATERDATA_URL)["X-Api-Key"] == "file-key" + assert "X-Api-Key" not in _default_headers("https://example.com/data") + assert "X-Api-Key" not in _default_headers( + "https://api.waterdata.usgs.gov.evil.com/x" + ) + + +def test_block_sourced_key_is_still_host_scoped(): + with dataretrieval.configure(Settings(api_key="block-key")): + assert _default_headers(WATERDATA_URL)["X-Api-Key"] == "block-key" + assert "X-Api-Key" not in _default_headers("https://example.com/data") + + +def test_no_public_getter_accepts_a_credential_parameter(): + """Guards the ``**queryables`` catch-all. + + Every Water Data getter forwards unknown keywords as OGC query + parameters, so a getter that grew an ``api_key`` or ``session`` + parameter could serialize a credential into a URL. Credentials must + arrive through ``dataretrieval.configure`` instead. + """ + import inspect + + from dataretrieval import waterdata + + offenders = [] + for name in waterdata.__all__: + obj = getattr(waterdata, name) + if not callable(obj) or inspect.isclass(obj): + continue + try: + params = inspect.signature(obj).parameters + except (TypeError, ValueError): # pragma: no cover - builtins + continue + for forbidden in ("api_key", "session", "token", "apikey"): + if forbidden in params: + offenders.append(f"{name}({forbidden}=)") + assert not offenders, ( + "public getters must not take credential parameters: " + ", ".join(offenders) + ) + + +@pytest.mark.parametrize("allowed", ["session", "session_id", "sampling_session"]) +def test_session_is_not_treated_as_a_credential(allowed): + """``session`` carries no secret, and the queryable namespace is the + server's — a substring rule would make any future field containing it + unreachable behind a credentials message that misstates the problem.""" + from dataretrieval.waterdata.utils import _flatten_queryables + + assert _flatten_queryables({"queryables": {allowed: 1}}) == {allowed: 1} + + +@pytest.mark.parametrize( + "forbidden", + ["api_key", "apikey", "apiKey", "API_KEY", "api-key", "token"], +) +def test_credential_keyword_cannot_enter_queryables(forbidden): + from dataretrieval import waterdata + + with pytest.raises(TypeError, match=forbidden): + waterdata.get_daily( + monitoring_location_id="USGS-01646500", **{forbidden: "secret"} + ) + + +# --- wiring into the rest of the package --------------------------------- + + +def test_retry_policy_reads_the_block(): + from dataretrieval.transport.retry import RetryPolicy + + with dataretrieval.configure(Settings(retries=3)): + assert RetryPolicy.from_settings().max_retries == 3 + + +def test_parallel_chunks_baseline_comes_from_config(config_file): + from dataretrieval.ogc.chunking import parallel_chunks + + assert settings.parallel_chunks() == 1 + config_file("parallel_chunks = 8\n") + assert settings.parallel_chunks() == 8 + with parallel_chunks(2): # an explicit block still wins over the file + assert settings.parallel_chunks() == 2 + assert settings.parallel_chunks() == 8 + + +def test_parallel_chunks_and_configure_share_one_mechanism(): + """``parallel_chunks(n)`` is sugar for a package-wide ``Settings``. + + They must not be two competing scopes: whichever block is innermost wins, + so ``show_settings()`` always reports the value the chunker will use. + """ + from dataretrieval.ogc.chunking import parallel_chunks + + with parallel_chunks(2): + with dataretrieval.configure(Settings(parallel_chunks=8)): + assert settings.parallel_chunks() == 8 + assert settings.parallel_chunks() == 2 + + with dataretrieval.configure(Settings(parallel_chunks=8)): + with parallel_chunks(2): + assert settings.parallel_chunks() == 2 + assert settings.parallel_chunks() == 8 + + +def test_parallel_chunks_has_no_environment_variable(): + """It spends quota, so it is deliberately file/block-only (see ENV_VARS).""" + assert "parallel_chunks" not in settings.ENV_VARS + assert "parallel_chunks" in settings.SETTINGS + + +def test_progress_reporter_reads_the_block(): + from dataretrieval.progress import ProgressReporter + + with dataretrieval.configure(Settings(progress=True)): + assert ProgressReporter(stream=io.StringIO()).enabled + with dataretrieval.configure(Settings(progress=False)): + assert not ProgressReporter(stream=io.StringIO()).enabled + + +# --- review regressions -------------------------------------------------- + + +def test_blank_env_does_not_mask_the_config_file(config_file, monkeypatch): + """A blank-but-set env var must not shadow a configured file. + + Container and CI tooling routinely materializes one (``docker run -e + API_USGS_PAT`` with nothing to pass, a workflow secret absent on a fork). + Letting that outrank the file silently dropped the API key and sent every + request unauthenticated. + """ + config_file('api_key = "file-key"\nconcurrency = 4\nretries = 7\nprogress = true\n') + for env in settings.ENV_VARS.values(): + monkeypatch.setenv(env, "") + + assert settings.api_key() == "file-key" + assert settings.concurrency() == 4 + assert settings.retries() == 7 + # ``progress`` is the documented exception: a blank API_USGS_PROGRESS has + # always meant "off", so for that setting blank *is* a value and outranks + # the file. The asymmetry is declared once, in settings._BLANK_MEANS_SET. + assert settings.progress() is False + assert set(settings._BLANK_MEANS_SET) == {"progress"} + + +def test_blank_progress_env_keeps_its_legacy_meaning(monkeypatch): + """With no file, blank keeps the environment-only meaning it always had.""" + monkeypatch.setenv("API_USGS_PROGRESS", "") + monkeypatch.setenv("API_USGS_CONCURRENT", "") + assert settings.progress() is False # blank has always meant "off" + assert settings.concurrency() == settings.DEFAULT_CONCURRENCY + + +def test_config_error_is_in_the_error_taxonomy(): + """A broken config surfaces from inside a getter, so it must be catchable.""" + import dataretrieval.exceptions as exceptions + + assert issubclass(settings.ConfigurationError, exceptions.DataRetrievalError) + assert issubclass( + settings.ConfigurationError, ValueError + ) # legacy handlers still work + assert settings.ConfigurationError is exceptions.ConfigurationError + + +def test_show_config_reports_a_broken_file_instead_of_raising(config_file): + """The tool that explains a configuration must survive a broken one.""" + config_file("this is not = valid toml [[[\n") + out = io.StringIO() + dataretrieval.show_settings(stream=out) # must not raise + text = out.getvalue() + assert "ERROR:" in text + # Every setting still gets a row rather than the report dying part-way. + for name in settings.SETTINGS: + assert name in text + + +def test_show_config_reports_a_bad_value_in_its_own_row(monkeypatch): + monkeypatch.setenv("API_USGS_CONCURRENT", "nope") + out = io.StringIO() + dataretrieval.show_settings(stream=out) + text = out.getvalue() + assert "= {"retries", "stall_timeout", "base_url"} + + +def test_registering_an_adapter_outside_the_roster_raises(): + """The roster is the authority, so a class cannot invent an adapter.""" + + @dataclass(frozen=True) + class BogusSettings(settings.AdapterSettings): + adapter: ClassVar[str] = "not-an-adapter" + + with pytest.raises(settings.ConfigurationError, match="not one of"): + settings._register(BogusSettings) + + +def test_settings_for_an_unimported_adapter_is_not_an_error(monkeypatch): + """``None`` means "cannot validate these keys yet", never "invalid". + + NLDI is imported on demand for the geopandas extra, so a roster built from + imports would reject a perfectly good ``[nldi]`` table until something + happened to import that module. + """ + monkeypatch.delitem(settings._REGISTRY, "nldi", raising=False) + assert settings.settings_for("nldi") is None + assert "nldi" in settings.ADAPTERS + + +def test_every_adapter_is_actually_wired_to_a_read_site(): + """A schema nothing passes is worse than no schema. + + ``show_settings()`` would report a ``[nwis]`` override as live while + every call ignored it -- the report whose whole job is answering "what will + this call use" being confidently wrong. Importability is the weaker half of + the invariant: it passed while ``waterdata.get_cql``, eight of nine WQP + getters, and all of ``nwis`` silently resolved package-wide. + """ + import pathlib + + source = "\n".join( + p.read_text(encoding="utf-8") + for p in pathlib.Path(settings.__file__).parent.rglob("*.py") + if p.name != "settings.py" + ) + missing = [a for a in settings.ADAPTERS if f'adapter="{a}"' not in source] + assert not missing, ( + f"adapters with a schema but no read site: {missing}. Either pass " + 'adapter="" where that adapter builds its policy or fan-out, or ' + "drop it from settings.ADAPTERS." + ) + + +def test_a_misspelled_adapter_at_a_read_site_raises(): + """The other half of the invariant above, which a grep cannot check. + + ``adapter="waterdatas"`` used to resolve *silently* package-wide: no table + matches the typo, every setting is accepted because nothing knows the + schema, and a ``[waterdata]`` table or a ``WaterdataSettings`` is then + ignored with nothing raised anywhere. The grep only sees that the correctly + spelled string occurs somewhere; it cannot see a second, wrong one. + """ + with pytest.raises(settings.ConfigurationError, match="not a configurable"): + settings.retries(adapter="waterdatas") + + # Every read site funnels through one resolver, so the check reaches them + # all -- including the accessors that would otherwise return a default. + with pytest.raises(settings.ConfigurationError, match="not a configurable"): + settings.base_url(adapter="nwis", default="https://example.invalid") + + +def test_a_non_finite_stall_timeout_is_refused(): + """``inf`` parses as a float and silently disables the bound it sets.""" + for bad in (float("inf"), float("nan")): + with pytest.raises(settings.ConfigurationError, match="finite"): + Settings(stall_timeout=bad) + + +def test_stall_timeout_resolves_through_the_chain(config_file, monkeypatch): + """It was read straight from os.environ, so a block and the file were mute.""" + config_file("stall_timeout = 15\n\n[wqp]\nstall_timeout = 300\n") + + assert settings.stall_timeout() == 15 + assert settings.stall_timeout(adapter="wqp") == 300 + + monkeypatch.setenv("API_USGS_STALL_TIMEOUT", "42") + assert settings.stall_timeout() == 42 + + with dataretrieval.configure(Settings(stall_timeout=2.5)): + assert settings.stall_timeout() == 2.5 + + +def test_base_url_applies_from_code_and_is_refused_from_the_file(config_file): + """A redirect belongs where a reader of the script sees it (ADR 0011). + + A configuration file that silently sent a data-retrieval library to another + host would be a supply-chain-shaped hazard, so the file refuses the setting + outright rather than accepting it and being trusted. + """ + config_file("") + + with dataretrieval.configure( + WaterdataSettings(base_url="https://mirror.example/ogcapi") + ): + assert settings.base_url(adapter="waterdata") == ( + "https://mirror.example/ogcapi" + ) + # It names one service, so it never reaches another. + assert settings.base_url(adapter="ngwmn") is None + assert settings.base_url(adapter="waterdata") is None + + for text in ( + 'base_url = "https://evil.example"\n', + "[waterdata]\nbase_url = 'x'\n", + ): + config_file(text) + with pytest.raises(settings.ConfigurationError, match="only be set in code"): + settings.base_url(adapter="waterdata") + + +def test_base_url_must_be_an_absolute_http_url(): + """A bare host would fail far from here, inside the request builder.""" + with pytest.raises(settings.ConfigurationError, match="absolute"): + WaterdataSettings(base_url="mirror.example") + with pytest.raises(settings.ConfigurationError, match="absolute"): + WaterdataSettings(base_url="file:///etc/passwd") + + +def test_base_url_is_refused_from_the_environment(monkeypatch): + """The environment is refused out loud, not merely unread. + + ``API_USGS_BASE_URL`` is the spelling every other setting's variable + predicts, so a caller who exports it believes they have redirected + something. Leaving it out of ``ENV_VARS`` would make that belief wrong and + silent; the error names the block to write instead. + """ + monkeypatch.setenv("API_USGS_BASE_URL", "https://evil.example") + + with pytest.raises(settings.ConfigurationError, match="only be set in code"): + settings.base_url(adapter="waterdata") + + # Refused even under a block that sets one, matching the file: the variable + # cannot work, and being quietly outranked is how it survives to a run where + # nothing outranks it. Unsetting it is the only fix. + with dataretrieval.configure(WaterdataSettings(base_url=_MIRROR)): + with pytest.raises(settings.ConfigurationError, match="only be set in code"): + settings.base_url(adapter="waterdata") + + # A configuration in this state is exactly what show_settings() exists + # to explain, so it reports the failure rather than raising out of it. + out = io.StringIO() + dataretrieval.show_settings(stream=out) + assert "only be set in code" in out.getvalue() + + +def test_a_water_data_redirect_moves_every_endpoint_family(): + """Water Data is one adapter serving four APIs, so all four move together. + + A redirect that reached the OGC collections but left samples, statistics, + and ratings on the service's own host would send most of a caller's traffic + to the host they were redirecting away from -- the one mistake a redirect + must not make. + """ + with dataretrieval.configure(WaterdataSettings(base_url=_MIRROR)): + moved = { + name: endpoints.redirected(getattr(endpoints, name)) + for name in ("OGC_API_URL", "SAMPLES_URL", "STATISTICS_API_URL", "STAC_URL") + } + + assert moved == { + "OGC_API_URL": f"{_MIRROR}/ogcapi/v0", + "SAMPLES_URL": f"{_MIRROR}/samples-data", + "STATISTICS_API_URL": f"{_MIRROR}/statistics/v0", + "STAC_URL": f"{_MIRROR}/stac/v0", + } + # Outside the block the constants are the service's own again. + assert endpoints.redirected(endpoints.OGC_API_URL) == endpoints.OGC_API_URL + + # The swap is a prefix swap, so an endpoint declared on some other root + # would be rewritten into nonsense rather than moved. Derived from the + # module's exports so a fifth family added later is covered the day it + # lands, which the four literals above cannot be. + declared = [n for n in endpoints.__all__ if n.endswith("_URL") and n != "BASE_URL"] + assert declared and all( + getattr(endpoints, name).startswith(endpoints.BASE_URL) for name in declared + ) + + +def test_every_water_data_endpoint_use_goes_through_redirected(): + """``redirected()`` is a wrap-at-every-use-site seam, so check every site. + + Water Data is the one adapter that cannot resolve its base at a single + choke point: four families hang off one root, each building its own URL + from a constant. The test above proves the constants all derive from + ``BASE_URL``; this one proves the *use sites* actually pass them through + the wrapper. Without it a new family module -- or a second use of an + existing constant -- would send traffic to api.waterdata.usgs.gov from + inside a block a caller opened precisely to avoid it, with nothing failing: + what ``redirected()``'s own docstring calls the one mistake a redirect must + not make. + + Written as an AST scan for the same reason as + ``test_every_adapter_is_actually_wired_to_a_read_site``: the invariant is a + fact about the source, and nothing at runtime can observe a use site that + was simply never written. + """ + import ast + import pathlib + + wrapped = {n for n in endpoints.__all__ if n.endswith("_URL")} + package = pathlib.Path(endpoints.__file__).parent + + bare: list[str] = [] + for path in sorted(package.rglob("*.py")): + if path.name == "endpoints.py": + continue # where the constants are declared and the wrapper lives + tree = ast.parse(path.read_text(encoding="utf-8")) + # Every ``redirected(X)`` argument is a legitimate use; anything else + # naming a constant is not. Collected first so the walk below can tell + # the two apart by node identity rather than by position. + allowed = { + node.args[0] + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "redirected" + and node.args + } + for node in ast.walk(tree): + # Loads only: the ``from ... import`` that binds the name and the + # ``__all__`` re-export (a string, not a Name) are not use sites. + if ( + isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Load) + and node.id in wrapped + and node not in allowed + ): + bare.append(f"{path.name}:{node.lineno}: {node.id}") + + assert not bare, ( + "Water Data endpoint constants used without redirected(): " + f"{bare}. Wrap each one -- redirected(OGC_API_URL) -- or the request " + "ignores WaterdataSettings(base_url=...)." + ) + + +def test_a_code_base_url_redirects_the_adapters_requests(httpx_mock): + """The setting has to move real traffic, not just resolve to a string. + + Two adapters with unrelated request machinery -- the OGC engine and a plain + one-shot GET -- because "the configuration reaches the request" is a claim + about each adapter's wiring, and one of them passing says nothing about the + other. + """ + httpx_mock.add_response(method=None, url=_MIRROR_RE, json=_DAILY_PAGE) + httpx_mock.add_response(method=None, url=_WATERDATA_RE, json=_DAILY_PAGE) + + with dataretrieval.configure(WaterdataSettings(base_url=_MIRROR)): + waterdata.get_daily(monitoring_location_id="USGS-05427718") + redirected_url = str(httpx_mock.get_requests()[-1].url) + + # Nothing configured: back to the service's own base, so the redirect is + # scoped to the block rather than latched somewhere at import. + waterdata.get_daily(monitoring_location_id="USGS-05427718") + direct_url = str(httpx_mock.get_requests()[-1].url) + + assert redirected_url.startswith(f"{_MIRROR}/ogcapi/v0/collections/daily/items") + assert direct_url.startswith(f"{endpoints.OGC_API_URL}/collections/daily/items") + + streamstats_mirror = "https://mirror.example/streamstats" + with dataretrieval.configure(StreamstatsSettings(base_url=streamstats_mirror)): + streamstats.download_workspace("workspace-id") + assert str(httpx_mock.get_requests()[-1].url).startswith( + f"{streamstats_mirror}/download" + ) + + +def test_a_redirected_adapter_is_not_sent_the_api_key(httpx_mock): + """The key is scoped to the host that honors it, and a mirror is not it. + + ``credentials.accepts_api_key`` is checked where the header is attached, so + a redirect needs no second rule to be safe -- but "needs no rule" is exactly + the kind of claim that stops being true silently, and the cost of it being + wrong is a credential handed to whatever host the block named. + """ + httpx_mock.add_response(method=None, url=_MIRROR_RE, json=_DAILY_PAGE) + httpx_mock.add_response(method=None, url=_WATERDATA_RE, json=_DAILY_PAGE) + + with dataretrieval.configure(Settings(api_key="secret")): + with dataretrieval.configure(WaterdataSettings(base_url=_MIRROR)): + waterdata.get_daily(monitoring_location_id="USGS-05427718") + redirected_request = httpx_mock.get_requests()[-1] + + # The same key, the same call, the service's own host: the control that + # keeps this test from passing because no key was configured at all. + waterdata.get_daily(monitoring_location_id="USGS-05427718") + direct_request = httpx_mock.get_requests()[-1] + + assert "X-Api-Key" not in redirected_request.headers + assert direct_request.headers["X-Api-Key"] == "secret" + + +def test_the_validate_hook_can_refuse_a_combination(): + """Per-setting grammar is shared with the file; this is for the rest. + + The hook is ``validate_settings`` rather than ``validate``: pydantic's + ``BaseModel`` already owns the latter name. + """ + + class Fussy(settings.AdapterSettings): + adapter: ClassVar[str] = "waterdata" + + concurrency: int | str | None = None + parallel_chunks: int | None = None + + def validate_settings(self) -> None: + supplied = self.values() + if supplied.get("parallel_chunks", 1) > supplied.get("concurrency", 1): + raise settings.ConfigurationError( + "parallel_chunks above concurrency only queues sub-requests." + ) + + assert Fussy(concurrency=8, parallel_chunks=4).settings() == { + "concurrency", + "parallel_chunks", + } + with pytest.raises(settings.ConfigurationError, match="only queues"): + Fussy(concurrency=2, parallel_chunks=8) + + +def test_show_settings_lists_only_real_overrides(config_file): + """A full adapter-by-setting grid would bury the answer in inherited rows.""" + config_file("concurrency = 16\n\n[ngwmn]\nconcurrency = 4\n") + out = io.StringIO() + + dataretrieval.show_settings(stream=out) + text = out.getvalue() + + assert "adapter overrides" in text + assert "ngwmn" in text + # waterdata inherits every setting, so it must not appear as an override. + override_section = text.split("adapter overrides", 1)[1] + assert "waterdata" not in override_section + assert "streamstats" not in override_section + + +def test_inner_block_can_lower_a_setting_an_outer_block_scoped(config_file): + """The innermost block wins across *both* scopes, not just within one. + + An adapter-scoped value is the more specific of two written by the same + block. It must not outrank one written by a block nested *inside* it, or + the documented recovery from QuotaExhausted -- wait, then re-issue more + gently -- cannot be expressed once any adapter table is in play. + """ + config_file("") + + with dataretrieval.configure(WaterdataSettings(concurrency=32)): + with dataretrieval.configure(Settings(concurrency=1)): + assert settings.concurrency(adapter="waterdata") == 1 + assert settings.concurrency(adapter="waterdata") == 32 + + +def test_adapter_scope_still_wins_within_one_block(config_file): + """Depth breaks ties between blocks, never within one.""" + config_file("") + + with dataretrieval.configure( + Settings(concurrency=16), WaterdataSettings(concurrency=4) + ): + assert settings.concurrency(adapter="waterdata") == 4 + assert settings.concurrency(adapter="wqp") == 16 + + +def test_parallel_chunks_block_survives_an_adapter_scoped_outer_block(): + """``parallel_chunks(n)`` is a per-call request and must not be discarded. + + It delegates to a package-wide ``Settings``, so it writes the + package-wide key -- and before blocks were kept as separate frames, any + enclosing ``WaterdataSettings(parallel_chunks=...)`` outranked it. + """ + from dataretrieval.waterdata import parallel_chunks + + with dataretrieval.configure(WaterdataSettings(parallel_chunks=2)): + with parallel_chunks(16): + assert settings.parallel_chunks(adapter="waterdata") == 16 + assert settings.parallel_chunks(adapter="waterdata") == 2 + + +# --- the precedence ladder (ADR 0011) ------------------------------------ +# +# ADR 0011 states the ladder in seven rungs, highest first: +# +# 1 a configuration instance passed to configure() +# 2 a profile selected in code, Settings.load("") +# 3 the setting's environment variable +# 4 the adapter's default profile in the file, [] +# 5 the package-wide keys at the top of the file +# 6 the adapter's built-in preference, passed by the adapter's own read site +# 7 the package built-in default +# +# The tests below walk it as a *chain*: each one knocks the rung above out and +# asserts the next takes over. Seven independent single-rung assertions would +# all still pass if two rungs collapsed into one, which is the mistake worth +# catching -- rungs 2 and 3 are the pair a refactor is most likely to fuse, +# since 2 above 3 is the one place ADR 0011 inverts ADR 0009. +# +# ``nwdc`` and ``concurrency`` are the pair that can express all seven. NWDC is +# the adapter that ships a built-in preference of its own -- 4 concurrent +# requests, because the service is only stress-tested that far -- distinct from +# the package default of 32, and that difference is the only way rungs 6 and 7 +# can be told apart at all. + +#: Rungs 5, 4 and 2, with a distinct value per rung so a resolved number +#: identifies the table it came from. Top-level keys are written first because +#: TOML assigns a bare key to whichever table header precedes it: moved below +#: ``[nwdc]``, ``concurrency = 15`` would quietly stop being a rung-5 key and +#: become a second rung-4 one, and the tests would still pass by coincidence. +_LADDER_FILE = ( + "concurrency = 15\n" # rung 5: the package-wide keys + "retries = 7\n" # rung 5 again, for a setting no profile names + "\n[nwdc]\n" + "concurrency = 14\n" # rung 4: the adapter's default profile + "\n[nwdc.tuned]\n" + "concurrency = 12\n" # rung 2: inert until load() selects it +) + +#: Rung 3, which is not in the file. +_LADDER_ENV = 13 + +#: Rung 1, which is not in the file either. +_LADDER_INSTANCE = 11 + + +def _nwdc_concurrency() -> int | None: + """Resolve ``concurrency`` the way NWDC's own fan-out does. + + Through the adapter's read site rather than a bare ``concurrency()``, so + the built-in preference at rung 6 is really in the chain and the ladder is + exercised as the adapter experiences it. + """ + return settings.concurrency(DEFAULT_CONCURRENT_REQUESTS, adapter="nwdc") + + +def test_a_configuration_instance_tops_the_ladder(config_file, monkeypatch): + """Rung 1 over every other rung, all six of them present at once.""" + config_file(_LADDER_FILE) + monkeypatch.setenv("API_USGS_CONCURRENT", str(_LADDER_ENV)) + + with dataretrieval.configure(NwdcSettings(concurrency=_LADDER_INSTANCE)): + assert _nwdc_concurrency() == _LADDER_INSTANCE + + +def test_a_loaded_profile_beats_the_environment(config_file, monkeypatch): + """Rung 2 over rung 3 -- the one inversion ADR 0011 exists to make. + + ADR 0009 put the environment above the file, and a named profile lives in + the file, so the naive reading is that ``API_USGS_CONCURRENT`` in the shell + wins. It does not: what reaches the chain is the caller *naming* the + profile in code, which is a more deliberate act than a variable inherited + from whatever started the process, and losing to that variable is the + behaviour a caller would file a bug about. + + The inversion is also bounded, which the second half asserts: it covers + what the profile names and nothing else, so ``retries`` -- which the file + sets at the top level and no selected profile mentions -- still follows the + original environment-above-file rule inside the very same block. + """ + config_file(_LADDER_FILE) + monkeypatch.setenv("API_USGS_CONCURRENT", str(_LADDER_ENV)) + monkeypatch.setenv("API_USGS_RETRIES", "9") + + assert _nwdc_concurrency() == _LADDER_ENV # rung 3, until a profile is selected + with dataretrieval.configure(NwdcSettings.load("tuned")): + assert _nwdc_concurrency() == 12 # rung 2 wins for the key it names... + assert settings.retries(adapter="nwdc") == 9 # ...and only that key + assert _nwdc_concurrency() == _LADDER_ENV # and the shell has it back on exit + + +def test_the_environment_beats_the_adapters_default_profile(config_file, monkeypatch): + """Rung 3 over rung 4: the file's always-on table is still just the file.""" + config_file(_LADDER_FILE) + monkeypatch.setenv("API_USGS_CONCURRENT", str(_LADDER_ENV)) + + assert _nwdc_concurrency() == _LADDER_ENV + monkeypatch.delenv("API_USGS_CONCURRENT") + assert _nwdc_concurrency() == 14 + + +def test_the_adapters_default_profile_beats_the_package_wide_keys(config_file): + """Rung 4 over rung 5: within the file, the narrower table decides.""" + config_file(_LADDER_FILE) + + assert _nwdc_concurrency() == 14 + config_file("concurrency = 15\n") # the [nwdc] table gone + assert _nwdc_concurrency() == 15 + + +def test_the_package_wide_keys_beat_the_adapters_built_in_preference(config_file): + """Rung 5 over rung 6: a user-written value outranks an adapter's taste. + + The adapter's preference is a default, not a cap. One able to override a + setting the user actually wrote would make that setting a lie -- so a + top-level key the user never scoped to NWDC still reaches NWDC's calls. + """ + config_file("concurrency = 15\n") + + assert _nwdc_concurrency() == 15 + config_file("") + assert _nwdc_concurrency() == DEFAULT_CONCURRENT_REQUESTS + + +def test_the_adapters_built_in_preference_beats_the_package_built_in_default( + config_file, +): + """Rung 6 over rung 7, and only for the adapter that stated a preference.""" + config_file("") + + assert _nwdc_concurrency() == DEFAULT_CONCURRENT_REQUESTS + assert DEFAULT_CONCURRENT_REQUESTS != settings.DEFAULT_CONCURRENCY + # It is the read site's own figure, not a property of the adapter, so a + # caller that states no preference lands on the package default instead -- + # which is what makes rungs 6 and 7 two rungs rather than one. + assert settings.concurrency(adapter="nwdc") == settings.DEFAULT_CONCURRENCY + + +def test_the_package_built_in_default_is_the_floor(config_file): + """Rung 7: with the six rungs above it empty, every setting still resolves. + + The floor is what makes the whole chain optional -- a caller who has + configured nothing at all gets working values rather than an error. + """ + config_file("") + + for adapter in (None, *settings.ADAPTERS): + assert settings.concurrency(adapter=adapter) == (settings.DEFAULT_CONCURRENCY) + assert settings.retries(adapter=adapter) == settings.DEFAULT_RETRIES + assert settings.parallel_chunks(adapter=adapter) == ( + settings.DEFAULT_PARALLEL_CHUNKS + ) + assert settings.stall_timeout(adapter=adapter) == ( + settings.DEFAULT_STALL_TIMEOUT + ) + + +def test_the_top_two_rungs_cannot_tie(config_file): + """Rungs 1 and 2 both target one adapter, so no block can hold both. + + That is what stops the ladder needing a tie-break nobody could remember: + the same-adapter rule refuses the pairing where the order would matter, + and between *nested* blocks the ordinary rule applies -- the innermost + decides, whichever kind of configuration it holds. + """ + config_file(_LADDER_FILE) + + with pytest.raises(settings.ConfigurationError, match="two settings profiles"): + with dataretrieval.configure( + NwdcSettings(concurrency=_LADDER_INSTANCE), + NwdcSettings.load("tuned"), + ): + pass + + with dataretrieval.configure(NwdcSettings(concurrency=_LADDER_INSTANCE)): + with dataretrieval.configure(NwdcSettings.load("tuned")): + assert _nwdc_concurrency() == 12 + with dataretrieval.configure(NwdcSettings.load("tuned")): + with dataretrieval.configure(NwdcSettings(concurrency=_LADDER_INSTANCE)): + assert _nwdc_concurrency() == _LADDER_INSTANCE + + # "Rung 1 above rung 2" is a claim about one adapter, so a *package-wide* + # instance is not the thing it is talking about: it targets no adapter at + # all. Alongside a loaded profile in one block the adapter-scoped value is + # the more specific of the two and wins for that adapter (ADR 0010), while + # the package-wide value still governs every other adapter. + with dataretrieval.configure( + Settings(concurrency=_LADDER_INSTANCE), NwdcSettings.load("tuned") + ): + assert _nwdc_concurrency() == 12 + assert settings.concurrency(adapter="wqp") == _LADDER_INSTANCE + + +def test_load_returns_an_instance_carrying_only_the_profiles_keys(config_file): + """``load`` is a constructor: it reads one table and returns the class. + + Only what the table names is carried, so every other setting stays unset + and keeps inheriting from the rungs below rather than being pinned to a + default the profile never asked for. That is what makes a profile a + *contribution* to the chain rather than a replacement for it. + """ + config_file( + "concurrency = 16\n\n" + "[waterdata]\nretries = 2\n\n" + '[waterdata.bulk]\nconcurrency = "unbounded"\nparallel_chunks = 8\n' + ) + + loaded = WaterdataSettings.load("bulk") + + assert isinstance(loaded, WaterdataSettings) + assert loaded.values() == {"concurrency": "unbounded", "parallel_chunks": 8} + # ``retries`` was not in the profile, so it stays unset and inherits the + # ``[waterdata]`` table below. ``model_fields_set`` is what carries that + # distinction now, so the field itself reads as ``None``. + assert "retries" not in loaded.model_fields_set + assert loaded.retries is None + + +# --- show_settings() reports profiles -------------------------------- +# +# The report exists to answer "why is this call using that value?", so every +# row names the source that supplied it. A value from a profile is the case a +# bare "configure() block" answers badly: a configuration written in code and +# one loaded from a table reach the chain by the same route, and only the +# latter has a name in a file the caller can go and read. + +#: The file the documented sample is generated from. Exercises every section: +#: package-wide keys, an adapter's default profile, and a named profile. +_SAMPLE_FILE = ( + 'api_key = "0123456789abcdef"\n' + "concurrency = 16\n" + "\n[ngwmn]\n" + "concurrency = 4\n" + "\n[waterdata.bulk]\n" + "parallel_chunks = 8\n" +) + +#: The illustrative path the samples print, standing in for the temporary file +#: the test actually writes. Substituting it is the *only* edit made to the +#: captured output -- everything else has to match what the function printed. +_SAMPLE_PATH = "/home/u/.dataretrieval/config.toml" + +#: The two lines above the captured output in both samples. +_SAMPLE_PROMPT = ( + '>>> with dataretrieval.configure(WaterdataSettings.load("bulk")):\n' + "... dataretrieval.show_settings()" +) + + +def _documented_sample() -> str: + """The sample output embedded in ``show_settings``'s docstring.""" + doc = inspect.getdoc(dataretrieval.show_settings) or "" + _, _, block = doc.partition(".. code-block:: text\n\n") + return textwrap.dedent(block).strip("\n") + + +def test_show_settings_names_the_profile_a_value_came_from(config_file): + """A value from a profile is reported with that profile, not with "a block". + + ``WaterdataSettings.load("bulk")`` and ``WaterdataSettings(...)`` + enter the chain by the same route and are indistinguishable once their + values are in the block, so a report that said only ``configure() block`` + left a caller who selected the wrong profile -- or who had forgotten a + profile was selected at all -- with nothing to look at. The label is the + table's own spelling, so it is greppable in the file that defines it. + """ + config_file("[waterdata.bulk]\nconcurrency = 6\n") + out = io.StringIO() + + with dataretrieval.configure(WaterdataSettings.load("bulk")): + dataretrieval.show_settings(stream=out) + + assert "configure() block [waterdata.bulk]" in out.getvalue() + + # A configuration written in code has no profile to name, so it names its + # adapter alone rather than inventing one -- and the package-wide one + # narrows to nothing, so it names neither. + out = io.StringIO() + with dataretrieval.configure(WaterdataSettings(concurrency=6), Settings(retries=3)): + dataretrieval.show_settings(stream=out) + text = out.getvalue() + + assert "configure() block [waterdata]" in text + # The file still *defines* the profile, so it is still listed as available; + # what must not happen is a value being attributed to it. + assert "configure() block [waterdata.bulk]" not in text + retries_row = next(line for line in text.splitlines() if line.startswith("retries")) + assert retries_row.endswith("configure() block") + + +def test_a_loaded_profile_remembers_its_name_without_becoming_a_setting(config_file): + """The profile name is provenance, so it is not a field and not a value. + + Keeping it off the fields is what stops it reaching :meth:`settings`, the + ``configure()`` frame, and equality: two configurations carrying the same + settings stay interchangeable however each was spelled, which is what + makes a configuration a value rather than a record of how it was built. + """ + config_file("[waterdata.bulk]\nconcurrency = 6\n") + + loaded = WaterdataSettings.load("bulk") + written = WaterdataSettings(concurrency=6) + + assert loaded.profile == "bulk" + assert written.profile is None + assert "profile" not in loaded.settings() + assert loaded == written + + +def test_show_settings_lists_the_profiles_the_file_defines(config_file, monkeypatch): + """A named profile is inert until selected, so the file's are listed too. + + "I added ``[waterdata.bulk]`` and nothing changed" is the confusion this + section exists for: the profiles are there, and no row above names one + because no caller selected one. A report that mentioned a profile only + once it had been selected would leave that silence unexplained. + + Names are read from the parsed file, so an adapter this process never + imported still has its profiles listed: what a table *means* needs the + import, what it is called does not, and hiding it would make the section + depend on which optional extras happened to be installed. + """ + monkeypatch.delitem(settings._REGISTRY, "nldi", raising=False) + config_file( + "[ngwmn]\nconcurrency = 4\n\n" + "[ngwmn.gentle]\nconcurrency = 2\n\n" + "[waterdata.bulk]\nparallel_chunks = 8\n\n" + "[nldi.gentle]\nretries = 1\n" + ) + out = io.StringIO() + + dataretrieval.show_settings(stream=out) + text = out.getvalue() + listed = text.split("profiles in the file: ", 1)[1].splitlines()[0] + + assert listed == "[waterdata.bulk], [ngwmn.gentle], [nldi.gentle]" + # The adapter's *default* profile is not a named one: it is always in + # effect and already shows up as a source, so listing it here is noise. + assert "[ngwmn]" not in listed + # Inert, and the report says so by never naming one as a source. + assert "configure() block" not in text + + +def test_show_settings_reports_an_unimported_adapter(config_file, monkeypatch): + """An adapter this process cannot report on is named, never omitted. + + NLDI is imported on demand for the geopandas extra, so a process that has + not touched it cannot say which settings it accepts -- the honest cost of + validating an adapter's keys lazily (ADR 0011). Leaving it out of the + report would read as "nothing is configured for nldi", which is a + different claim from "this report could not check", and the caller cannot + tell which one they are looking at. + """ + config_file("") + monkeypatch.delitem(settings._REGISTRY, "nldi", raising=False) + out = io.StringIO() + + dataretrieval.show_settings(stream=out) + text = out.getvalue() + + assert "not reported: nldi" in text + assert "not imported" in text + # An adapter that *was* imported is covered by the rows above, so it must + # not be named as uncoverable. + assert "waterdata" not in text.split("not reported:", 1)[1] + + # The line is a statement about this process, not about nldi: once the + # module is imported its configuration registers and the caveat goes away. + class _AsImported(settings.AdapterSettings): + adapter: ClassVar[str] = "nldi" + + retries: int | None = None + + monkeypatch.setitem(settings._REGISTRY, "nldi", _AsImported) + out = io.StringIO() + dataretrieval.show_settings(stream=out) + assert "not reported" not in out.getvalue() + + +def test_show_settings_sample_output_is_current(config_file, monkeypatch): + """The documented samples are this function's real output, not a drawing. + + Both had drifted from it -- the docstring wrapped a line the function + prints whole, the user guide had lost a paragraph -- because a sample kept + by hand is only ever as fresh as the last person who remembered it. So + the scenario is rebuilt here and the output compared verbatim; the only + edit is swapping the temporary path for the illustrative one. + + Regenerate by running this test and copying the reported ``actual`` into + both places, never by editing them to taste. + """ + path = config_file(_SAMPLE_FILE) + monkeypatch.setenv("API_USGS_RETRIES", "8") + # The sample shows the report a caller with the geopandas extra uninstalled + # sees; in this suite something has usually imported nldi already. + monkeypatch.delitem(settings._REGISTRY, "nldi", raising=False) + + out = io.StringIO() + with dataretrieval.configure(WaterdataSettings.load("bulk")): + dataretrieval.show_settings(stream=out) + actual = out.getvalue().replace(str(path), _SAMPLE_PATH).strip("\n") + + assert _documented_sample() == f"{_SAMPLE_PROMPT}\n{actual}" + + # The user guide shows the same sample, indented into its code block, and + # goes stale the same way. Checked here rather than in a docs test because + # the thing that makes it stale is a change to this function's output. + guide = ( + pathlib.Path(__file__).resolve().parents[1] + / "docs" + / "source" + / "userguide" + / "settings.rst" + ) + if not guide.exists(): # pragma: no cover - docs are absent from an sdist + pytest.skip("docs tree not present") + block = textwrap.indent(f"{_SAMPLE_PROMPT}\n{actual}", " ") + assert block in guide.read_text(encoding="utf-8") + + +def test_show_settings_survives_a_malformed_profile(config_file): + """Explaining a broken configuration is the job, so nothing here validates. + + The section lists what the file *defines*; a profile's keys are checked when a + caller selects it. So a profile holding a value that fails its grammar -- + or the nested table a file migrated from the retired ``[profiles.]`` + layout still carries -- is reported rather than taking the report down + with it, which is the one moment a caller most needs it. + """ + config_file( + '[waterdata.bulk]\nconcurrency = "nope"\n\n' + "[ngwmn.gentle]\n\n[ngwmn.gentle.nested]\nconcurrency = 2\n" + ) + out = io.StringIO() + + dataretrieval.show_settings(stream=out) # must not raise + + listed = out.getvalue().split("profiles in the file: ", 1)[1].splitlines()[0] + assert listed == "[waterdata.bulk], [ngwmn.gentle]" + # Selecting one is where the grammar is checked, and it still is. + with pytest.raises(settings.ConfigurationError, match="integer"): + WaterdataSettings.load("bulk") + with pytest.raises(settings.ConfigurationError, match="contains a table"): + NgwmnSettings.load("gentle") diff --git a/tests/transport_test.py b/tests/transport_test.py index dcd4b5c0..68b8d331 100644 --- a/tests/transport_test.py +++ b/tests/transport_test.py @@ -441,15 +441,15 @@ def test_bad_retry_environment_raises_a_catchable_error(monkeypatch) -> None: """ monkeypatch.setenv("API_USGS_RETRIES", "off") with pytest.raises(DataRetrievalError): - retry.RetryPolicy.from_env() + retry.RetryPolicy.from_settings() monkeypatch.setenv("API_USGS_RETRIES", "2") monkeypatch.setenv("API_USGS_STALL_TIMEOUT", "none") with pytest.raises(ConfigurationError): - retry.RetryPolicy.from_env() + retry.RetryPolicy.from_settings() monkeypatch.setenv("API_USGS_STALL_TIMEOUT", "10") - assert retry.RetryPolicy.from_env().stall_timeout == 10.0 + assert retry.RetryPolicy.from_settings().stall_timeout == 10.0 # Still a ValueError, so existing handling of a bad setting keeps working. assert issubclass(ConfigurationError, ValueError) diff --git a/tests/waterdata_chunking_test.py b/tests/waterdata_chunking_test.py index 3454dc31..faa83a3b 100644 --- a/tests/waterdata_chunking_test.py +++ b/tests/waterdata_chunking_test.py @@ -32,6 +32,8 @@ import pandas as pd import pytest +import dataretrieval +from dataretrieval import settings as _settings from dataretrieval.combining import ( _QUOTA_HEADER, _combine_chunk_frames, @@ -50,7 +52,6 @@ from dataretrieval.ogc.chunking import ( ChunkedCall, _chunked_client, - _parallel_chunks, get_active_client, multi_value_chunked, parallel_chunks, @@ -73,9 +74,9 @@ from dataretrieval.ogc.requests import ( _construct_api_requests as _construct_api_requests_explicit, ) +from dataretrieval.settings import Settings from dataretrieval.transport import retry as _retry_mod from dataretrieval.transport.retry import ( - _RETRIES_DEFAULT, RetryPolicy, _retryable, ) @@ -731,6 +732,45 @@ async def fetch(args, *, base): assert sorted(df["id"].tolist()) == sorted(sites) +def test_resume_reads_concurrency_from_the_caller_not_the_snapshot(monkeypatch): + """A ``configure()`` block around a ``resume()`` must actually take effect. + + The concurrency cap is the one dial a caller adjusts precisely *when* + retrying -- the documented recovery from ``QuotaExhausted`` is to wait and + re-issue more gently -- so ``resume()`` resolves it per drive rather than + carrying a value fixed when the call was constructed. A ``configure()`` + block entered between the interruption and the resume therefore wins. + """ + state = {"calls": 0} + + async def fetch(args): + state["calls"] += 1 + if state["calls"] == 3: + raise RateLimited("429: Too many requests made.") + sites = list(args["sites"]) + return (pd.DataFrame({"id": sites}), _quota_response(500)) + + sites = ["S" * 10 + str(i) for i in range(16)] + decorated = multi_value_chunked(build_request=_fake_build, url_limit=240)(fetch) + with pytest.raises(QuotaExhausted) as excinfo: + decorated({"sites": sites}) + + # Spy on the real, decorator-built ChunkedCall rather than a hand-made one. + seen: list[int | None] = [] + original_run = _chunking.ChunkedCall._run + + async def spy_run(self, max_concurrent): + seen.append(max_concurrent) + return await original_run(self, max_concurrent) + + monkeypatch.setattr(_chunking.ChunkedCall, "_run", spy_run) + + with dataretrieval.configure(Settings(concurrency=2)): + excinfo.value.call.resume() + + assert seen == [2], seen + + def test_chunker_passes_through_non_429_runtime_error(): """A non-429 ``RuntimeError`` (e.g. a 500) is not a quota signal; it must propagate unchanged so callers see the real cause.""" @@ -1421,8 +1461,8 @@ def test_iter_chunk_args_passthrough_yields_a_copy(): # --- async fan-out path ---------------------------------------------------- # # Every chunk is gathered over one ``httpx.AsyncClient`` and -# concurrency is bounded by an ``asyncio.Semaphore`` sized from -# ``API_USGS_CONCURRENT`` (the client's connection pool is sized to +# concurrency is bounded by an ``asyncio.Semaphore`` sized from the effective +# configuration (the client's connection pool is sized to # match, but the semaphore is the throttle — see ``ChunkedCall._run``). # The conftest's ``_pin_chunker_env`` autouse pins # ``API_USGS_CONCURRENT=1`` (sequential dispatch) for the whole suite; @@ -1650,6 +1690,21 @@ def test_fan_out_in_flight_high_water_mark_is_the_cap( assert in_flight["max"] == expected_high_water +def test_configure_concurrency_controls_dispatch(monkeypatch): + """The highest-precedence block setting reaches the execution semaphore.""" + monkeypatch.setenv("API_USGS_CONCURRENT", "1") + in_flight = {"now": 0, "max": 0} + fetch = multi_value_chunked(build_request=_fake_build, url_limit=240)( + _concurrency_probe(in_flight) + ) + + with dataretrieval.configure(Settings(concurrency=2)): + df, _ = fetch({"sites": list(_EIGHT_SINGLETON_SITES)}) + + assert len(df) == len(_EIGHT_SINGLETON_SITES) + assert in_flight["max"] == 2 + + def test_fan_out_outlives_pool_timeout_on_real_transport(monkeypatch): """End-to-end regression for the pool-timeout starvation bug: the fan-out must survive every pooled connection staying busy past the @@ -1847,19 +1902,19 @@ def test_retry_policy_long_retry_after_escalates(): assert not policy.should_retry(attempt=1, retry_after=120.0) # escalates -def test_retry_policy_from_env(monkeypatch): +def test_retry_policy_from_config(monkeypatch): monkeypatch.setenv("API_USGS_RETRIES", "2") - assert RetryPolicy.from_env().max_retries == 2 + assert RetryPolicy.from_settings().max_retries == 2 monkeypatch.setenv("API_USGS_RETRIES", "0") - assert RetryPolicy.from_env().max_retries == 0 + assert RetryPolicy.from_settings().max_retries == 0 monkeypatch.delenv("API_USGS_RETRIES", raising=False) - assert RetryPolicy.from_env().max_retries == _RETRIES_DEFAULT + assert RetryPolicy.from_settings().max_retries == _settings.DEFAULT_RETRIES monkeypatch.setenv("API_USGS_RETRIES", "-1") with pytest.raises(ValueError): - RetryPolicy.from_env() + RetryPolicy.from_settings() monkeypatch.setenv("API_USGS_RETRIES", "lots") with pytest.raises(ValueError): - RetryPolicy.from_env() + RetryPolicy.from_settings() def test_retry_policy_rejects_invalid_settings(): @@ -1871,12 +1926,12 @@ def test_retry_policy_rejects_invalid_settings(): RetryPolicy(max_backoff=-1.0) -def test_retry_policy_from_env_honors_monkeypatched_constants(monkeypatch): +def test_retry_policy_from_config_honors_monkeypatched_constants(monkeypatch): # The timing knobs are read from the module constants at call time, so # monkeypatching them (as the module comment promises) takes effect. monkeypatch.setattr(_retry_mod, "_RETRY_MAX_BACKOFF", 0.0) monkeypatch.setattr(_retry_mod, "_RETRY_BASE_BACKOFF", 0.0) - policy = RetryPolicy.from_env() + policy = RetryPolicy.from_settings() assert policy.max_backoff == 0.0 and policy.base_backoff == 0.0 @@ -2384,16 +2439,24 @@ def test_cap_does_not_mask_unchunkable(): ChunkPlan(args, _fake_build, url_limit=10, max_chunks=32) -def test_parallel_chunks_publishes_n_on_the_ambient(): - """The context manager publishes ``n`` on the ambient for the block and - restores the previous value on exit — including proper nesting.""" - assert _parallel_chunks.get() == 1 # default (off, = no extra fan-out) +def test_parallel_chunks_publishes_n_as_the_effective_setting(): + """The context manager sets ``n`` for the block and restores the previous + value on exit — including proper nesting. + + ``parallel_chunks(n)`` is sugar for ``configure(parallel_chunks=n)``, so + both forms share one scoping mechanism and the innermost block wins. + Outside any block the configured baseline applies, which is ``1`` — off — + unless a config file raised it.""" + assert _settings.parallel_chunks() == 1 # default (off, = no extra fan-out) with parallel_chunks(32): - assert _parallel_chunks.get() == 32 + assert _settings.parallel_chunks() == 32 with parallel_chunks(2): - assert _parallel_chunks.get() == 2 - assert _parallel_chunks.get() == 32 # outer restored - assert _parallel_chunks.get() == 1 # default (off) outside any block + assert _settings.parallel_chunks() == 2 + assert _settings.parallel_chunks() == 32 # outer restored + with dataretrieval.configure(Settings(parallel_chunks=4)): # the other spelling + assert _settings.parallel_chunks() == 4 + assert _settings.parallel_chunks() == 32 + assert _settings.parallel_chunks() == 1 # default (off) outside any block @pytest.mark.parametrize( @@ -2413,11 +2476,16 @@ def test_parallel_chunks_rejects_non_positive_int(bad): """``n`` must be a positive integer; every other shape — zero, negative, a float, a string (including a numeric one and the old level names), ``None``, a ``bool``, a list — raises ``ValueError`` at ``with`` entry, before any - request, and leaves the ambient untouched.""" - with pytest.raises(ValueError, match="must be a positive integer"): + request, and leaves the ambient untouched. + + The message comes from the ``parallel_chunks`` grammar in the configuration + chain, which is the one that owns this setting's bound; a + ``ConfigurationError`` is a ``ValueError``, which is the contract callers + were given here.""" + with pytest.raises(ValueError, match="must be an integer"): with parallel_chunks(bad): pass - assert _parallel_chunks.get() == 1 # default (off) — unchanged by a rejected call + assert _settings.parallel_chunks() == 1 # unchanged by a rejected call def test_parallel_chunks_drives_end_to_end_fan_out(): diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 9081b701..a17ad16e 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -43,6 +43,7 @@ _EXTRA_ID_COLS, OGC_API_URL, WATERDATA_DIALECT, + _flatten_queryables, _get_args, ) @@ -1146,3 +1147,24 @@ def fake_engine_get_ogc_data(args, collection, output_id, **k): ): _utils_module.get_ogc_data({"state": "WI"}, "monitoring-locations") assert captured["args"] == {"state": "WI"} + + +@pytest.mark.parametrize( + "name", ["x_api_key", "x-api-key", "api_token", "access_token", "pat", "auth"] +) +def test_credential_shaped_queryables_are_rejected(name): + """The denylist matches spellings, not just a few exact names. + + ``x_api_key`` is the tempting one -- it mirrors the ``X-Api-Key`` header + the README documents -- and an exact-match list let it through into the + query string. + """ + with pytest.raises(TypeError, match="Credentials cannot be passed"): + _flatten_queryables({"queryables": {name: "SECRET"}}) + + +@pytest.mark.parametrize( + "name", ["state_name", "site_type_code", "monitoring_location_id", "qualifier"] +) +def test_real_queryables_still_pass_through(name): + assert _flatten_queryables({"queryables": {name: "v"}}) == {name: "v"} diff --git a/tests/wqp_test.py b/tests/wqp_test.py index cde3c84e..c96400c4 100644 --- a/tests/wqp_test.py +++ b/tests/wqp_test.py @@ -4,6 +4,7 @@ import pytest from pandas import DataFrame +import dataretrieval import dataretrieval.wqp as wqp from dataretrieval.wqp import ( WQP_Metadata, @@ -142,6 +143,30 @@ def test_wqp_url_profiles(builder, service, expected, warning): assert builder(service) == expected +def test_a_configured_base_url_moves_both_interfaces(): + """One root, both paths: the portal serves legacy and WQX3 from one host. + + Redirecting only the interface a caller happened to use first would leave + the other pointed at the service they were redirecting away from, which is + the failure a redirect exists to prevent. + """ + mirror = "https://mirror.example/wqp" + + with dataretrieval.configure(wqp.WqpSettings(base_url=mirror)): + with pytest.warns(DeprecationWarning): + legacy = wqp.wqp_url("Result") + with pytest.warns(UserWarning): + wqx3 = wqp.wqx3_url("Result") + + assert legacy == f"{mirror}/data/Result/Search?" + assert wqx3 == f"{mirror}/wqx3/Result/search?" + + # Outside the block, the portal's own root again -- the redirect is scoped + # to the ``with`` statement, not latched at import. + with pytest.warns(DeprecationWarning): + assert wqp.wqp_url("Result").startswith("https://www.waterqualitydata.us/") + + @pytest.mark.parametrize( ("builder", "profile", "valid_services", "warning"), [ @@ -231,6 +256,30 @@ def test_check_kwargs(): kwargs = _check_kwargs(kwargs) +@pytest.mark.parametrize( + "name", ["api_key", "x_api_key", "access_token", "password", "pat", "auth"] +) +def test_credential_shaped_wqp_kwargs_are_rejected(name): + """WQP has the widest ``**kwargs`` passthrough in the package. + + Its ten getters forward whatever the caller names straight into the query + string, so ``api_key=`` -- the plausible guess now that ``configure()`` + takes ``Settings(api_key=...)`` -- would put a secret in a URL that + clients, proxies and logs retain. Same predicate and same message as Water + Data's ``**queryables`` guard, because it is the same mistake. + """ + with pytest.raises(TypeError, match="Credentials cannot be passed"): + _check_kwargs({name: "SECRET"}) + + +@pytest.mark.parametrize( + "name", ["siteid", "characteristicName", "statecode", "providers", "pCode"] +) +def test_real_wqp_filters_still_pass_through(name): + """The denylist must not claim names the portal owns.""" + assert _check_kwargs({name: "v"})[name] == "v" + + def test_get_results_wqx3_preserves_user_dataProfile(httpx_mock): """A valid user-supplied WQX3.0 profile must not be overwritten.