diff --git a/.github/workflows/openapi-spec-updates.yml b/.github/workflows/openapi-spec-updates.yml new file mode 100644 index 00000000..42441333 --- /dev/null +++ b/.github/workflows/openapi-spec-updates.yml @@ -0,0 +1,96 @@ +name: OpenAPI spec updates + +on: + schedule: + - cron: "0 8 * * 1" # Mondays at 08:00 UTC + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: openapi-spec-updates + cancel-in-progress: false + +jobs: + update: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup Python environment + uses: ./.github/actions/setup-python-env + + - name: Update pinned specification + id: update + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mise exec -- python py/scripts/update-openapi-spec.py \ + --summary-file "$RUNNER_TEMP/openapi-update.md" >> "$GITHUB_OUTPUT" + + - name: Regenerate client and public reference + id: generate + if: steps.update.outputs.changed == 'true' + continue-on-error: true + run: mise exec -- make -C py generate-api-client + + - name: Run codegen tests + id: codegen + if: steps.update.outputs.changed == 'true' + continue-on-error: true + run: mise exec -- make -C py test-api-codegen + + - name: Run REST client runtime tests + id: runtime + if: steps.update.outputs.changed == 'true' + continue-on-error: true + run: mise exec -- make -C py test-core + + - name: Run public API type tests + id: types + if: steps.update.outputs.changed == 'true' + continue-on-error: true + run: mise exec -- uv run --project ./py nox -f ./py/noxfile.py -s test_types + + - name: Add validation results to pull request body + if: steps.update.outputs.changed == 'true' + env: + GENERATE_OUTCOME: ${{ steps.generate.outcome }} + CODEGEN_OUTCOME: ${{ steps.codegen.outcome }} + RUNTIME_OUTCOME: ${{ steps.runtime.outcome }} + TYPES_OUTCOME: ${{ steps.types.outcome }} + run: | + { + echo + echo "### Workflow results" + echo + echo "| Check | Result |" + echo "| --- | --- |" + echo "| Regeneration | \`$GENERATE_OUTCOME\` |" + echo "| Codegen tests | \`$CODEGEN_OUTCOME\` |" + echo "| Runtime tests | \`$RUNTIME_OUTCOME\` |" + echo "| Type tests | \`$TYPES_OUTCOME\` |" + } >> "$RUNNER_TEMP/openapi-update.md" + + - name: Create pull request + if: steps.update.outputs.changed == 'true' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + commit-message: "chore(api): update pinned OpenAPI spec" + branch: auto/update-openapi-spec + title: "chore(api): update pinned OpenAPI spec" + body-path: ${{ runner.temp }}/openapi-update.md + delete-branch: true + + - name: Fail when automated validation did not pass + if: >- + steps.update.outputs.changed == 'true' && + (steps.generate.outcome != 'success' || + steps.codegen.outcome != 'success' || + steps.runtime.outcome != 'success' || + steps.types.outcome != 'success') + run: | + echo "The update PR was created, but at least one automated validation failed." + exit 1 diff --git a/openapi/README.md b/openapi/README.md index 3cb30087..0b0c4336 100644 --- a/openapi/README.md +++ b/openapi/README.md @@ -15,7 +15,8 @@ make generate-api-client make check-api-client-codegen ``` -The check regenerates in a temporary directory and reports drift without changing the worktree. +The check regenerates in a temporary directory and reports drift without changing the worktree. Generation also synchronizes the reviewed resource, method, and type inventories in the [public REST API client README](../py/src/braintrust/api/README.md). + The reviewed generated surface includes these tags: - core resources: Projects, Experiments, Datasets, Prompts, and Functions; @@ -70,6 +71,6 @@ BRAINTRUST_OPENAPI_ROOT=../../braintrust-openapi make fetch-openapi-spec ``` The checkout must be at the commit pinned in `config.json`, and its spec must match the pinned hash. -To update the snapshot, update the commit and hash in `config.json`, fetch, regenerate, and review both -the upstream spec diff and generated-source diff. Validation and generation apply only to selected tags -and their transitively reachable schemas. +To update the snapshot manually, update the commit and hash in `config.json`, fetch, regenerate, and review both the upstream spec diff and generated-source diff. Validation and generation apply only to selected tags and their transitively reachable schemas. + +The scheduled and manually dispatchable [OpenAPI spec updates workflow](../.github/workflows/openapi-spec-updates.yml) checks the latest upstream commit that changed the spec. When the pin changes, it updates the snapshot, regenerates the client and public reference, runs codegen, runtime, and type tests, and opens or updates a review PR containing operation/schema summaries and a link to the upstream diff. The workflow never auto-merges its PR. If generation or validation fails, it still opens the update PR with the failure status and then fails the workflow so the new API shape can be reviewed explicitly. diff --git a/py/Makefile b/py/Makefile index 87cee9e4..5eb7952d 100644 --- a/py/Makefile +++ b/py/Makefile @@ -41,9 +41,11 @@ fetch-openapi-spec: generate-api-client: uv run --no-default-groups --group api-codegen python scripts/generate-api-client.py + $(PYTHON) scripts/generate-api-docs.py check-api-client-codegen: uv run --no-default-groups --group api-codegen python scripts/generate-api-client.py --check + $(PYTHON) scripts/generate-api-docs.py --check test-api-codegen: uv run nox -s test_api_codegen @@ -91,8 +93,8 @@ help: @echo " build - Build Python package" @echo " check-stale-cassettes - Detect orphaned cassette version directories" @echo " fetch-openapi-spec - Fetch the hash-verified pinned OpenAPI spec" - @echo " generate-api-client - Generate private REST API models from the pinned spec" - @echo " check-api-client-codegen - Check committed REST API models for drift" + @echo " generate-api-client - Generate the REST API client and public reference from the pinned spec" + @echo " check-api-client-codegen - Check the committed REST API client and public reference for drift" @echo " test-api-codegen - Run OpenAPI validator and generator tests" @echo " sync-pytest-pin - Sync [dependency-groups].test pytest pin from matrix" @echo " clean - Remove build artifacts" diff --git a/py/scripts/generate-api-docs.py b/py/scripts/generate-api-docs.py new file mode 100644 index 00000000..00ff5e8b --- /dev/null +++ b/py/scripts/generate-api-docs.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Update the generated reference sections in the public REST API README.""" + +import argparse +import ast +import sys +from pathlib import Path +from typing import Iterable + +from openapi_codegen import ( + CONFIG_PATH, + SPEC_PATH, + collect_generated_operations, + generated_resource_name, + load_config, + read_and_verify_spec, +) + + +API_ROOT = Path(__file__).resolve().parents[1] / "src" / "braintrust" / "api" +README_PATH = API_ROOT / "README.md" +RESOURCE_START = "" +RESOURCE_END = "" +API_EXPORT_START = "" +API_EXPORT_END = "" +TYPE_START = "" +TYPE_END = "" + + +def _literal_all(path: Path) -> list[str]: + tree = ast.parse(path.read_text(encoding="utf-8")) + assignment = next( + node + for node in tree.body + if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "__all__" for target in node.targets) + ) + value = ast.literal_eval(assignment.value) + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{path}.__all__ must be a literal list of strings") + return value + + +def _render_resource_reference() -> str: + config = load_config(CONFIG_PATH) + spec = read_and_verify_spec(config, SPEC_PATH) + operations, _ = collect_generated_operations(spec, config) + operations_by_tag = {tag: [] for tag in config["endpoint_generator"]["generated_tags"]} + for operation in operations: + operations_by_tag[operation.tag].append(operation) + + rows = ["| Client property | Methods |", "| --- | --- |"] + for tag, tag_operations in operations_by_tag.items(): + methods = "
".join( + f"`{operation.constant_name.lower()}` — `{operation.method} {operation.path}`" + for operation in tag_operations + ) + rows.append(f"| `client.{generated_resource_name(tag)}` | {methods} |") + return "\n".join(rows) + + +def _render_name_table(names: Iterable[str], columns: int = 3) -> str: + values = [f"`{name}`" for name in names] + rows = ["| " + " | ".join(["Name"] * columns) + " |", "| " + " | ".join(["---"] * columns) + " |"] + for index in range(0, len(values), columns): + row = values[index : index + columns] + row.extend([""] * (columns - len(row))) + rows.append("| " + " | ".join(row) + " |") + return "\n".join(rows) + + +def _replace_section(content: str, start: str, end: str, replacement: str) -> str: + if content.count(start) != 1 or content.count(end) != 1: + raise ValueError(f"{README_PATH} must contain exactly one {start!r} and {end!r} marker") + prefix, remainder = content.split(start, 1) + _, suffix = remainder.split(end, 1) + return f"{prefix}{start}\n{replacement}\n{end}{suffix}" + + +def render_readme(content: str) -> str: + content = _replace_section(content, RESOURCE_START, RESOURCE_END, _render_resource_reference()) + content = _replace_section( + content, + API_EXPORT_START, + API_EXPORT_END, + _render_name_table(_literal_all(API_ROOT / "__init__.py")), + ) + return _replace_section( + content, + TYPE_START, + TYPE_END, + _render_name_table(_literal_all(API_ROOT / "types" / "__init__.py")), + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="Report README drift without changing the file.") + args = parser.parse_args() + + current = README_PATH.read_text(encoding="utf-8") + rendered = render_readme(current) + if current == rendered: + print(f"Public REST API documentation is current: {README_PATH}") + return 0 + if args.check: + print( + "Public REST API documentation drift detected. Run `cd py && make generate-api-client`.", + file=sys.stderr, + ) + return 1 + README_PATH.write_text(rendered, encoding="utf-8") + print(f"Updated public REST API documentation: {README_PATH}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/py/scripts/openapi_codegen.py b/py/scripts/openapi_codegen.py index 4a151673..63d18726 100644 --- a/py/scripts/openapi_codegen.py +++ b/py/scripts/openapi_codegen.py @@ -433,7 +433,8 @@ def _partition_model_source( common_names = {name for name, tags in owners.items() if len(tags) > 1} module_for_name = { - name: "common" if name in common_names else _snake_case(next(iter(tags))) for name, tags in owners.items() + name: "common" if name in common_names else generated_resource_name(next(iter(tags))) + for name, tags in owners.items() } def source_for(node: ast.stmt) -> str: @@ -467,7 +468,7 @@ def source_for(node: ast.stmt) -> str: def generate_tree(output_root: Path, config: Mapping[str, Any], spec: Mapping[str, Any]) -> ValidationReport: validate_config(config) report = validate_spec(spec, config) - operations, inline_models = _collect_generated_operations(spec, config) + operations, inline_models = collect_generated_operations(spec, config) selected_spec = _slice_model_spec(spec, {operation.operation_id for operation in operations}) model_spec = _with_inline_models(_extract_colliding_inline_models(selected_spec), inline_models) output_root.mkdir(parents=True, exist_ok=True) @@ -840,7 +841,7 @@ def _operation_retry_mode(method: str, operation_id: str, safe_reads: Set[str], return "NONE" -def _collect_generated_operations( +def collect_generated_operations( spec: Mapping[str, Any], config: Mapping[str, Any] ) -> Tuple[List[GeneratedOperation], List[Tuple[str, Mapping[str, Any]]]]: endpoint = _endpoint_config(config) @@ -1002,7 +1003,7 @@ def _generate_resources( generated_paths = [] for tag, tag_operations in sorted(by_tag.items()): - resource_path = root / f"{_snake_case(tag)}.py" + resource_path = root / f"{generated_resource_name(tag)}.py" _write_generated_file(resource_path, _resource_module_source(tag, tag_operations, model_modules), config) generated_paths.append(resource_path) return generated_paths @@ -1143,6 +1144,12 @@ def _python_argument_name(value: str) -> str: return result +def generated_resource_name(tag: str) -> str: + """Return the Python client property and module name for an OpenAPI tag.""" + + return _snake_case(tag) + + def _snake_case(value: str) -> str: value = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", value) return re.sub(r"[^A-Za-z0-9]+", "_", value).strip("_").lower() diff --git a/py/scripts/update-openapi-spec.py b/py/scripts/update-openapi-spec.py new file mode 100644 index 00000000..ed304f69 --- /dev/null +++ b/py/scripts/update-openapi-spec.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Update the pinned OpenAPI snapshot to the latest upstream spec commit.""" + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Mapping, NamedTuple + +from openapi_codegen import ( + CONFIG_PATH, + HTTP_METHODS, + SPEC_PATH, + CodegenError, + load_config, + read_and_verify_spec, + validate_config, + validate_spec, +) + + +class Source(NamedTuple): + commit: str + content: bytes + description: str + + +def _request(url: str, *, accept: str) -> bytes: + headers = {"Accept": accept, "User-Agent": "braintrust-sdk-python-openapi-updater"} + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + request = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 - fixed GitHub endpoints. + return response.read() + except (OSError, urllib.error.URLError) as exc: + raise CodegenError(f"Unable to fetch {url}: {exc}") from exc + + +def _latest_source(config: Mapping[str, Any]) -> Source: + spec_config = config["spec"] + local_root = os.environ.get("BRAINTRUST_OPENAPI_ROOT") + if local_root: + root = Path(local_root).expanduser().resolve() + try: + commit = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + content = subprocess.run( + ["git", "-C", str(root), "show", f"{commit}:{spec_config['path']}"], + check=True, + capture_output=True, + ).stdout + except (OSError, subprocess.CalledProcessError) as exc: + raise CodegenError(f"BRAINTRUST_OPENAPI_ROOT is not a readable git checkout: {root}") from exc + return Source(commit, content, f"{root}@{commit}:{spec_config['path']}") + + repository = spec_config["repository"] + query = urllib.parse.urlencode({"path": spec_config["path"], "per_page": 1}) + commits_url = f"https://api.github.com/repos/{repository}/commits?{query}" + try: + commits = json.loads(_request(commits_url, accept="application/vnd.github+json")) + except json.JSONDecodeError as exc: + raise CodegenError(f"GitHub returned invalid JSON for {commits_url}: {exc}") from exc + if not isinstance(commits, list) or not commits or not isinstance(commits[0], dict): + raise CodegenError(f"GitHub returned no commits for {repository}/{spec_config['path']}") + commit = commits[0].get("sha") + if not isinstance(commit, str) or len(commit) != 40: + raise CodegenError(f"GitHub returned an invalid commit SHA for {repository}/{spec_config['path']}") + + quoted_path = urllib.parse.quote(spec_config["path"], safe="/") + raw_url = f"https://raw.githubusercontent.com/{repository}/{commit}/{quoted_path}" + content = _request(raw_url, accept="application/json") + return Source(commit, content, raw_url) + + +def _parse_spec(content: bytes, source: str) -> dict[str, Any]: + try: + spec = json.loads(content) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise CodegenError(f"OpenAPI spec from {source} is not valid JSON: {exc}") from exc + if not isinstance(spec, dict): + raise CodegenError(f"OpenAPI spec from {source} must contain a JSON object") + return spec + + +def _operation_ids(spec: Mapping[str, Any]) -> set[str]: + paths = spec.get("paths", {}) + if not isinstance(paths, dict): + return set() + return { + operation_id + for path_item in paths.values() + if isinstance(path_item, dict) + for method, operation in path_item.items() + if method in HTTP_METHODS + and isinstance(operation, dict) + and isinstance((operation_id := operation.get("operationId")), str) + } + + +def _schema_names(spec: Mapping[str, Any]) -> set[str]: + components = spec.get("components", {}) + schemas = components.get("schemas", {}) if isinstance(components, dict) else {} + return set(schemas) if isinstance(schemas, dict) else set() + + +def _format_changes(added: set[str], removed: set[str]) -> str: + lines = [] + if added: + lines.append("- Added: " + ", ".join(f"`{name}`" for name in sorted(added))) + if removed: + lines.append("- Removed: " + ", ".join(f"`{name}`" for name in sorted(removed))) + return "\n".join(lines) if lines else "- No names added or removed." + + +def _validation_summary(spec: Mapping[str, Any], config: Mapping[str, Any]) -> str: + try: + return str(validate_spec(spec, config)) + except CodegenError as exc: + return f"requires manual review (`{exc}`)" + + +def _build_summary( + old_spec: Mapping[str, Any], + new_spec: Mapping[str, Any], + old_config: Mapping[str, Any], + new_config: Mapping[str, Any], +) -> str: + repository = new_config["spec"]["repository"] + old_commit = old_config["spec"]["commit"] + new_commit = new_config["spec"]["commit"] + old_operations = _operation_ids(old_spec) + new_operations = _operation_ids(new_spec) + old_schemas = _schema_names(old_spec) + new_schemas = _schema_names(new_spec) + return f"""Automated update of the pinned Braintrust OpenAPI specification. + +- Upstream commit: [`{old_commit[:12]}`](https://github.com/{repository}/commit/{old_commit}) → [`{new_commit[:12]}`](https://github.com/{repository}/commit/{new_commit}) +- [Upstream spec diff](https://github.com/{repository}/compare/{old_commit}...{new_commit}) +- Reviewed generated surface: {_validation_summary(old_spec, old_config)} → {_validation_summary(new_spec, new_config)} +- All upstream operations: {len(old_operations)} → {len(new_operations)} +- All upstream component schemas: {len(old_schemas)} → {len(new_schemas)} + +### Operation changes + +{_format_changes(new_operations - old_operations, old_operations - new_operations)} + +### Component schema changes + +{_format_changes(new_schemas - old_schemas, old_schemas - new_schemas)} + +### Automated validation + +The update workflow regenerates committed sources and public API reference sections, then runs: + +- `make -C py test-api-codegen` +- `make -C py test-core` +- `cd py && nox -s test_types` + +This pull request is never auto-merged. Review the upstream and generated diffs, retry-policy classifications, public type changes, and any intentionally unsupported tags before merging. +""" + + +def _atomic_write(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary: + temporary.write(content) + temporary_path = Path(temporary.name) + if path.exists(): + os.chmod(temporary_path, path.stat().st_mode) + os.replace(temporary_path, path) + + +def update(config_path: Path, spec_path: Path, summary_path: Path | None) -> bool: + config = load_config(config_path) + validate_config(config, check_installed_tools=False) + old_spec = read_and_verify_spec(config, spec_path) + source = _latest_source(config) + new_spec = _parse_spec(source.content, source.description) + new_hash = hashlib.sha256(source.content).hexdigest() + + old_commit = config["spec"]["commit"] + old_hash = config["spec"]["sha256"] + if source.commit == old_commit and new_hash == old_hash: + return False + + new_config = json.loads(json.dumps(config)) + new_config["spec"]["commit"] = source.commit + new_config["spec"]["sha256"] = new_hash + validate_config(new_config, check_installed_tools=False) + summary = _build_summary(old_spec, new_spec, config, new_config) + + _atomic_write(spec_path, source.content) + _atomic_write(config_path, (json.dumps(new_config, indent=2) + "\n").encode()) + if summary_path: + _atomic_write(summary_path, summary.encode()) + print(f"Updated OpenAPI pin {old_commit} -> {source.commit} from {source.description}", file=sys.stderr) + return True + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=CONFIG_PATH, help=argparse.SUPPRESS) + parser.add_argument("--spec", type=Path, default=SPEC_PATH, help=argparse.SUPPRESS) + parser.add_argument("--summary-file", type=Path, help="Write the pull request body to this path.") + args = parser.parse_args() + + changed = update(args.config, args.spec, args.summary_file) + print(f"changed={str(changed).lower()}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except CodegenError as exc: + print(f"error: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/py/src/braintrust/api/README.md b/py/src/braintrust/api/README.md new file mode 100644 index 00000000..b462b73b --- /dev/null +++ b/py/src/braintrust/api/README.md @@ -0,0 +1,172 @@ +# Braintrust synchronous REST API client + +`braintrust.api` provides a synchronous, typed, resource-oriented client for the reviewed Braintrust REST API. It uses the SDK's existing `requests` transport, authentication, organization routing, retry policies, and typed errors. Client construction does not make a network request. + +> [!NOTE] +> Most applications should continue to use the higher-level APIs exported from `braintrust`. Use this REST client when you need direct access to a reviewed REST resource. + +## Create a client + +### Share authentication and routing + +Use `BraintrustClient` when organization discovery and REST resources should share one transport. `auth.login()` configures the API route selected for the organization. + +```python +from braintrust.api import BraintrustClient +from braintrust.api.types import CreateProject, Project + +with BraintrustClient(api_key="...") as client: + client.auth.login(org_name="my-organization") + + request: CreateProject = {"name": "my-project"} + project: Project = client.openapi.projects.post_project(body=request) + print(project["id"]) +``` + +The API key can be omitted when `BRAINTRUST_API_KEY` is set. `BraintrustClient` also accepts `app_url`, `api_url`, and `proxy_url` for custom deployments. + +### Connect directly to an API URL + +Use `BraintrustOpenApiClient` when organization discovery is unnecessary and the API URL is already known. + +```python +from braintrust.api import BraintrustOpenApiClient + +with BraintrustOpenApiClient( + api_key="...", + api_url="https://api.example.com", +) as client: + response = client.projects.get_project(project_name="my-project") +``` + +A direct client requires `api_url` or `BRAINTRUST_API_URL`. Both clients accept an existing `requests.Session`, an `HTTPAdapter`, or a Braintrust `Transport`; a supplied transport is not owned or closed by the client. + +## Calling resources + +Resources are loaded and cached on first access. Generated method names are the snake-case form of the OpenAPI `operationId`. Arguments mirror path and query parameters. JSON request bodies are passed through `body=` and results are mappings matching the exported `TypedDict` return type. + +The client does not inject organization names or other request defaults. Pass `org_name`, project identifiers, pagination parameters, and other filters explicitly when the operation exposes them. List responses are returned as one response page; the client does not create an implicit iterator. + +Responses may contain additive server fields that are not yet in the pinned types. Those keys are preserved at runtime. + +## Errors and retries + +REST failures use the error classes exported from `braintrust.api`, including `BraintrustHTTPError`, `BraintrustJSONDecodeError`, `BraintrustTransportError`, and the retry-exhaustion variants. HTTP errors preserve response status, opaque response bodies, and Braintrust request IDs when available. + +GET and HEAD operations use the safe-read retry policy. A small reviewed allowlist of logical POST reads and verified idempotent writes is also retried. Other writes are not retried automatically. Set `enable_sdk_retries=False` on either client to disable SDK retries. + +## Stability + +The handwritten clients, routing types, retry types, and error classes listed in the public exports section are supported public API. + +The generated resource methods and `braintrust.api.types` are a **preview API**. Every published tag and operation is reviewed before it is added, and pinned-spec updates are reviewed rather than auto-merged. While the API is in preview, method signatures and generated type shapes may change between SDK minor releases as the REST contract evolves. Presence in the upstream OpenAPI document alone does not make an operation public. + +The `braintrust.api._generated` package is private implementation detail. Do not import resource classes, operation metadata, or models from it directly. + +## REST types versus SDK payload types + +Import request and response types for this client from `braintrust.api.types`: + +```python +from braintrust.api.types import CreateExperiment, Experiment +``` + +These are dependency-free `TypedDict`s and type aliases generated from the pinned REST specification. They describe direct REST wire payloads. + +`braintrust.generated_types` is a separate public surface for high-level SDK logging and evaluation payloads. Some names intentionally overlap, but the shapes and compatibility contracts can differ. Do not substitute a type from one module for the same-named type in the other module. + +## Resource reference + +The table lists every reviewed generated method. Consult the method's Python signature in your editor for its path parameters, keyword-only query parameters, `body` type, and return type. + + +| Client property | Methods | +| --- | --- | +| `client.projects` | `post_project` — `POST /v1/project`
`get_project` — `GET /v1/project`
`get_project_id` — `GET /v1/project/{project_id}`
`patch_project_id` — `PATCH /v1/project/{project_id}`
`delete_project_id` — `DELETE /v1/project/{project_id}` | +| `client.experiments` | `post_experiment` — `POST /v1/experiment`
`get_experiment` — `GET /v1/experiment`
`get_experiment_id` — `GET /v1/experiment/{experiment_id}`
`patch_experiment_id` — `PATCH /v1/experiment/{experiment_id}`
`delete_experiment_id` — `DELETE /v1/experiment/{experiment_id}`
`post_experiment_id_insert` — `POST /v1/experiment/{experiment_id}/insert`
`post_experiment_id_fetch` — `POST /v1/experiment/{experiment_id}/fetch`
`get_experiment_id_fetch` — `GET /v1/experiment/{experiment_id}/fetch`
`post_experiment_id_feedback` — `POST /v1/experiment/{experiment_id}/feedback`
`get_experiment_id_summarize` — `GET /v1/experiment/{experiment_id}/summarize` | +| `client.datasets` | `post_dataset` — `POST /v1/dataset`
`get_dataset` — `GET /v1/dataset`
`get_dataset_id` — `GET /v1/dataset/{dataset_id}`
`patch_dataset_id` — `PATCH /v1/dataset/{dataset_id}`
`delete_dataset_id` — `DELETE /v1/dataset/{dataset_id}`
`post_dataset_id_insert` — `POST /v1/dataset/{dataset_id}/insert`
`post_dataset_id_fetch` — `POST /v1/dataset/{dataset_id}/fetch`
`get_dataset_id_fetch` — `GET /v1/dataset/{dataset_id}/fetch`
`post_dataset_id_feedback` — `POST /v1/dataset/{dataset_id}/feedback`
`get_dataset_id_summarize` — `GET /v1/dataset/{dataset_id}/summarize` | +| `client.prompts` | `post_prompt` — `POST /v1/prompt`
`put_prompt` — `PUT /v1/prompt`
`get_prompt` — `GET /v1/prompt`
`get_prompt_id` — `GET /v1/prompt/{prompt_id}`
`patch_prompt_id` — `PATCH /v1/prompt/{prompt_id}`
`delete_prompt_id` — `DELETE /v1/prompt/{prompt_id}` | +| `client.functions` | `post_function` — `POST /v1/function`
`put_function` — `PUT /v1/function`
`get_function` — `GET /v1/function`
`get_function_id` — `GET /v1/function/{function_id}`
`patch_function_id` — `PATCH /v1/function/{function_id}`
`delete_function_id` — `DELETE /v1/function/{function_id}` | +| `client.acls` | `post_acl` — `POST /v1/acl`
`delete_acl` — `DELETE /v1/acl`
`get_acl` — `GET /v1/acl`
`get_acl_id` — `GET /v1/acl/{acl_id}`
`delete_acl_id` — `DELETE /v1/acl/{acl_id}`
`acl_batch_update` — `POST /v1/acl/batch_update`
`acl_list_org` — `GET /v1/acl/list_org` | +| `client.agents` | `post_agent` — `POST /v1/agent`
`put_agent` — `PUT /v1/agent`
`get_agent` — `GET /v1/agent`
`get_agent_id` — `GET /v1/agent/{agent_id}`
`patch_agent_id` — `PATCH /v1/agent/{agent_id}`
`delete_agent_id` — `DELETE /v1/agent/{agent_id}` | +| `client.ai_secrets` | `post_ai_secret` — `POST /v1/ai_secret`
`put_ai_secret` — `PUT /v1/ai_secret`
`delete_ai_secret` — `DELETE /v1/ai_secret`
`get_ai_secret` — `GET /v1/ai_secret`
`get_ai_secret_id` — `GET /v1/ai_secret/{ai_secret_id}`
`patch_ai_secret_id` — `PATCH /v1/ai_secret/{ai_secret_id}`
`delete_ai_secret_id` — `DELETE /v1/ai_secret/{ai_secret_id}` | +| `client.api_keys` | `get_api_key` — `GET /v1/api_key`
`get_api_key_id` — `GET /v1/api_key/{api_key_id}`
`delete_api_key_id` — `DELETE /v1/api_key/{api_key_id}` | +| `client.dataset_snapshots` | `post_dataset_snapshot` — `POST /v1/dataset_snapshot`
`put_dataset_snapshot` — `PUT /v1/dataset_snapshot`
`get_dataset_snapshot` — `GET /v1/dataset_snapshot`
`get_dataset_snapshot_id` — `GET /v1/dataset_snapshot/{dataset_snapshot_id}`
`patch_dataset_snapshot_id` — `PATCH /v1/dataset_snapshot/{dataset_snapshot_id}`
`delete_dataset_snapshot_id` — `DELETE /v1/dataset_snapshot/{dataset_snapshot_id}` | +| `client.env_vars` | `post_env_var` — `POST /v1/env_var`
`put_env_var` — `PUT /v1/env_var`
`get_env_var` — `GET /v1/env_var`
`get_env_var_id` — `GET /v1/env_var/{env_var_id}`
`patch_env_var_id` — `PATCH /v1/env_var/{env_var_id}`
`delete_env_var_id` — `DELETE /v1/env_var/{env_var_id}` | +| `client.environments` | `list_environments` — `GET /environment`
`create_environment` — `POST /environment`
`get_environment` — `GET /environment/{environment_id}`
`update_environment` — `PATCH /environment/{environment_id}`
`delete_environment` — `DELETE /environment/{environment_id}` | +| `client.groups` | `post_group` — `POST /v1/group`
`put_group` — `PUT /v1/group`
`get_group` — `GET /v1/group`
`get_group_id` — `GET /v1/group/{group_id}`
`patch_group_id` — `PATCH /v1/group/{group_id}`
`delete_group_id` — `DELETE /v1/group/{group_id}` | +| `client.mcp_servers` | `post_mcp_server` — `POST /v1/mcp_server`
`put_mcp_server` — `PUT /v1/mcp_server`
`get_mcp_server` — `GET /v1/mcp_server`
`get_mcp_server_id` — `GET /v1/mcp_server/{mcp_server_id}`
`patch_mcp_server_id` — `PATCH /v1/mcp_server/{mcp_server_id}`
`delete_mcp_server_id` — `DELETE /v1/mcp_server/{mcp_server_id}` | +| `client.org_automations` | `post_org_automation` — `POST /v1/org_automation`
`put_org_automation` — `PUT /v1/org_automation`
`get_org_automation` — `GET /v1/org_automation`
`get_org_automation_id` — `GET /v1/org_automation/{org_automation_id}`
`patch_org_automation_id` — `PATCH /v1/org_automation/{org_automation_id}`
`delete_org_automation_id` — `DELETE /v1/org_automation/{org_automation_id}` | +| `client.organizations` | `get_organization` — `GET /v1/organization`
`get_organization_id` — `GET /v1/organization/{organization_id}`
`patch_organization_id` — `PATCH /v1/organization/{organization_id}`
`patch_organization_members` — `PATCH /v1/organization/members` | +| `client.project_automations` | `post_project_automation` — `POST /v1/project_automation`
`put_project_automation` — `PUT /v1/project_automation`
`get_project_automation` — `GET /v1/project_automation`
`get_project_automation_id` — `GET /v1/project_automation/{project_automation_id}`
`patch_project_automation_id` — `PATCH /v1/project_automation/{project_automation_id}`
`delete_project_automation_id` — `DELETE /v1/project_automation/{project_automation_id}` | +| `client.project_groups` | `post_project_group` — `POST /v1/project_group`
`put_project_group` — `PUT /v1/project_group`
`get_project_group` — `GET /v1/project_group`
`get_project_group_id` — `GET /v1/project_group/{project_group_id}`
`patch_project_group_id` — `PATCH /v1/project_group/{project_group_id}`
`delete_project_group_id` — `DELETE /v1/project_group/{project_group_id}` | +| `client.project_scores` | `post_project_score` — `POST /v1/project_score`
`put_project_score` — `PUT /v1/project_score`
`get_project_score` — `GET /v1/project_score`
`get_project_score_id` — `GET /v1/project_score/{project_score_id}`
`patch_project_score_id` — `PATCH /v1/project_score/{project_score_id}`
`delete_project_score_id` — `DELETE /v1/project_score/{project_score_id}` | +| `client.project_tags` | `post_project_tag` — `POST /v1/project_tag`
`put_project_tag` — `PUT /v1/project_tag`
`get_project_tag` — `GET /v1/project_tag`
`get_project_tag_id` — `GET /v1/project_tag/{project_tag_id}`
`patch_project_tag_id` — `PATCH /v1/project_tag/{project_tag_id}`
`delete_project_tag_id` — `DELETE /v1/project_tag/{project_tag_id}` | +| `client.roles` | `post_role` — `POST /v1/role`
`put_role` — `PUT /v1/role`
`get_role` — `GET /v1/role`
`get_role_id` — `GET /v1/role/{role_id}`
`patch_role_id` — `PATCH /v1/role/{role_id}`
`delete_role_id` — `DELETE /v1/role/{role_id}` | +| `client.service_tokens` | `post_service_token` — `POST /v1/service_token`
`put_service_token` — `PUT /v1/service_token`
`delete_service_token` — `DELETE /v1/service_token`
`get_service_token` — `GET /v1/service_token`
`get_service_token_id` — `GET /v1/service_token/{service_token_id}`
`delete_service_token_id` — `DELETE /v1/service_token/{service_token_id}` | +| `client.span_iframes` | `post_span_iframe` — `POST /v1/span_iframe`
`put_span_iframe` — `PUT /v1/span_iframe`
`get_span_iframe` — `GET /v1/span_iframe`
`get_span_iframe_id` — `GET /v1/span_iframe/{span_iframe_id}`
`patch_span_iframe_id` — `PATCH /v1/span_iframe/{span_iframe_id}`
`delete_span_iframe_id` — `DELETE /v1/span_iframe/{span_iframe_id}` | +| `client.users` | `get_user` — `GET /v1/user`
`get_user_id` — `GET /v1/user/{user_id}` | +| `client.views` | `post_view` — `POST /v1/view`
`put_view` — `PUT /v1/view`
`get_view` — `GET /v1/view`
`get_view_id` — `GET /v1/view/{view_id}`
`patch_view_id` — `PATCH /v1/view/{view_id}`
`delete_view_id` — `DELETE /v1/view/{view_id}` | + + +Streaming, log ingestion, eval launch, cross-object insertion, browser CORS, diagnostics, provider proxying, and function invocation remain on specialized SDK paths and are intentionally absent from this client. + +## Public `braintrust.api` exports + + +| Name | Name | Name | +| --- | --- | --- | +| `BraintrustAPIError` | `BraintrustClient` | `BraintrustOpenApiClient` | +| `BraintrustHTTPError` | `BraintrustJSONDecodeError` | `BraintrustRetryExhaustedError` | +| `BraintrustTransportError` | `BraintrustTransportRetryExhaustedError` | `EndpointRouter` | +| `LoginResult` | `OrganizationInfo` | `RequestTarget` | +| `RetryMode` | `RetryPolicy` | | + + +## Public `braintrust.api.types` exports + +Only request and response models reachable directly from reviewed resource method signatures are exported. + + +| Name | Name | Name | +| --- | --- | --- | +| `AISecret` | `Acl` | `AclBatchUpdateRequest` | +| `AclBatchUpdateResponse` | `AclItem` | `AclListOrgResponse` | +| `Agent` | `ApiKey` | `CreateAISecret` | +| `CreateAgent` | `CreateDataset` | `CreateDatasetSnapshot` | +| `CreateEnvironment` | `CreateExperiment` | `CreateFunction` | +| `CreateGroup` | `CreateMCPServer` | `CreateOrgAutomation` | +| `CreateProject` | `CreateProjectAutomation` | `CreateProjectGroup` | +| `CreateProjectScore` | `CreateProjectTag` | `CreatePrompt` | +| `CreateRole` | `CreateServiceTokenOutput` | `CreateSpanIFrame` | +| `CreateView` | `Dataset` | `DatasetSnapshot` | +| `DeleteAISecret` | `DeleteServiceToken` | `DeleteView` | +| `EnvVar` | `Environment` | `Experiment` | +| `FeedbackDatasetEventRequest` | `FeedbackExperimentEventRequest` | `FeedbackResponseSchema` | +| `FetchDatasetEventsResponse` | `FetchEventsRequest` | `FetchExperimentEventsResponse` | +| `Function` | `GetAclResponse` | `GetAgentResponse` | +| `GetAiSecretResponse` | `GetApiKeyResponse` | `GetDatasetResponse` | +| `GetDatasetSnapshotResponse` | `GetEnvVarResponse` | `GetExperimentResponse` | +| `GetFunctionResponse` | `GetGroupResponse` | `GetMcpServerResponse` | +| `GetOrgAutomationResponse` | `GetOrganizationResponse` | `GetProjectAutomationResponse` | +| `GetProjectGroupResponse` | `GetProjectResponse` | `GetProjectScoreResponse` | +| `GetProjectTagResponse` | `GetPromptResponse` | `GetRoleResponse` | +| `GetServiceTokenResponse` | `GetSpanIframeResponse` | `GetUserResponse` | +| `GetViewResponse` | `Group` | `InsertDatasetEventRequest` | +| `InsertEventsResponse` | `InsertExperimentEventRequest` | `ListEnvironmentsResponse` | +| `MCPServer` | `OrgAutomation` | `Organization` | +| `PatchAISecret` | `PatchAgent` | `PatchDataset` | +| `PatchDatasetSnapshot` | `PatchEnvironment` | `PatchExperiment` | +| `PatchFunction` | `PatchGroup` | `PatchMCPServer` | +| `PatchOrgAutomation` | `PatchOrganization` | `PatchOrganizationMembers` | +| `PatchOrganizationMembersOutput` | `PatchProject` | `PatchProjectAutomation` | +| `PatchProjectGroup` | `PatchProjectScore` | `PatchProjectTag` | +| `PatchPrompt` | `PatchRole` | `PatchSpanIFrame` | +| `PatchView` | `Project` | `ProjectAutomation` | +| `ProjectGroup` | `ProjectScore` | `ProjectTag` | +| `Prompt` | `Role` | `ServiceToken` | +| `SpanIFrame` | `SummarizeDatasetResponse` | `SummarizeExperimentResponse` | +| `User` | `View` | | + + +The reference sections above are synchronized with the Python surface by `make generate-api-client`; do not edit those sections manually. diff --git a/py/tests/api_codegen/test_generation.py b/py/tests/api_codegen/test_generation.py index d7cbb98a..08c3a203 100644 --- a/py/tests/api_codegen/test_generation.py +++ b/py/tests/api_codegen/test_generation.py @@ -2,6 +2,9 @@ import copy import re import runpy +import subprocess +import sys +from pathlib import Path import pytest from openapi_codegen import ( @@ -10,11 +13,11 @@ GENERATED_ROOT, SPEC_PATH, CodegenError, - _collect_generated_operations, - _snake_case, atomic_replace_tree, + collect_generated_operations, compare_generated, generate_tree, + generated_resource_name, load_config, read_and_verify_spec, ) @@ -62,7 +65,7 @@ def test_pinned_selected_spec_operations_match_generated_registries(): and tag in operation.get("tags", []) and operation["operationId"] not in specialized_operations } - tree = ast.parse((GENERATED_ROOT / f"{_snake_case(tag)}.py").read_text()) + tree = ast.parse((GENERATED_ROOT / f"{generated_resource_name(tag)}.py").read_text()) registry = next( node.value for node in tree.body @@ -94,6 +97,12 @@ def test_pinned_selected_spec_operations_match_generated_registries(): ) +def test_public_api_readme_matches_reviewed_python_surface(): + script = Path(__file__).resolve().parents[2] / "scripts" / "generate-api-docs.py" + + subprocess.run([sys.executable, str(script), "--check"], check=True) + + def test_pinned_unsupported_tags_are_explicit_and_proxy_is_excluded(): config = load_config(CONFIG_PATH) endpoint_config = config["endpoint_generator"] @@ -112,7 +121,7 @@ def test_pinned_unsupported_tags_are_explicit_and_proxy_is_excluded(): def test_generated_tags_are_wired_to_openapi_client(): config = load_config(CONFIG_PATH) - expected_resources = {_snake_case(tag) for tag in config["endpoint_generator"]["generated_tags"]} + expected_resources = {generated_resource_name(tag) for tag in config["endpoint_generator"]["generated_tags"]} client_tree = ast.parse((GENERATED_ROOT.parent / "client.py").read_text()) openapi_client = next( node for node in client_tree.body if isinstance(node, ast.ClassDef) and node.name == "BraintrustOpenApiClient" @@ -130,7 +139,7 @@ def test_generated_tags_are_wired_to_openapi_client(): def test_public_rest_types_match_generated_request_and_response_models(): config = load_config(CONFIG_PATH) spec = read_and_verify_spec(config, SPEC_PATH) - operations, _ = _collect_generated_operations(spec, config) + operations, _ = collect_generated_operations(spec, config) expected = set() for operation in operations: for type_name in (operation.request_body_type, operation.response_type): diff --git a/py/tests/api_codegen/test_spec_update.py b/py/tests/api_codegen/test_spec_update.py new file mode 100644 index 00000000..afd4759e --- /dev/null +++ b/py/tests/api_codegen/test_spec_update.py @@ -0,0 +1,120 @@ +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + + +UPDATE_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "update-openapi-spec.py" + + +def _write_json(path, value): + content = (json.dumps(value, indent=2) + "\n").encode() + path.write_bytes(content) + return hashlib.sha256(content).hexdigest() + + +def _commit(repository, message): + subprocess.run(["git", "-C", str(repository), "add", "."], check=True) + subprocess.run( + [ + "git", + "-C", + str(repository), + "-c", + "user.name=OpenAPI Test", + "-c", + "user.email=openapi-test@example.com", + "commit", + "-m", + message, + ], + check=True, + capture_output=True, + ) + return subprocess.check_output(["git", "-C", str(repository), "rev-parse", "HEAD"], text=True).strip() + + +def test_updater_pins_local_head_and_writes_review_summary(tmp_path, codegen_config, minimal_spec): + upstream = tmp_path / "upstream" + upstream.mkdir() + subprocess.run(["git", "-C", str(upstream), "init", "--quiet"], check=True) + upstream_spec = upstream / "spec.json" + old_hash = _write_json(upstream_spec, minimal_spec) + old_commit = _commit(upstream, "old spec") + + config = codegen_config + config["spec"].update( + { + "repository": "braintrustdata/braintrust-openapi", + "path": "spec.json", + "commit": old_commit, + "sha256": old_hash, + } + ) + config_path = tmp_path / "config.json" + spec_path = tmp_path / "pinned-spec.json" + summary_path = tmp_path / "summary.md" + _write_json(config_path, config) + spec_path.write_bytes(upstream_spec.read_bytes()) + + minimal_spec["components"]["schemas"]["WidgetDetails"] = { + "type": "object", + "properties": {"count": {"type": "integer"}}, + } + minimal_spec["components"]["schemas"]["Widget"]["properties"]["details"] = { + "$ref": "#/components/schemas/WidgetDetails" + } + _write_json(upstream_spec, minimal_spec) + new_commit = _commit(upstream, "new spec") + committed_content = upstream_spec.read_bytes() + upstream_spec.write_text("uncommitted content must not be pinned\n") + + environment = {**os.environ, "BRAINTRUST_OPENAPI_ROOT": str(upstream)} + result = subprocess.run( + [ + sys.executable, + str(UPDATE_SCRIPT), + "--config", + str(config_path), + "--spec", + str(spec_path), + "--summary-file", + str(summary_path), + ], + check=True, + capture_output=True, + text=True, + env=environment, + ) + + updated_config = json.loads(config_path.read_text()) + assert result.stdout == "changed=true\n" + assert updated_config["spec"]["commit"] == new_commit + assert updated_config["spec"]["sha256"] == hashlib.sha256(committed_content).hexdigest() + assert spec_path.read_bytes() == committed_content + summary = summary_path.read_text() + assert f"compare/{old_commit}...{new_commit}" in summary + assert "1 selected operations, 1 reachable schemas → 1 selected operations, 2 reachable schemas" in summary + assert "All upstream component schemas: 1 → 2" in summary + assert "Added: `WidgetDetails`" in summary + assert "never auto-merged" in summary + + unchanged = subprocess.run( + [ + sys.executable, + str(UPDATE_SCRIPT), + "--config", + str(config_path), + "--spec", + str(spec_path), + "--summary-file", + str(summary_path), + ], + check=True, + capture_output=True, + text=True, + env=environment, + ) + assert unchanged.stdout == "changed=false\n"