From 3bb652d0e9360fb6ef8548c0d99b07bd9c5c61ff Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 17 Sep 2026 09:50:40 +0000 Subject: [PATCH 1/3] Generate PyDABs reference doc pages from generated modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Sphinx .rst pages under python/docs/ were hand-written and listed only 5 resources, while codegen now emits 31 resource modules — so the published Python reference docs (databricks.github.io/cli/python) went stale. Generate the per-resource .rst pages and index.rst toctree in pydabs-codegen, driven by scanning the generated databricks/bundles/*/ package dirs (decoupled from RESOURCE_NAMESPACE). The existing hand-written pages regenerate byte-identical; core.rst stays hand-written. CI's generate-check + git diff now enforces the doc list never drifts from the generated resources. Co-authored-by: Isaac --- python/Taskfile.yml | 2 + python/codegen/codegen/generated_docs.py | 83 +++++++++++++++++++ python/codegen/codegen/main.py | 5 ++ python/docs/databricks.bundles.alerts.rst | 11 +++ python/docs/databricks.bundles.apps.rst | 11 +++ python/docs/databricks.bundles.catalogs.rst | 11 +++ .../databricks.bundles.cluster_policies.rst | 11 +++ python/docs/databricks.bundles.clusters.rst | 11 +++ python/docs/databricks.bundles.dashboards.rst | 11 +++ .../databricks.bundles.database_catalogs.rst | 11 +++ .../databricks.bundles.database_instances.rst | 11 +++ .../docs/databricks.bundles.experiments.rst | 11 +++ .../databricks.bundles.external_locations.rst | 11 +++ .../docs/databricks.bundles.genie_spaces.rst | 11 +++ .../databricks.bundles.instance_pools.rst | 11 +++ python/docs/databricks.bundles.job_runs.rst | 11 +++ .../docs/databricks.bundles.mcp_services.rst | 11 +++ ...bricks.bundles.model_provider_services.rst | 11 +++ .../databricks.bundles.model_services.rst | 11 +++ ...bricks.bundles.model_serving_endpoints.rst | 11 +++ python/docs/databricks.bundles.models.rst | 11 +++ ...ks.bundles.postgres_snapshot_schedules.rst | 11 +++ .../databricks.bundles.quality_monitors.rst | 11 +++ .../databricks.bundles.registered_models.rst | 11 +++ .../docs/databricks.bundles.secret_scopes.rst | 11 +++ python/docs/databricks.bundles.secrets.rst | 11 +++ .../databricks.bundles.sql_warehouses.rst | 11 +++ ...abricks.bundles.synced_database_tables.rst | 11 +++ ...bricks.bundles.vector_search_endpoints.rst | 11 +++ ...tabricks.bundles.vector_search_indexes.rst | 11 +++ python/docs/index.rst | 27 ++++++ 31 files changed, 414 insertions(+) create mode 100644 python/codegen/codegen/generated_docs.py create mode 100644 python/docs/databricks.bundles.alerts.rst create mode 100644 python/docs/databricks.bundles.apps.rst create mode 100644 python/docs/databricks.bundles.catalogs.rst create mode 100644 python/docs/databricks.bundles.cluster_policies.rst create mode 100644 python/docs/databricks.bundles.clusters.rst create mode 100644 python/docs/databricks.bundles.dashboards.rst create mode 100644 python/docs/databricks.bundles.database_catalogs.rst create mode 100644 python/docs/databricks.bundles.database_instances.rst create mode 100644 python/docs/databricks.bundles.experiments.rst create mode 100644 python/docs/databricks.bundles.external_locations.rst create mode 100644 python/docs/databricks.bundles.genie_spaces.rst create mode 100644 python/docs/databricks.bundles.instance_pools.rst create mode 100644 python/docs/databricks.bundles.job_runs.rst create mode 100644 python/docs/databricks.bundles.mcp_services.rst create mode 100644 python/docs/databricks.bundles.model_provider_services.rst create mode 100644 python/docs/databricks.bundles.model_services.rst create mode 100644 python/docs/databricks.bundles.model_serving_endpoints.rst create mode 100644 python/docs/databricks.bundles.models.rst create mode 100644 python/docs/databricks.bundles.postgres_snapshot_schedules.rst create mode 100644 python/docs/databricks.bundles.quality_monitors.rst create mode 100644 python/docs/databricks.bundles.registered_models.rst create mode 100644 python/docs/databricks.bundles.secret_scopes.rst create mode 100644 python/docs/databricks.bundles.secrets.rst create mode 100644 python/docs/databricks.bundles.sql_warehouses.rst create mode 100644 python/docs/databricks.bundles.synced_database_tables.rst create mode 100644 python/docs/databricks.bundles.vector_search_endpoints.rst create mode 100644 python/docs/databricks.bundles.vector_search_indexes.rst diff --git a/python/Taskfile.yml b/python/Taskfile.yml index 49621efd6a6..fd4b18687be 100644 --- a/python/Taskfile.yml +++ b/python/Taskfile.yml @@ -70,6 +70,8 @@ tasks: - ../bundle/schema/jsonschema.json generates: - databricks/bundles/** + - docs/index.rst + - docs/databricks.bundles.*.rst cmds: - | find databricks/bundles -type d -mindepth 1 -maxdepth 1 \ diff --git a/python/codegen/codegen/generated_docs.py b/python/codegen/codegen/generated_docs.py new file mode 100644 index 00000000000..f54aefd747b --- /dev/null +++ b/python/codegen/codegen/generated_docs.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Generate the Sphinx .rst pages from the resource module dirs under +databricks/bundles/, so the documented resource list can never drift from the +generated code. Driven by scanning the output tree, not RESOURCE_NAMESPACE.""" + +from pathlib import Path + +import codegen.packages as packages + +# .title() mangles acronyms; override those namespaces. +_TITLE_OVERRIDES = { + "sql_warehouses": "SQL Warehouses", + "mcp_services": "MCP Services", +} + +# Fixed-width underlines matching the hand-written pages so the resource pages +# that already exist regenerate byte-identical. +_H1 = "=" * 31 +_H2 = "-" * 15 + +# Prose header kept verbatim from the hand-written index.rst; only the toctree +# below it is generated. +_INDEX_HEADER = """\ +databricks-bundles +-------------------------------- + +`databricks-bundles` package implements Python support for Declarative Automation Bundles. + +See `What is Python support for Declarative Automation Bundles? (TBD) <#>`_. + + +.. toctree:: + :maxdepth: 7 +""" + + +def _title(namespace: str) -> str: + return _TITLE_OVERRIDES.get(namespace, namespace.replace("_", " ").title()) + + +def _page(namespace: str) -> str: + module = packages.get_root_package(namespace) + return ( + f"{_title(namespace)}\n" + f"{_H1}\n" + "\n" + f".. currentmodule:: {module}\n" + "\n" + f"**Package:** ``{module}``\n" + "\n" + "Classes\n" + f"{_H2}\n" + "\n" + f".. automodule:: {module}\n" + ) + + +def write_docs(output: str): + docs = Path(output) / "docs" + bundles = Path(output) / "databricks" / "bundles" + + # core is hand-written; every other package dir gets a generated page. + namespaces = sorted( + p.name + for p in bundles.iterdir() + if p.name != "core" and (p / "__init__.py").exists() + ) + + # Drop stale pages so a removed resource loses its page; keep core.rst. + for rst in docs.glob("databricks.bundles.*.rst"): + if rst.name != "databricks.bundles.core.rst": + rst.unlink() + + for namespace in namespaces: + (docs / f"databricks.bundles.{namespace}.rst").write_text(_page(namespace)) + + entries = ["databricks.bundles.core"] + [ + packages.get_root_package(ns) for ns in namespaces + ] + toctree = "".join(f" {entry}\n" for entry in entries) + (docs / "index.rst").write_text(_INDEX_HEADER + "\n" + toctree) + + print(f"Writing {len(namespaces) + 1} doc pages into {docs}") diff --git a/python/codegen/codegen/main.py b/python/codegen/codegen/main.py index e9ae2047b1e..9f56b546b00 100644 --- a/python/codegen/codegen/main.py +++ b/python/codegen/codegen/main.py @@ -6,6 +6,7 @@ import codegen.aliases_patch as aliases_patch import codegen.generated_dataclass as generated_dataclass import codegen.generated_dataclass_patch as generated_dataclass_patch +import codegen.generated_docs as generated_docs import codegen.generated_enum as generated_enum import codegen.generated_imports as generated_imports import codegen.generated_test_cases as generated_test_cases @@ -56,6 +57,10 @@ def main(output: str): # Generate the per-resource ResourceTestCase data driving test_resources.py. generated_test_cases.write_test_cases(output, schemas) + # Generate the Sphinx .rst pages so the documented resource list stays in + # sync with the generated modules. + generated_docs.write_docs(output) + def _transitively_mark_deprecated_and_private( roots: list[str], diff --git a/python/docs/databricks.bundles.alerts.rst b/python/docs/databricks.bundles.alerts.rst new file mode 100644 index 00000000000..08a621c813d --- /dev/null +++ b/python/docs/databricks.bundles.alerts.rst @@ -0,0 +1,11 @@ +Alerts +=============================== + +.. currentmodule:: databricks.bundles.alerts + +**Package:** ``databricks.bundles.alerts`` + +Classes +--------------- + +.. automodule:: databricks.bundles.alerts diff --git a/python/docs/databricks.bundles.apps.rst b/python/docs/databricks.bundles.apps.rst new file mode 100644 index 00000000000..a1ccd11ac58 --- /dev/null +++ b/python/docs/databricks.bundles.apps.rst @@ -0,0 +1,11 @@ +Apps +=============================== + +.. currentmodule:: databricks.bundles.apps + +**Package:** ``databricks.bundles.apps`` + +Classes +--------------- + +.. automodule:: databricks.bundles.apps diff --git a/python/docs/databricks.bundles.catalogs.rst b/python/docs/databricks.bundles.catalogs.rst new file mode 100644 index 00000000000..bbd9054723d --- /dev/null +++ b/python/docs/databricks.bundles.catalogs.rst @@ -0,0 +1,11 @@ +Catalogs +=============================== + +.. currentmodule:: databricks.bundles.catalogs + +**Package:** ``databricks.bundles.catalogs`` + +Classes +--------------- + +.. automodule:: databricks.bundles.catalogs diff --git a/python/docs/databricks.bundles.cluster_policies.rst b/python/docs/databricks.bundles.cluster_policies.rst new file mode 100644 index 00000000000..a82cbec3e36 --- /dev/null +++ b/python/docs/databricks.bundles.cluster_policies.rst @@ -0,0 +1,11 @@ +Cluster Policies +=============================== + +.. currentmodule:: databricks.bundles.cluster_policies + +**Package:** ``databricks.bundles.cluster_policies`` + +Classes +--------------- + +.. automodule:: databricks.bundles.cluster_policies diff --git a/python/docs/databricks.bundles.clusters.rst b/python/docs/databricks.bundles.clusters.rst new file mode 100644 index 00000000000..0babd19d3aa --- /dev/null +++ b/python/docs/databricks.bundles.clusters.rst @@ -0,0 +1,11 @@ +Clusters +=============================== + +.. currentmodule:: databricks.bundles.clusters + +**Package:** ``databricks.bundles.clusters`` + +Classes +--------------- + +.. automodule:: databricks.bundles.clusters diff --git a/python/docs/databricks.bundles.dashboards.rst b/python/docs/databricks.bundles.dashboards.rst new file mode 100644 index 00000000000..70f7d1539af --- /dev/null +++ b/python/docs/databricks.bundles.dashboards.rst @@ -0,0 +1,11 @@ +Dashboards +=============================== + +.. currentmodule:: databricks.bundles.dashboards + +**Package:** ``databricks.bundles.dashboards`` + +Classes +--------------- + +.. automodule:: databricks.bundles.dashboards diff --git a/python/docs/databricks.bundles.database_catalogs.rst b/python/docs/databricks.bundles.database_catalogs.rst new file mode 100644 index 00000000000..005aae777cf --- /dev/null +++ b/python/docs/databricks.bundles.database_catalogs.rst @@ -0,0 +1,11 @@ +Database Catalogs +=============================== + +.. currentmodule:: databricks.bundles.database_catalogs + +**Package:** ``databricks.bundles.database_catalogs`` + +Classes +--------------- + +.. automodule:: databricks.bundles.database_catalogs diff --git a/python/docs/databricks.bundles.database_instances.rst b/python/docs/databricks.bundles.database_instances.rst new file mode 100644 index 00000000000..eea7a14d4b3 --- /dev/null +++ b/python/docs/databricks.bundles.database_instances.rst @@ -0,0 +1,11 @@ +Database Instances +=============================== + +.. currentmodule:: databricks.bundles.database_instances + +**Package:** ``databricks.bundles.database_instances`` + +Classes +--------------- + +.. automodule:: databricks.bundles.database_instances diff --git a/python/docs/databricks.bundles.experiments.rst b/python/docs/databricks.bundles.experiments.rst new file mode 100644 index 00000000000..b2ed4d21ef0 --- /dev/null +++ b/python/docs/databricks.bundles.experiments.rst @@ -0,0 +1,11 @@ +Experiments +=============================== + +.. currentmodule:: databricks.bundles.experiments + +**Package:** ``databricks.bundles.experiments`` + +Classes +--------------- + +.. automodule:: databricks.bundles.experiments diff --git a/python/docs/databricks.bundles.external_locations.rst b/python/docs/databricks.bundles.external_locations.rst new file mode 100644 index 00000000000..0044ce58e66 --- /dev/null +++ b/python/docs/databricks.bundles.external_locations.rst @@ -0,0 +1,11 @@ +External Locations +=============================== + +.. currentmodule:: databricks.bundles.external_locations + +**Package:** ``databricks.bundles.external_locations`` + +Classes +--------------- + +.. automodule:: databricks.bundles.external_locations diff --git a/python/docs/databricks.bundles.genie_spaces.rst b/python/docs/databricks.bundles.genie_spaces.rst new file mode 100644 index 00000000000..0e7b8e13b77 --- /dev/null +++ b/python/docs/databricks.bundles.genie_spaces.rst @@ -0,0 +1,11 @@ +Genie Spaces +=============================== + +.. currentmodule:: databricks.bundles.genie_spaces + +**Package:** ``databricks.bundles.genie_spaces`` + +Classes +--------------- + +.. automodule:: databricks.bundles.genie_spaces diff --git a/python/docs/databricks.bundles.instance_pools.rst b/python/docs/databricks.bundles.instance_pools.rst new file mode 100644 index 00000000000..57fc6543340 --- /dev/null +++ b/python/docs/databricks.bundles.instance_pools.rst @@ -0,0 +1,11 @@ +Instance Pools +=============================== + +.. currentmodule:: databricks.bundles.instance_pools + +**Package:** ``databricks.bundles.instance_pools`` + +Classes +--------------- + +.. automodule:: databricks.bundles.instance_pools diff --git a/python/docs/databricks.bundles.job_runs.rst b/python/docs/databricks.bundles.job_runs.rst new file mode 100644 index 00000000000..1cd97700c53 --- /dev/null +++ b/python/docs/databricks.bundles.job_runs.rst @@ -0,0 +1,11 @@ +Job Runs +=============================== + +.. currentmodule:: databricks.bundles.job_runs + +**Package:** ``databricks.bundles.job_runs`` + +Classes +--------------- + +.. automodule:: databricks.bundles.job_runs diff --git a/python/docs/databricks.bundles.mcp_services.rst b/python/docs/databricks.bundles.mcp_services.rst new file mode 100644 index 00000000000..d13fd3f7956 --- /dev/null +++ b/python/docs/databricks.bundles.mcp_services.rst @@ -0,0 +1,11 @@ +MCP Services +=============================== + +.. currentmodule:: databricks.bundles.mcp_services + +**Package:** ``databricks.bundles.mcp_services`` + +Classes +--------------- + +.. automodule:: databricks.bundles.mcp_services diff --git a/python/docs/databricks.bundles.model_provider_services.rst b/python/docs/databricks.bundles.model_provider_services.rst new file mode 100644 index 00000000000..657522bacc8 --- /dev/null +++ b/python/docs/databricks.bundles.model_provider_services.rst @@ -0,0 +1,11 @@ +Model Provider Services +=============================== + +.. currentmodule:: databricks.bundles.model_provider_services + +**Package:** ``databricks.bundles.model_provider_services`` + +Classes +--------------- + +.. automodule:: databricks.bundles.model_provider_services diff --git a/python/docs/databricks.bundles.model_services.rst b/python/docs/databricks.bundles.model_services.rst new file mode 100644 index 00000000000..38d4ebdff1a --- /dev/null +++ b/python/docs/databricks.bundles.model_services.rst @@ -0,0 +1,11 @@ +Model Services +=============================== + +.. currentmodule:: databricks.bundles.model_services + +**Package:** ``databricks.bundles.model_services`` + +Classes +--------------- + +.. automodule:: databricks.bundles.model_services diff --git a/python/docs/databricks.bundles.model_serving_endpoints.rst b/python/docs/databricks.bundles.model_serving_endpoints.rst new file mode 100644 index 00000000000..842dc7e722c --- /dev/null +++ b/python/docs/databricks.bundles.model_serving_endpoints.rst @@ -0,0 +1,11 @@ +Model Serving Endpoints +=============================== + +.. currentmodule:: databricks.bundles.model_serving_endpoints + +**Package:** ``databricks.bundles.model_serving_endpoints`` + +Classes +--------------- + +.. automodule:: databricks.bundles.model_serving_endpoints diff --git a/python/docs/databricks.bundles.models.rst b/python/docs/databricks.bundles.models.rst new file mode 100644 index 00000000000..d0c71bc68ba --- /dev/null +++ b/python/docs/databricks.bundles.models.rst @@ -0,0 +1,11 @@ +Models +=============================== + +.. currentmodule:: databricks.bundles.models + +**Package:** ``databricks.bundles.models`` + +Classes +--------------- + +.. automodule:: databricks.bundles.models diff --git a/python/docs/databricks.bundles.postgres_snapshot_schedules.rst b/python/docs/databricks.bundles.postgres_snapshot_schedules.rst new file mode 100644 index 00000000000..20108a34625 --- /dev/null +++ b/python/docs/databricks.bundles.postgres_snapshot_schedules.rst @@ -0,0 +1,11 @@ +Postgres Snapshot Schedules +=============================== + +.. currentmodule:: databricks.bundles.postgres_snapshot_schedules + +**Package:** ``databricks.bundles.postgres_snapshot_schedules`` + +Classes +--------------- + +.. automodule:: databricks.bundles.postgres_snapshot_schedules diff --git a/python/docs/databricks.bundles.quality_monitors.rst b/python/docs/databricks.bundles.quality_monitors.rst new file mode 100644 index 00000000000..069916c1f1a --- /dev/null +++ b/python/docs/databricks.bundles.quality_monitors.rst @@ -0,0 +1,11 @@ +Quality Monitors +=============================== + +.. currentmodule:: databricks.bundles.quality_monitors + +**Package:** ``databricks.bundles.quality_monitors`` + +Classes +--------------- + +.. automodule:: databricks.bundles.quality_monitors diff --git a/python/docs/databricks.bundles.registered_models.rst b/python/docs/databricks.bundles.registered_models.rst new file mode 100644 index 00000000000..8389050f4cd --- /dev/null +++ b/python/docs/databricks.bundles.registered_models.rst @@ -0,0 +1,11 @@ +Registered Models +=============================== + +.. currentmodule:: databricks.bundles.registered_models + +**Package:** ``databricks.bundles.registered_models`` + +Classes +--------------- + +.. automodule:: databricks.bundles.registered_models diff --git a/python/docs/databricks.bundles.secret_scopes.rst b/python/docs/databricks.bundles.secret_scopes.rst new file mode 100644 index 00000000000..088b6902283 --- /dev/null +++ b/python/docs/databricks.bundles.secret_scopes.rst @@ -0,0 +1,11 @@ +Secret Scopes +=============================== + +.. currentmodule:: databricks.bundles.secret_scopes + +**Package:** ``databricks.bundles.secret_scopes`` + +Classes +--------------- + +.. automodule:: databricks.bundles.secret_scopes diff --git a/python/docs/databricks.bundles.secrets.rst b/python/docs/databricks.bundles.secrets.rst new file mode 100644 index 00000000000..64ff4ad70b3 --- /dev/null +++ b/python/docs/databricks.bundles.secrets.rst @@ -0,0 +1,11 @@ +Secrets +=============================== + +.. currentmodule:: databricks.bundles.secrets + +**Package:** ``databricks.bundles.secrets`` + +Classes +--------------- + +.. automodule:: databricks.bundles.secrets diff --git a/python/docs/databricks.bundles.sql_warehouses.rst b/python/docs/databricks.bundles.sql_warehouses.rst new file mode 100644 index 00000000000..774ec65cf3f --- /dev/null +++ b/python/docs/databricks.bundles.sql_warehouses.rst @@ -0,0 +1,11 @@ +SQL Warehouses +=============================== + +.. currentmodule:: databricks.bundles.sql_warehouses + +**Package:** ``databricks.bundles.sql_warehouses`` + +Classes +--------------- + +.. automodule:: databricks.bundles.sql_warehouses diff --git a/python/docs/databricks.bundles.synced_database_tables.rst b/python/docs/databricks.bundles.synced_database_tables.rst new file mode 100644 index 00000000000..5d63776b076 --- /dev/null +++ b/python/docs/databricks.bundles.synced_database_tables.rst @@ -0,0 +1,11 @@ +Synced Database Tables +=============================== + +.. currentmodule:: databricks.bundles.synced_database_tables + +**Package:** ``databricks.bundles.synced_database_tables`` + +Classes +--------------- + +.. automodule:: databricks.bundles.synced_database_tables diff --git a/python/docs/databricks.bundles.vector_search_endpoints.rst b/python/docs/databricks.bundles.vector_search_endpoints.rst new file mode 100644 index 00000000000..48a6f792783 --- /dev/null +++ b/python/docs/databricks.bundles.vector_search_endpoints.rst @@ -0,0 +1,11 @@ +Vector Search Endpoints +=============================== + +.. currentmodule:: databricks.bundles.vector_search_endpoints + +**Package:** ``databricks.bundles.vector_search_endpoints`` + +Classes +--------------- + +.. automodule:: databricks.bundles.vector_search_endpoints diff --git a/python/docs/databricks.bundles.vector_search_indexes.rst b/python/docs/databricks.bundles.vector_search_indexes.rst new file mode 100644 index 00000000000..0766cb57517 --- /dev/null +++ b/python/docs/databricks.bundles.vector_search_indexes.rst @@ -0,0 +1,11 @@ +Vector Search Indexes +=============================== + +.. currentmodule:: databricks.bundles.vector_search_indexes + +**Package:** ``databricks.bundles.vector_search_indexes`` + +Classes +--------------- + +.. automodule:: databricks.bundles.vector_search_indexes diff --git a/python/docs/index.rst b/python/docs/index.rst index 96eada6fdd4..1672685fab8 100644 --- a/python/docs/index.rst +++ b/python/docs/index.rst @@ -10,7 +10,34 @@ See `What is Python support for Declarative Automation Bundles? (TBD) <#>`_. :maxdepth: 7 databricks.bundles.core + databricks.bundles.alerts + databricks.bundles.apps + databricks.bundles.catalogs + databricks.bundles.cluster_policies + databricks.bundles.clusters + databricks.bundles.dashboards + databricks.bundles.database_catalogs + databricks.bundles.database_instances + databricks.bundles.experiments + databricks.bundles.external_locations + databricks.bundles.genie_spaces + databricks.bundles.instance_pools + databricks.bundles.job_runs databricks.bundles.jobs + databricks.bundles.mcp_services + databricks.bundles.model_provider_services + databricks.bundles.model_services + databricks.bundles.model_serving_endpoints + databricks.bundles.models databricks.bundles.pipelines + databricks.bundles.postgres_snapshot_schedules + databricks.bundles.quality_monitors + databricks.bundles.registered_models databricks.bundles.schemas + databricks.bundles.secret_scopes + databricks.bundles.secrets + databricks.bundles.sql_warehouses + databricks.bundles.synced_database_tables + databricks.bundles.vector_search_endpoints + databricks.bundles.vector_search_indexes databricks.bundles.volumes From 318c71668aa801618e3ef013a5a58a72aa18705d Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 17 Sep 2026 10:04:07 +0000 Subject: [PATCH 2/3] Move PyDABs doc structure into .rst.tmpl templates Extract the RST page and index layout from generated_docs.py into doc_page.rst.tmpl and doc_index.rst.tmpl, loaded via string.Template like the existing *.py.tmpl codegen templates. This lets the doc structure be reviewed and tracked separately from the generation code. Output is byte-identical. Co-authored-by: Isaac --- python/codegen/codegen/doc_index.rst.tmpl | 12 ++++++ python/codegen/codegen/doc_page.rst.tmpl | 11 +++++ python/codegen/codegen/generated_docs.py | 52 +++++++---------------- 3 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 python/codegen/codegen/doc_index.rst.tmpl create mode 100644 python/codegen/codegen/doc_page.rst.tmpl diff --git a/python/codegen/codegen/doc_index.rst.tmpl b/python/codegen/codegen/doc_index.rst.tmpl new file mode 100644 index 00000000000..e7453f3920a --- /dev/null +++ b/python/codegen/codegen/doc_index.rst.tmpl @@ -0,0 +1,12 @@ +databricks-bundles +-------------------------------- + +`databricks-bundles` package implements Python support for Declarative Automation Bundles. + +See `What is Python support for Declarative Automation Bundles? (TBD) <#>`_. + + +.. toctree:: + :maxdepth: 7 + +${toctree} diff --git a/python/codegen/codegen/doc_page.rst.tmpl b/python/codegen/codegen/doc_page.rst.tmpl new file mode 100644 index 00000000000..3278ef09c30 --- /dev/null +++ b/python/codegen/codegen/doc_page.rst.tmpl @@ -0,0 +1,11 @@ +${title} +=============================== + +.. currentmodule:: ${module} + +**Package:** ``${module}`` + +Classes +--------------- + +.. automodule:: ${module} diff --git a/python/codegen/codegen/generated_docs.py b/python/codegen/codegen/generated_docs.py index f54aefd747b..3ae866a09d3 100644 --- a/python/codegen/codegen/generated_docs.py +++ b/python/codegen/codegen/generated_docs.py @@ -1,9 +1,14 @@ #!/usr/bin/env python3 """Generate the Sphinx .rst pages from the resource module dirs under databricks/bundles/, so the documented resource list can never drift from the -generated code. Driven by scanning the output tree, not RESOURCE_NAMESPACE.""" +generated code. Driven by scanning the output tree, not RESOURCE_NAMESPACE. + +The doc structure lives in the *.rst.tmpl templates so it can be reviewed +separately from this code: doc_page.rst.tmpl (one page per resource) and +doc_index.rst.tmpl (the index prose header + generated toctree).""" from pathlib import Path +from string import Template import codegen.packages as packages @@ -13,48 +18,19 @@ "mcp_services": "MCP Services", } -# Fixed-width underlines matching the hand-written pages so the resource pages -# that already exist regenerate byte-identical. -_H1 = "=" * 31 -_H2 = "-" * 15 - -# Prose header kept verbatim from the hand-written index.rst; only the toctree -# below it is generated. -_INDEX_HEADER = """\ -databricks-bundles --------------------------------- - -`databricks-bundles` package implements Python support for Declarative Automation Bundles. -See `What is Python support for Declarative Automation Bundles? (TBD) <#>`_. +def _load_template(name: str) -> Template: + return Template((Path(__file__).parent / name).read_text()) -.. toctree:: - :maxdepth: 7 -""" +_PAGE_TEMPLATE = _load_template("doc_page.rst.tmpl") +_INDEX_TEMPLATE = _load_template("doc_index.rst.tmpl") def _title(namespace: str) -> str: return _TITLE_OVERRIDES.get(namespace, namespace.replace("_", " ").title()) -def _page(namespace: str) -> str: - module = packages.get_root_package(namespace) - return ( - f"{_title(namespace)}\n" - f"{_H1}\n" - "\n" - f".. currentmodule:: {module}\n" - "\n" - f"**Package:** ``{module}``\n" - "\n" - "Classes\n" - f"{_H2}\n" - "\n" - f".. automodule:: {module}\n" - ) - - def write_docs(output: str): docs = Path(output) / "docs" bundles = Path(output) / "databricks" / "bundles" @@ -72,12 +48,14 @@ def write_docs(output: str): rst.unlink() for namespace in namespaces: - (docs / f"databricks.bundles.{namespace}.rst").write_text(_page(namespace)) + module = packages.get_root_package(namespace) + page = _PAGE_TEMPLATE.substitute(title=_title(namespace), module=module) + (docs / f"databricks.bundles.{namespace}.rst").write_text(page) entries = ["databricks.bundles.core"] + [ packages.get_root_package(ns) for ns in namespaces ] - toctree = "".join(f" {entry}\n" for entry in entries) - (docs / "index.rst").write_text(_INDEX_HEADER + "\n" + toctree) + toctree = "\n".join(f" {entry}" for entry in entries) + (docs / "index.rst").write_text(_INDEX_TEMPLATE.substitute(toctree=toctree)) print(f"Writing {len(namespaces) + 1} doc pages into {docs}") From eb02251b5f19f03bb731d761353c4d144103d57c Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 17 Sep 2026 15:09:10 +0000 Subject: [PATCH 3/3] Override malformed upstream descriptions that break the docs build The sql.SpotInstancePolicy description is a hard-wrapped ASCII grid table that docutils rejects as malformed, so the sql_warehouses page rendered broken and the Sphinx build logged errors. Add a burn-down override list in jsonschema_patch that rewrites such upstream descriptions into valid reStructuredText until the proto comment is fixed upstream (see sqlgateway/scheduler/api/proto/endpoint_common.proto). The override raises if it becomes a no-op, flagging entries to remove once upstream lands. Pairs with the strict (-W) docs build added later in the stack. Co-authored-by: Isaac --- python/codegen/codegen/jsonschema_patch.py | 48 +++++++++++++++++++ python/codegen/codegen/main.py | 1 + .../_models/spot_instance_policy.py | 30 +++++++----- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/python/codegen/codegen/jsonschema_patch.py b/python/codegen/codegen/jsonschema_patch.py index 81267393637..ae27d16a5a0 100644 --- a/python/codegen/codegen/jsonschema_patch.py +++ b/python/codegen/codegen/jsonschema_patch.py @@ -18,6 +18,35 @@ "jobs.SparkJarTask": ["main_class_name"], } +# Burn-down list of upstream API descriptions that aren't valid reStructuredText +# and break the Sphinx docs build. Each entry is a temporary override until the +# proto comment is fixed upstream; remove it once the fix lands (the no-op guard +# in override_descriptions flags entries that upstream has already fixed). +# +# sql.SpotInstancePolicy: the upstream comment is a hard-wrapped ASCII grid table +# that docutils rejects as malformed. Rewritten as a list-table. +# See sqlgateway/scheduler/api/proto/endpoint_common.proto. +DESCRIPTIONS: dict[str, str] = { + "sql.SpotInstancePolicy": ( + "EndpointSpotInstancePolicy configures whether the endpoint should use spot instances.\n" + "\n" + "The breakdown of how the EndpointSpotInstancePolicy converts to per cloud configurations is:\n" + "\n" + ".. list-table::\n" + " :header-rows: 1\n" + "\n" + " * - Cloud\n" + " - COST_OPTIMIZED\n" + " - RELIABILITY_OPTIMIZED\n" + " * - AWS\n" + " - On Demand Driver with Spot Executors\n" + " - On Demand Driver and Executors\n" + " * - AZURE\n" + " - On Demand Driver and Executors\n" + " - On Demand Driver and Executors\n" + ), +} + def add_extra_required_fields(schemas: dict[str, Schema]): output = {} @@ -41,6 +70,25 @@ def add_extra_required_fields(schemas: dict[str, Schema]): return output +def override_descriptions(schemas: dict[str, Schema]): + if missing := DESCRIPTIONS.keys() - schemas.keys(): + raise ValueError(f"Cannot override description for unknown schemas: {missing}") + + output = {} + for name, schema in schemas.items(): + if override := DESCRIPTIONS.get(name): + if schema.description == override: + raise ValueError( + f"Description override for {name} is a no-op; the upstream " + "description was fixed, so remove the override" + ) + output[name] = replace(schema, description=override) + else: + output[name] = schema + + return output + + def remove_unsupported_fields(schemas: dict[str, Schema]): output = {} diff --git a/python/codegen/codegen/main.py b/python/codegen/codegen/main.py index 9f56b546b00..3dea5144de9 100644 --- a/python/codegen/codegen/main.py +++ b/python/codegen/codegen/main.py @@ -25,6 +25,7 @@ def main(output: str): schemas = openapi.get_schemas() schemas = openapi_patch.add_extra_required_fields(schemas) schemas = openapi_patch.remove_unsupported_fields(schemas) + schemas = openapi_patch.override_descriptions(schemas) schemas = _transitively_mark_deprecated_and_private( packages.RESOURCE_TYPES, schemas diff --git a/python/databricks/bundles/sql_warehouses/_models/spot_instance_policy.py b/python/databricks/bundles/sql_warehouses/_models/spot_instance_policy.py index d44908ae895..232731b57ff 100644 --- a/python/databricks/bundles/sql_warehouses/_models/spot_instance_policy.py +++ b/python/databricks/bundles/sql_warehouses/_models/spot_instance_policy.py @@ -6,19 +6,23 @@ class SpotInstancePolicy(Enum): """ - EndpointSpotInstancePolicy configures whether the endpoint should use spot - instances. - - The breakdown of how the EndpointSpotInstancePolicy converts to per cloud - configurations is: - - +-------+--------------------------------------+--------------------------------+ - | Cloud | COST_OPTIMIZED | RELIABILITY_OPTIMIZED | - +-------+--------------------------------------+--------------------------------+ - | AWS | On Demand Driver with Spot Executors | On Demand Driver and - Executors | | AZURE | On Demand Driver and Executors | On Demand Driver - and Executors | - +-------+--------------------------------------+--------------------------------+ + EndpointSpotInstancePolicy configures whether the endpoint should use spot instances. + + The breakdown of how the EndpointSpotInstancePolicy converts to per cloud configurations is: + + .. list-table:: + :header-rows: 1 + + * - Cloud + - COST_OPTIMIZED + - RELIABILITY_OPTIMIZED + * - AWS + - On Demand Driver with Spot Executors + - On Demand Driver and Executors + * - AZURE + - On Demand Driver and Executors + - On Demand Driver and Executors + """ POLICY_UNSPECIFIED = "POLICY_UNSPECIFIED"