Generate PyDABs reference doc pages from generated modules

python/Taskfile.yml

@@ -70,6 +70,8 @@ tasks:
       - ../bundle/schema/jsonschema.json
     generates:
       - databricks/bundles/**
+      - docs/index.rst
+      - docs/databricks.bundles.*.rst
     cmds:
       - |
         find databricks/bundles -type d -mindepth 1 -maxdepth 1 \

python/codegen/codegen/doc_index.rst.tmpl

@@ -0,0 +1,12 @@
+databricks-bundles
+--------------------------------
+
+`databricks-bundles` package implements Python support for Declarative Automation Bundles.
+
+See `What is Python support for Declarative Automation Bundles? (TBD) <#>`_.
+
+
+.. toctree::
+   :maxdepth: 7
+
+${toctree}

python/codegen/codegen/doc_page.rst.tmpl

@@ -0,0 +1,11 @@
+${title}
+===============================
+
+.. currentmodule:: ${module}
+
+**Package:** ``${module}``
+
+Classes
+---------------
+
+.. automodule:: ${module}

python/codegen/codegen/generated_docs.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Generate the Sphinx .rst pages from the resource module dirs under
+databricks/bundles/, so the documented resource list can never drift from the
+generated code. Driven by scanning the output tree, not RESOURCE_NAMESPACE.
+
+The doc structure lives in the *.rst.tmpl templates so it can be reviewed
+separately from this code: doc_page.rst.tmpl (one page per resource) and
+doc_index.rst.tmpl (the index prose header + generated toctree)."""
+
+from pathlib import Path
+from string import Template
+
+import codegen.packages as packages
+
+# .title() mangles acronyms; override those namespaces.
+_TITLE_OVERRIDES = {
+    "sql_warehouses": "SQL Warehouses",
+    "mcp_services": "MCP Services",
+}
+
+
+def _load_template(name: str) -> Template:
+    return Template((Path(__file__).parent / name).read_text())
+
+
+_PAGE_TEMPLATE = _load_template("doc_page.rst.tmpl")
+_INDEX_TEMPLATE = _load_template("doc_index.rst.tmpl")
+
+
+def _title(namespace: str) -> str:
+    return _TITLE_OVERRIDES.get(namespace, namespace.replace("_", " ").title())
+
+
+def write_docs(output: str):
+    docs = Path(output) / "docs"
+    bundles = Path(output) / "databricks" / "bundles"
+
+    # core is hand-written; every other package dir gets a generated page.
+    namespaces = sorted(
+        p.name
+        for p in bundles.iterdir()
+        if p.name != "core" and (p / "__init__.py").exists()
+    )
+
+    # Drop stale pages so a removed resource loses its page; keep core.rst.
+    for rst in docs.glob("databricks.bundles.*.rst"):
+        if rst.name != "databricks.bundles.core.rst":
+            rst.unlink()
+
+    for namespace in namespaces:
+        module = packages.get_root_package(namespace)
+        page = _PAGE_TEMPLATE.substitute(title=_title(namespace), module=module)
+        (docs / f"databricks.bundles.{namespace}.rst").write_text(page)
+
+    entries = ["databricks.bundles.core"] + [
+        packages.get_root_package(ns) for ns in namespaces
+    ]
+    toctree = "\n".join(f"   {entry}" for entry in entries)
+    (docs / "index.rst").write_text(_INDEX_TEMPLATE.substitute(toctree=toctree))
+
+    print(f"Writing {len(namespaces) + 1} doc pages into {docs}")

python/codegen/codegen/jsonschema_patch.py

@@ -18,6 +18,35 @@
     "jobs.SparkJarTask": ["main_class_name"],
 }
 
+# Burn-down list of upstream API descriptions that aren't valid reStructuredText
+# and break the Sphinx docs build. Each entry is a temporary override until the
+# proto comment is fixed upstream; remove it once the fix lands (the no-op guard
+# in override_descriptions flags entries that upstream has already fixed).
+#
+# sql.SpotInstancePolicy: the upstream comment is a hard-wrapped ASCII grid table
+# that docutils rejects as malformed. Rewritten as a list-table.
+# See sqlgateway/scheduler/api/proto/endpoint_common.proto.
+DESCRIPTIONS: dict[str, str] = {
+    "sql.SpotInstancePolicy": (
+        "EndpointSpotInstancePolicy configures whether the endpoint should use spot instances.\n"
+        "\n"
+        "The breakdown of how the EndpointSpotInstancePolicy converts to per cloud configurations is:\n"
+        "\n"
+        ".. list-table::\n"
+        "   :header-rows: 1\n"
+        "\n"
+        "   * - Cloud\n"
+        "     - COST_OPTIMIZED\n"
+        "     - RELIABILITY_OPTIMIZED\n"
+        "   * - AWS\n"
+        "     - On Demand Driver with Spot Executors\n"
+        "     - On Demand Driver and Executors\n"
+        "   * - AZURE\n"
+        "     - On Demand Driver and Executors\n"
+        "     - On Demand Driver and Executors\n"
+    ),
+}
+
 
 def add_extra_required_fields(schemas: dict[str, Schema]):
     output = {}
@@ -41,6 +70,25 @@ def add_extra_required_fields(schemas: dict[str, Schema]):
     return output
 
 
+def override_descriptions(schemas: dict[str, Schema]):
+    if missing := DESCRIPTIONS.keys() - schemas.keys():
+        raise ValueError(f"Cannot override description for unknown schemas: {missing}")
+
+    output = {}
+    for name, schema in schemas.items():
+        if override := DESCRIPTIONS.get(name):
+            if schema.description == override:
+                raise ValueError(
+                    f"Description override for {name} is a no-op; the upstream "
+                    "description was fixed, so remove the override"
+                )
+            output[name] = replace(schema, description=override)
+        else:
+            output[name] = schema
+
+    return output
+
+
 def remove_unsupported_fields(schemas: dict[str, Schema]):
     output = {}
 

python/codegen/codegen/main.py

@@ -6,6 +6,7 @@
 import codegen.aliases_patch as aliases_patch
 import codegen.generated_dataclass as generated_dataclass
 import codegen.generated_dataclass_patch as generated_dataclass_patch
+import codegen.generated_docs as generated_docs
 import codegen.generated_enum as generated_enum
 import codegen.generated_imports as generated_imports
 import codegen.generated_test_cases as generated_test_cases
@@ -24,6 +25,7 @@ def main(output: str):
     schemas = openapi.get_schemas()
     schemas = openapi_patch.add_extra_required_fields(schemas)
     schemas = openapi_patch.remove_unsupported_fields(schemas)
+    schemas = openapi_patch.override_descriptions(schemas)
 
     schemas = _transitively_mark_deprecated_and_private(
         packages.RESOURCE_TYPES, schemas
@@ -56,6 +58,10 @@ def main(output: str):
     # Generate the per-resource ResourceTestCase data driving test_resources.py.
     generated_test_cases.write_test_cases(output, schemas)
 
+    # Generate the Sphinx .rst pages so the documented resource list stays in
+    # sync with the generated modules.
+    generated_docs.write_docs(output)
+
 
 def _transitively_mark_deprecated_and_private(
     roots: list[str],

python/docs/databricks.bundles.alerts.rst

@@ -0,0 +1,11 @@
+Alerts
+===============================
+
+.. currentmodule:: databricks.bundles.alerts
+
+**Package:** ``databricks.bundles.alerts``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.alerts

python/docs/databricks.bundles.apps.rst

@@ -0,0 +1,11 @@
+Apps
+===============================
+
+.. currentmodule:: databricks.bundles.apps
+
+**Package:** ``databricks.bundles.apps``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.apps

python/docs/databricks.bundles.catalogs.rst

@@ -0,0 +1,11 @@
+Catalogs
+===============================
+
+.. currentmodule:: databricks.bundles.catalogs
+
+**Package:** ``databricks.bundles.catalogs``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.catalogs

python/docs/databricks.bundles.cluster_policies.rst

@@ -0,0 +1,11 @@
+Cluster Policies
+===============================
+
+.. currentmodule:: databricks.bundles.cluster_policies
+
+**Package:** ``databricks.bundles.cluster_policies``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.cluster_policies

python/docs/databricks.bundles.clusters.rst

@@ -0,0 +1,11 @@
+Clusters
+===============================
+
+.. currentmodule:: databricks.bundles.clusters
+
+**Package:** ``databricks.bundles.clusters``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.clusters

python/docs/databricks.bundles.dashboards.rst

@@ -0,0 +1,11 @@
+Dashboards
+===============================
+
+.. currentmodule:: databricks.bundles.dashboards
+
+**Package:** ``databricks.bundles.dashboards``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.dashboards

python/docs/databricks.bundles.database_catalogs.rst

@@ -0,0 +1,11 @@
+Database Catalogs
+===============================
+
+.. currentmodule:: databricks.bundles.database_catalogs
+
+**Package:** ``databricks.bundles.database_catalogs``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.database_catalogs

python/docs/databricks.bundles.database_instances.rst

@@ -0,0 +1,11 @@
+Database Instances
+===============================
+
+.. currentmodule:: databricks.bundles.database_instances
+
+**Package:** ``databricks.bundles.database_instances``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.database_instances

python/docs/databricks.bundles.experiments.rst

@@ -0,0 +1,11 @@
+Experiments
+===============================
+
+.. currentmodule:: databricks.bundles.experiments
+
+**Package:** ``databricks.bundles.experiments``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.experiments

python/docs/databricks.bundles.external_locations.rst

@@ -0,0 +1,11 @@
+External Locations
+===============================
+
+.. currentmodule:: databricks.bundles.external_locations
+
+**Package:** ``databricks.bundles.external_locations``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.external_locations

python/docs/databricks.bundles.genie_spaces.rst

@@ -0,0 +1,11 @@
+Genie Spaces
+===============================
+
+.. currentmodule:: databricks.bundles.genie_spaces
+
+**Package:** ``databricks.bundles.genie_spaces``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.genie_spaces

python/docs/databricks.bundles.instance_pools.rst

@@ -0,0 +1,11 @@
+Instance Pools
+===============================
+
+.. currentmodule:: databricks.bundles.instance_pools
+
+**Package:** ``databricks.bundles.instance_pools``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.instance_pools

python/docs/databricks.bundles.job_runs.rst

@@ -0,0 +1,11 @@
+Job Runs
+===============================
+
+.. currentmodule:: databricks.bundles.job_runs
+
+**Package:** ``databricks.bundles.job_runs``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.job_runs

python/docs/databricks.bundles.mcp_services.rst

@@ -0,0 +1,11 @@
+MCP Services
+===============================
+
+.. currentmodule:: databricks.bundles.mcp_services
+
+**Package:** ``databricks.bundles.mcp_services``
+
+Classes
+---------------
+
+.. automodule:: databricks.bundles.mcp_services