diff --git a/python/Taskfile.yml b/python/Taskfile.yml index 49621efd6a6..fd4b18687be 100644 --- a/python/Taskfile.yml +++ b/python/Taskfile.yml @@ -70,6 +70,8 @@ tasks: - ../bundle/schema/jsonschema.json generates: - databricks/bundles/** + - docs/index.rst + - docs/databricks.bundles.*.rst cmds: - | find databricks/bundles -type d -mindepth 1 -maxdepth 1 \ diff --git a/python/codegen/codegen/doc_index.rst.tmpl b/python/codegen/codegen/doc_index.rst.tmpl new file mode 100644 index 00000000000..e7453f3920a --- /dev/null +++ b/python/codegen/codegen/doc_index.rst.tmpl @@ -0,0 +1,12 @@ +databricks-bundles +-------------------------------- + +`databricks-bundles` package implements Python support for Declarative Automation Bundles. + +See `What is Python support for Declarative Automation Bundles? (TBD) <#>`_. + + +.. toctree:: + :maxdepth: 7 + +${toctree} diff --git a/python/codegen/codegen/doc_page.rst.tmpl b/python/codegen/codegen/doc_page.rst.tmpl new file mode 100644 index 00000000000..3278ef09c30 --- /dev/null +++ b/python/codegen/codegen/doc_page.rst.tmpl @@ -0,0 +1,11 @@ +${title} +=============================== + +.. currentmodule:: ${module} + +**Package:** ``${module}`` + +Classes +--------------- + +.. automodule:: ${module} diff --git a/python/codegen/codegen/generated_docs.py b/python/codegen/codegen/generated_docs.py new file mode 100644 index 00000000000..3ae866a09d3 --- /dev/null +++ b/python/codegen/codegen/generated_docs.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Generate the Sphinx .rst pages from the resource module dirs under +databricks/bundles/, so the documented resource list can never drift from the +generated code. Driven by scanning the output tree, not RESOURCE_NAMESPACE. + +The doc structure lives in the *.rst.tmpl templates so it can be reviewed +separately from this code: doc_page.rst.tmpl (one page per resource) and +doc_index.rst.tmpl (the index prose header + generated toctree).""" + +from pathlib import Path +from string import Template + +import codegen.packages as packages + +# .title() mangles acronyms; override those namespaces. +_TITLE_OVERRIDES = { + "sql_warehouses": "SQL Warehouses", + "mcp_services": "MCP Services", +} + + +def _load_template(name: str) -> Template: + return Template((Path(__file__).parent / name).read_text()) + + +_PAGE_TEMPLATE = _load_template("doc_page.rst.tmpl") +_INDEX_TEMPLATE = _load_template("doc_index.rst.tmpl") + + +def _title(namespace: str) -> str: + return _TITLE_OVERRIDES.get(namespace, namespace.replace("_", " ").title()) + + +def write_docs(output: str): + docs = Path(output) / "docs" + bundles = Path(output) / "databricks" / "bundles" + + # core is hand-written; every other package dir gets a generated page. + namespaces = sorted( + p.name + for p in bundles.iterdir() + if p.name != "core" and (p / "__init__.py").exists() + ) + + # Drop stale pages so a removed resource loses its page; keep core.rst. + for rst in docs.glob("databricks.bundles.*.rst"): + if rst.name != "databricks.bundles.core.rst": + rst.unlink() + + for namespace in namespaces: + module = packages.get_root_package(namespace) + page = _PAGE_TEMPLATE.substitute(title=_title(namespace), module=module) + (docs / f"databricks.bundles.{namespace}.rst").write_text(page) + + entries = ["databricks.bundles.core"] + [ + packages.get_root_package(ns) for ns in namespaces + ] + toctree = "\n".join(f" {entry}" for entry in entries) + (docs / "index.rst").write_text(_INDEX_TEMPLATE.substitute(toctree=toctree)) + + print(f"Writing {len(namespaces) + 1} doc pages into {docs}") diff --git a/python/codegen/codegen/jsonschema_patch.py b/python/codegen/codegen/jsonschema_patch.py index 81267393637..ae27d16a5a0 100644 --- a/python/codegen/codegen/jsonschema_patch.py +++ b/python/codegen/codegen/jsonschema_patch.py @@ -18,6 +18,35 @@ "jobs.SparkJarTask": ["main_class_name"], } +# Burn-down list of upstream API descriptions that aren't valid reStructuredText +# and break the Sphinx docs build. Each entry is a temporary override until the +# proto comment is fixed upstream; remove it once the fix lands (the no-op guard +# in override_descriptions flags entries that upstream has already fixed). +# +# sql.SpotInstancePolicy: the upstream comment is a hard-wrapped ASCII grid table +# that docutils rejects as malformed. Rewritten as a list-table. +# See sqlgateway/scheduler/api/proto/endpoint_common.proto. +DESCRIPTIONS: dict[str, str] = { + "sql.SpotInstancePolicy": ( + "EndpointSpotInstancePolicy configures whether the endpoint should use spot instances.\n" + "\n" + "The breakdown of how the EndpointSpotInstancePolicy converts to per cloud configurations is:\n" + "\n" + ".. list-table::\n" + " :header-rows: 1\n" + "\n" + " * - Cloud\n" + " - COST_OPTIMIZED\n" + " - RELIABILITY_OPTIMIZED\n" + " * - AWS\n" + " - On Demand Driver with Spot Executors\n" + " - On Demand Driver and Executors\n" + " * - AZURE\n" + " - On Demand Driver and Executors\n" + " - On Demand Driver and Executors\n" + ), +} + def add_extra_required_fields(schemas: dict[str, Schema]): output = {} @@ -41,6 +70,25 @@ def add_extra_required_fields(schemas: dict[str, Schema]): return output +def override_descriptions(schemas: dict[str, Schema]): + if missing := DESCRIPTIONS.keys() - schemas.keys(): + raise ValueError(f"Cannot override description for unknown schemas: {missing}") + + output = {} + for name, schema in schemas.items(): + if override := DESCRIPTIONS.get(name): + if schema.description == override: + raise ValueError( + f"Description override for {name} is a no-op; the upstream " + "description was fixed, so remove the override" + ) + output[name] = replace(schema, description=override) + else: + output[name] = schema + + return output + + def remove_unsupported_fields(schemas: dict[str, Schema]): output = {} diff --git a/python/codegen/codegen/main.py b/python/codegen/codegen/main.py index e9ae2047b1e..3dea5144de9 100644 --- a/python/codegen/codegen/main.py +++ b/python/codegen/codegen/main.py @@ -6,6 +6,7 @@ import codegen.aliases_patch as aliases_patch import codegen.generated_dataclass as generated_dataclass import codegen.generated_dataclass_patch as generated_dataclass_patch +import codegen.generated_docs as generated_docs import codegen.generated_enum as generated_enum import codegen.generated_imports as generated_imports import codegen.generated_test_cases as generated_test_cases @@ -24,6 +25,7 @@ def main(output: str): schemas = openapi.get_schemas() schemas = openapi_patch.add_extra_required_fields(schemas) schemas = openapi_patch.remove_unsupported_fields(schemas) + schemas = openapi_patch.override_descriptions(schemas) schemas = _transitively_mark_deprecated_and_private( packages.RESOURCE_TYPES, schemas @@ -56,6 +58,10 @@ def main(output: str): # Generate the per-resource ResourceTestCase data driving test_resources.py. generated_test_cases.write_test_cases(output, schemas) + # Generate the Sphinx .rst pages so the documented resource list stays in + # sync with the generated modules. + generated_docs.write_docs(output) + def _transitively_mark_deprecated_and_private( roots: list[str], diff --git a/python/databricks/bundles/sql_warehouses/_models/spot_instance_policy.py b/python/databricks/bundles/sql_warehouses/_models/spot_instance_policy.py index d44908ae895..232731b57ff 100644 --- a/python/databricks/bundles/sql_warehouses/_models/spot_instance_policy.py +++ b/python/databricks/bundles/sql_warehouses/_models/spot_instance_policy.py @@ -6,19 +6,23 @@ class SpotInstancePolicy(Enum): """ - EndpointSpotInstancePolicy configures whether the endpoint should use spot - instances. - - The breakdown of how the EndpointSpotInstancePolicy converts to per cloud - configurations is: - - +-------+--------------------------------------+--------------------------------+ - | Cloud | COST_OPTIMIZED | RELIABILITY_OPTIMIZED | - +-------+--------------------------------------+--------------------------------+ - | AWS | On Demand Driver with Spot Executors | On Demand Driver and - Executors | | AZURE | On Demand Driver and Executors | On Demand Driver - and Executors | - +-------+--------------------------------------+--------------------------------+ + EndpointSpotInstancePolicy configures whether the endpoint should use spot instances. + + The breakdown of how the EndpointSpotInstancePolicy converts to per cloud configurations is: + + .. list-table:: + :header-rows: 1 + + * - Cloud + - COST_OPTIMIZED + - RELIABILITY_OPTIMIZED + * - AWS + - On Demand Driver with Spot Executors + - On Demand Driver and Executors + * - AZURE + - On Demand Driver and Executors + - On Demand Driver and Executors + """ POLICY_UNSPECIFIED = "POLICY_UNSPECIFIED" diff --git a/python/docs/databricks.bundles.alerts.rst b/python/docs/databricks.bundles.alerts.rst new file mode 100644 index 00000000000..08a621c813d --- /dev/null +++ b/python/docs/databricks.bundles.alerts.rst @@ -0,0 +1,11 @@ +Alerts +=============================== + +.. currentmodule:: databricks.bundles.alerts + +**Package:** ``databricks.bundles.alerts`` + +Classes +--------------- + +.. automodule:: databricks.bundles.alerts diff --git a/python/docs/databricks.bundles.apps.rst b/python/docs/databricks.bundles.apps.rst new file mode 100644 index 00000000000..a1ccd11ac58 --- /dev/null +++ b/python/docs/databricks.bundles.apps.rst @@ -0,0 +1,11 @@ +Apps +=============================== + +.. currentmodule:: databricks.bundles.apps + +**Package:** ``databricks.bundles.apps`` + +Classes +--------------- + +.. automodule:: databricks.bundles.apps diff --git a/python/docs/databricks.bundles.catalogs.rst b/python/docs/databricks.bundles.catalogs.rst new file mode 100644 index 00000000000..bbd9054723d --- /dev/null +++ b/python/docs/databricks.bundles.catalogs.rst @@ -0,0 +1,11 @@ +Catalogs +=============================== + +.. currentmodule:: databricks.bundles.catalogs + +**Package:** ``databricks.bundles.catalogs`` + +Classes +--------------- + +.. automodule:: databricks.bundles.catalogs diff --git a/python/docs/databricks.bundles.cluster_policies.rst b/python/docs/databricks.bundles.cluster_policies.rst new file mode 100644 index 00000000000..a82cbec3e36 --- /dev/null +++ b/python/docs/databricks.bundles.cluster_policies.rst @@ -0,0 +1,11 @@ +Cluster Policies +=============================== + +.. currentmodule:: databricks.bundles.cluster_policies + +**Package:** ``databricks.bundles.cluster_policies`` + +Classes +--------------- + +.. automodule:: databricks.bundles.cluster_policies diff --git a/python/docs/databricks.bundles.clusters.rst b/python/docs/databricks.bundles.clusters.rst new file mode 100644 index 00000000000..0babd19d3aa --- /dev/null +++ b/python/docs/databricks.bundles.clusters.rst @@ -0,0 +1,11 @@ +Clusters +=============================== + +.. currentmodule:: databricks.bundles.clusters + +**Package:** ``databricks.bundles.clusters`` + +Classes +--------------- + +.. automodule:: databricks.bundles.clusters diff --git a/python/docs/databricks.bundles.dashboards.rst b/python/docs/databricks.bundles.dashboards.rst new file mode 100644 index 00000000000..70f7d1539af --- /dev/null +++ b/python/docs/databricks.bundles.dashboards.rst @@ -0,0 +1,11 @@ +Dashboards +=============================== + +.. currentmodule:: databricks.bundles.dashboards + +**Package:** ``databricks.bundles.dashboards`` + +Classes +--------------- + +.. automodule:: databricks.bundles.dashboards diff --git a/python/docs/databricks.bundles.database_catalogs.rst b/python/docs/databricks.bundles.database_catalogs.rst new file mode 100644 index 00000000000..005aae777cf --- /dev/null +++ b/python/docs/databricks.bundles.database_catalogs.rst @@ -0,0 +1,11 @@ +Database Catalogs +=============================== + +.. currentmodule:: databricks.bundles.database_catalogs + +**Package:** ``databricks.bundles.database_catalogs`` + +Classes +--------------- + +.. automodule:: databricks.bundles.database_catalogs diff --git a/python/docs/databricks.bundles.database_instances.rst b/python/docs/databricks.bundles.database_instances.rst new file mode 100644 index 00000000000..eea7a14d4b3 --- /dev/null +++ b/python/docs/databricks.bundles.database_instances.rst @@ -0,0 +1,11 @@ +Database Instances +=============================== + +.. currentmodule:: databricks.bundles.database_instances + +**Package:** ``databricks.bundles.database_instances`` + +Classes +--------------- + +.. automodule:: databricks.bundles.database_instances diff --git a/python/docs/databricks.bundles.experiments.rst b/python/docs/databricks.bundles.experiments.rst new file mode 100644 index 00000000000..b2ed4d21ef0 --- /dev/null +++ b/python/docs/databricks.bundles.experiments.rst @@ -0,0 +1,11 @@ +Experiments +=============================== + +.. currentmodule:: databricks.bundles.experiments + +**Package:** ``databricks.bundles.experiments`` + +Classes +--------------- + +.. automodule:: databricks.bundles.experiments diff --git a/python/docs/databricks.bundles.external_locations.rst b/python/docs/databricks.bundles.external_locations.rst new file mode 100644 index 00000000000..0044ce58e66 --- /dev/null +++ b/python/docs/databricks.bundles.external_locations.rst @@ -0,0 +1,11 @@ +External Locations +=============================== + +.. currentmodule:: databricks.bundles.external_locations + +**Package:** ``databricks.bundles.external_locations`` + +Classes +--------------- + +.. automodule:: databricks.bundles.external_locations diff --git a/python/docs/databricks.bundles.genie_spaces.rst b/python/docs/databricks.bundles.genie_spaces.rst new file mode 100644 index 00000000000..0e7b8e13b77 --- /dev/null +++ b/python/docs/databricks.bundles.genie_spaces.rst @@ -0,0 +1,11 @@ +Genie Spaces +=============================== + +.. currentmodule:: databricks.bundles.genie_spaces + +**Package:** ``databricks.bundles.genie_spaces`` + +Classes +--------------- + +.. automodule:: databricks.bundles.genie_spaces diff --git a/python/docs/databricks.bundles.instance_pools.rst b/python/docs/databricks.bundles.instance_pools.rst new file mode 100644 index 00000000000..57fc6543340 --- /dev/null +++ b/python/docs/databricks.bundles.instance_pools.rst @@ -0,0 +1,11 @@ +Instance Pools +=============================== + +.. currentmodule:: databricks.bundles.instance_pools + +**Package:** ``databricks.bundles.instance_pools`` + +Classes +--------------- + +.. automodule:: databricks.bundles.instance_pools diff --git a/python/docs/databricks.bundles.job_runs.rst b/python/docs/databricks.bundles.job_runs.rst new file mode 100644 index 00000000000..1cd97700c53 --- /dev/null +++ b/python/docs/databricks.bundles.job_runs.rst @@ -0,0 +1,11 @@ +Job Runs +=============================== + +.. currentmodule:: databricks.bundles.job_runs + +**Package:** ``databricks.bundles.job_runs`` + +Classes +--------------- + +.. automodule:: databricks.bundles.job_runs diff --git a/python/docs/databricks.bundles.mcp_services.rst b/python/docs/databricks.bundles.mcp_services.rst new file mode 100644 index 00000000000..d13fd3f7956 --- /dev/null +++ b/python/docs/databricks.bundles.mcp_services.rst @@ -0,0 +1,11 @@ +MCP Services +=============================== + +.. currentmodule:: databricks.bundles.mcp_services + +**Package:** ``databricks.bundles.mcp_services`` + +Classes +--------------- + +.. automodule:: databricks.bundles.mcp_services diff --git a/python/docs/databricks.bundles.model_provider_services.rst b/python/docs/databricks.bundles.model_provider_services.rst new file mode 100644 index 00000000000..657522bacc8 --- /dev/null +++ b/python/docs/databricks.bundles.model_provider_services.rst @@ -0,0 +1,11 @@ +Model Provider Services +=============================== + +.. currentmodule:: databricks.bundles.model_provider_services + +**Package:** ``databricks.bundles.model_provider_services`` + +Classes +--------------- + +.. automodule:: databricks.bundles.model_provider_services diff --git a/python/docs/databricks.bundles.model_services.rst b/python/docs/databricks.bundles.model_services.rst new file mode 100644 index 00000000000..38d4ebdff1a --- /dev/null +++ b/python/docs/databricks.bundles.model_services.rst @@ -0,0 +1,11 @@ +Model Services +=============================== + +.. currentmodule:: databricks.bundles.model_services + +**Package:** ``databricks.bundles.model_services`` + +Classes +--------------- + +.. automodule:: databricks.bundles.model_services diff --git a/python/docs/databricks.bundles.model_serving_endpoints.rst b/python/docs/databricks.bundles.model_serving_endpoints.rst new file mode 100644 index 00000000000..842dc7e722c --- /dev/null +++ b/python/docs/databricks.bundles.model_serving_endpoints.rst @@ -0,0 +1,11 @@ +Model Serving Endpoints +=============================== + +.. currentmodule:: databricks.bundles.model_serving_endpoints + +**Package:** ``databricks.bundles.model_serving_endpoints`` + +Classes +--------------- + +.. automodule:: databricks.bundles.model_serving_endpoints diff --git a/python/docs/databricks.bundles.models.rst b/python/docs/databricks.bundles.models.rst new file mode 100644 index 00000000000..d0c71bc68ba --- /dev/null +++ b/python/docs/databricks.bundles.models.rst @@ -0,0 +1,11 @@ +Models +=============================== + +.. currentmodule:: databricks.bundles.models + +**Package:** ``databricks.bundles.models`` + +Classes +--------------- + +.. automodule:: databricks.bundles.models diff --git a/python/docs/databricks.bundles.postgres_snapshot_schedules.rst b/python/docs/databricks.bundles.postgres_snapshot_schedules.rst new file mode 100644 index 00000000000..20108a34625 --- /dev/null +++ b/python/docs/databricks.bundles.postgres_snapshot_schedules.rst @@ -0,0 +1,11 @@ +Postgres Snapshot Schedules +=============================== + +.. currentmodule:: databricks.bundles.postgres_snapshot_schedules + +**Package:** ``databricks.bundles.postgres_snapshot_schedules`` + +Classes +--------------- + +.. automodule:: databricks.bundles.postgres_snapshot_schedules diff --git a/python/docs/databricks.bundles.quality_monitors.rst b/python/docs/databricks.bundles.quality_monitors.rst new file mode 100644 index 00000000000..069916c1f1a --- /dev/null +++ b/python/docs/databricks.bundles.quality_monitors.rst @@ -0,0 +1,11 @@ +Quality Monitors +=============================== + +.. currentmodule:: databricks.bundles.quality_monitors + +**Package:** ``databricks.bundles.quality_monitors`` + +Classes +--------------- + +.. automodule:: databricks.bundles.quality_monitors diff --git a/python/docs/databricks.bundles.registered_models.rst b/python/docs/databricks.bundles.registered_models.rst new file mode 100644 index 00000000000..8389050f4cd --- /dev/null +++ b/python/docs/databricks.bundles.registered_models.rst @@ -0,0 +1,11 @@ +Registered Models +=============================== + +.. currentmodule:: databricks.bundles.registered_models + +**Package:** ``databricks.bundles.registered_models`` + +Classes +--------------- + +.. automodule:: databricks.bundles.registered_models diff --git a/python/docs/databricks.bundles.secret_scopes.rst b/python/docs/databricks.bundles.secret_scopes.rst new file mode 100644 index 00000000000..088b6902283 --- /dev/null +++ b/python/docs/databricks.bundles.secret_scopes.rst @@ -0,0 +1,11 @@ +Secret Scopes +=============================== + +.. currentmodule:: databricks.bundles.secret_scopes + +**Package:** ``databricks.bundles.secret_scopes`` + +Classes +--------------- + +.. automodule:: databricks.bundles.secret_scopes diff --git a/python/docs/databricks.bundles.secrets.rst b/python/docs/databricks.bundles.secrets.rst new file mode 100644 index 00000000000..64ff4ad70b3 --- /dev/null +++ b/python/docs/databricks.bundles.secrets.rst @@ -0,0 +1,11 @@ +Secrets +=============================== + +.. currentmodule:: databricks.bundles.secrets + +**Package:** ``databricks.bundles.secrets`` + +Classes +--------------- + +.. automodule:: databricks.bundles.secrets diff --git a/python/docs/databricks.bundles.sql_warehouses.rst b/python/docs/databricks.bundles.sql_warehouses.rst new file mode 100644 index 00000000000..774ec65cf3f --- /dev/null +++ b/python/docs/databricks.bundles.sql_warehouses.rst @@ -0,0 +1,11 @@ +SQL Warehouses +=============================== + +.. currentmodule:: databricks.bundles.sql_warehouses + +**Package:** ``databricks.bundles.sql_warehouses`` + +Classes +--------------- + +.. automodule:: databricks.bundles.sql_warehouses diff --git a/python/docs/databricks.bundles.synced_database_tables.rst b/python/docs/databricks.bundles.synced_database_tables.rst new file mode 100644 index 00000000000..5d63776b076 --- /dev/null +++ b/python/docs/databricks.bundles.synced_database_tables.rst @@ -0,0 +1,11 @@ +Synced Database Tables +=============================== + +.. currentmodule:: databricks.bundles.synced_database_tables + +**Package:** ``databricks.bundles.synced_database_tables`` + +Classes +--------------- + +.. automodule:: databricks.bundles.synced_database_tables diff --git a/python/docs/databricks.bundles.vector_search_endpoints.rst b/python/docs/databricks.bundles.vector_search_endpoints.rst new file mode 100644 index 00000000000..48a6f792783 --- /dev/null +++ b/python/docs/databricks.bundles.vector_search_endpoints.rst @@ -0,0 +1,11 @@ +Vector Search Endpoints +=============================== + +.. currentmodule:: databricks.bundles.vector_search_endpoints + +**Package:** ``databricks.bundles.vector_search_endpoints`` + +Classes +--------------- + +.. automodule:: databricks.bundles.vector_search_endpoints diff --git a/python/docs/databricks.bundles.vector_search_indexes.rst b/python/docs/databricks.bundles.vector_search_indexes.rst new file mode 100644 index 00000000000..0766cb57517 --- /dev/null +++ b/python/docs/databricks.bundles.vector_search_indexes.rst @@ -0,0 +1,11 @@ +Vector Search Indexes +=============================== + +.. currentmodule:: databricks.bundles.vector_search_indexes + +**Package:** ``databricks.bundles.vector_search_indexes`` + +Classes +--------------- + +.. automodule:: databricks.bundles.vector_search_indexes diff --git a/python/docs/index.rst b/python/docs/index.rst index 96eada6fdd4..1672685fab8 100644 --- a/python/docs/index.rst +++ b/python/docs/index.rst @@ -10,7 +10,34 @@ See `What is Python support for Declarative Automation Bundles? (TBD) <#>`_. :maxdepth: 7 databricks.bundles.core + databricks.bundles.alerts + databricks.bundles.apps + databricks.bundles.catalogs + databricks.bundles.cluster_policies + databricks.bundles.clusters + databricks.bundles.dashboards + databricks.bundles.database_catalogs + databricks.bundles.database_instances + databricks.bundles.experiments + databricks.bundles.external_locations + databricks.bundles.genie_spaces + databricks.bundles.instance_pools + databricks.bundles.job_runs databricks.bundles.jobs + databricks.bundles.mcp_services + databricks.bundles.model_provider_services + databricks.bundles.model_services + databricks.bundles.model_serving_endpoints + databricks.bundles.models databricks.bundles.pipelines + databricks.bundles.postgres_snapshot_schedules + databricks.bundles.quality_monitors + databricks.bundles.registered_models databricks.bundles.schemas + databricks.bundles.secret_scopes + databricks.bundles.secrets + databricks.bundles.sql_warehouses + databricks.bundles.synced_database_tables + databricks.bundles.vector_search_endpoints + databricks.bundles.vector_search_indexes databricks.bundles.volumes