From 9eba2304d12f499f42855137cb5ee418a1b0799a Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 28 Aug 2026 23:18:51 +0200 Subject: [PATCH 1/2] wip: preserve upward bundle integration before PR split --- BUILD | 1 + backlog.md | 209 ++++++++ bundle_needs_conf.py.tpl | 23 + bzl/bundle_rules.bzl | 56 ++- docs.bzl | 317 +++++++++++- docs/concepts/hierarchical_bundle_needs.md | 473 ++++++++++++++++++ docs/concepts/index.rst | 1 + docs/how-to/index.rst | 2 + docs/how-to/upward_bundles.rst | 217 ++++++++ docs/reference/bazel_macros.rst | 36 +- scripts_bazel/docs_targets.sh | 133 +++++ scripts_bazel/tests/BUILD | 14 + .../tests/merge_needs_json_app_tests.py | 182 +++++++ src/extensions/docs/BUILD | 1 + src/extensions/score_metamodel/__init__.py | 9 + src/extensions/score_metamodel/docs/BUILD | 1 + .../score_metamodel/external_needs.py | 58 ++- .../tests/test_external_needs.py | 70 +++ .../score_sphinx_bundle/__init__.py | 6 + src/tests/docs_bzl/README.md | 1 + src/tests/docs_bzl/cross_module_fixture/BUILD | 1 + .../cross_module_fixture/docs/conf.py | 18 + .../cross_module_fixture/docs/metamodel.yaml | 28 ++ .../scenarios/data_files_runfiles/BUILD | 1 + .../scenarios/invalid_bundle_placements/BUILD | 15 + .../docs_bzl/scenarios/upward_bundles/BUILD | 68 +++ .../upward_bundles/component/index.rst | 37 ++ .../scenarios/upward_bundles/module/conf.py | 22 + .../scenarios/upward_bundles/module/index.rst | 43 ++ .../upward_bundles/platform/index.rst | 35 ++ .../upward_bundles/subcomponent/index.rst | 36 ++ src/tests/docs_bzl/test_basic_docs.py | 7 + .../test_cross_module_compatibility.py | 64 +++ src/tests/docs_bzl/test_external_needs.py | 4 +- .../test_invalid_bundle_placements.py | 1 + src/tests/docs_bzl/test_upward_bundles.py | 159 ++++++ 36 files changed, 2307 insertions(+), 42 deletions(-) create mode 100644 backlog.md create mode 100644 bundle_needs_conf.py.tpl create mode 100644 docs/concepts/hierarchical_bundle_needs.md create mode 100644 docs/how-to/upward_bundles.rst create mode 100755 scripts_bazel/docs_targets.sh create mode 100644 scripts_bazel/tests/merge_needs_json_app_tests.py create mode 100644 src/tests/docs_bzl/cross_module_fixture/docs/conf.py create mode 100644 src/tests/docs_bzl/cross_module_fixture/docs/metamodel.yaml create mode 100644 src/tests/docs_bzl/scenarios/upward_bundles/BUILD create mode 100644 src/tests/docs_bzl/scenarios/upward_bundles/component/index.rst create mode 100644 src/tests/docs_bzl/scenarios/upward_bundles/module/conf.py create mode 100644 src/tests/docs_bzl/scenarios/upward_bundles/module/index.rst create mode 100644 src/tests/docs_bzl/scenarios/upward_bundles/platform/index.rst create mode 100644 src/tests/docs_bzl/scenarios/upward_bundles/subcomponent/index.rst create mode 100644 src/tests/docs_bzl/test_upward_bundles.py diff --git a/BUILD b/BUILD index d28698186..146cf71b6 100644 --- a/BUILD +++ b/BUILD @@ -15,6 +15,7 @@ load("//:docs.bzl", "docs") package(default_visibility = ["//visibility:public"]) exports_files([ + "bundle_needs_conf.py.tpl", "default_conf.py.tpl", "pyproject.toml", ]) diff --git a/backlog.md b/backlog.md new file mode 100644 index 000000000..707f1c09a --- /dev/null +++ b/backlog.md @@ -0,0 +1,209 @@ + + +# Backlog: upstream/main integration and `upward_bundles` + +Last updated: 2026-08-28 + +## Current state + +The local branch is based on `upstream/main` at `298732e4`. The previous merge +commit was intentionally removed with a soft reset; the resulting changes are +currently staged and deliberately have no commit yet. + +The staged delta is currently 37 files with approximately 2,580 additions and +42 deletions. It is a working integration result, not yet a reviewable PR. + +Verification completed before the split: + +```text +bazel test //... 23/23 passed +.venv_docs/bin/python -m pytest -q src/tests/docs_bzl 28 passed +``` + +The full pre-commit hook also passed after the final formatting fix. + +## Topics contained in the staged delta + +### 1. Bundle-local Needs and bundle ownership + +`docs_bundle` now distinguishes a bundle's own sources and data from nested +bundle content. It creates a local Needs export for the bundle's own sources, +with generated configuration, source-code links, selected metamodel, and the +correct `entry_doc` as Sphinx master document. + +Relevant implementation files: + +- `bzl/bundle_rules.bzl` +- `docs.bzl` +- `bundle_needs_conf.py.tpl` +- root `BUILD` + +### 2. Hierarchical Needs via `upward_bundles` + +The bundle provider carries direct and transitive upward dependencies. A +bundle can export its own Needs together with explicitly declared ancestors; +source-less hierarchy groups, multiple parents, diamond-shaped graphs, and +cyclic declarations are covered. + +The top-level `docs()` macro receives the same hierarchy and exposes the +stable public `docs_source_bundle` and +`docs_source_bundle_needs_upward` targets. Downward `bundles` composition and +upward Needs interfaces remain separate concepts. + +### 3. Cross-module external Needs + +External Needs loading understands named `*_needs_upward` exports and their +namespaced runfiles paths. The consumer documentation project's +`project_url` is registered early enough and is used as the canonical base +URL for imported Needs. + +This is covered by the cross-module fixture and the compatibility integration +tests. + +### 4. Documentation and examples + +The staged documentation explains the ownership model, hierarchy contract, +build graph, generated targets, and usage of `upward_bundles`: + +- `docs/concepts/hierarchical_bundle_needs.md` +- `docs/how-to/upward_bundles.rst` +- `docs/reference/bazel_macros.rst` + +The Mermaid-fence handling in `score_sphinx_bundle` supports the diagrams in +the Markdown documentation. It should remain only if the documentation PR +needs it and has a rendering regression test. + +### 5. Merger application tests + +`merge_needs_json.py` is already present in `upstream/main`. The staged +application-level test exercises merging actual Sphinx-generated inventories. +The staged `merge_needs_json_unit_tests.py`, however, is byte-for-byte +identical to the existing `merge_needs_json_tests.py` and must be removed +before splitting the PRs. + +### 6. Developer tooling + +`scripts_bazel/docs_targets.sh` lists documentation, bundle-local, and upward +Needs targets. It is useful convenience tooling but is not required by the +hierarchy implementation. + +## Proposed PR plan + +The PRs should be stacked on `upstream/main` in this order. Test fixtures and +tests belong with the feature they verify; the documentation is intentionally +separate from the implementation review. + +### PR 1 — `docs_bundle`: local Needs and hierarchical `upward_bundles` + +Approximate size: 900–1,100 LOC including focused tests. + +Include: + +- direct bundle ownership metadata and `bundle_own_files`; +- bundle-local `*_needs_local` and merged `*_needs_upward` exports; +- `upward_bundles` on `docs_bundle` and `docs()`; +- source-less hierarchy groups and transitive parent propagation; +- stable source-bundle aliases; +- cycle, multiple-parent, diamond, and local-versus-upward tests; +- Sphinx sandbox-safe metamodel inputs and `entry_doc` handling. + +This is the main implementation PR. Keep the core hierarchy documentation +out of this PR except for concise API comments and test descriptions. + +### PR 2 — Cross-module external Needs and URL semantics + +Approximate size: 200–250 LOC including tests. + +Include: + +- parsing and resolving named `*_needs_upward` exports; +- consumer-owned canonical `project_url` handling; +- cross-module fixture and compatibility tests. + +This PR depends on PR 1 because it consumes the generated upward export +targets. + +### PR 3 — Documentation for hierarchical bundles + +Approximate size: 700 LOC, mostly documentation. + +Include: + +- the hierarchy concept document; +- the `upward_bundles` How-to; +- the Bazel macro reference updates; +- Mermaid-fence support if required by the new Markdown diagrams, together + with a small rendering test. + +### PR 4 — Sphinx application-level tests for `merge_needs_json` (optional) + +Approximate size: 190 LOC. + +Keep the application-level test if the stronger Sphinx-generated-inventory +contract is useful. Do not include the duplicate unit-test file or its BUILD +target; the existing upstream unit test already covers that behavior. + +### PR 5 — `docs_targets.sh` developer utility (optional) + +Approximate size: 133 LOC. + +Submit this independently if the target-discovery command is useful to users. +It has no production dependency on the hierarchy implementation. + +## Cleanup before creating the PRs + +Remove or split out the following from the current staged integration: + +- the duplicate `merge_needs_json_unit_tests.py` and its BUILD entry; +- `scripts_bazel/docs_targets.sh` unless it is wanted as separate tooling; +- Mermaid-fence support unless it is required by the documentation PR and is + covered by a regression test. + +Keep this file as the planning record, but do not mix the planning backlog into +the implementation PRs unless repository policy requires it. + +## Open follow-ups + +These points are not blockers for the first hierarchy PR and should become +separate issues if they remain relevant after review: + +- add a black-box test proving that a bundle-local export cannot see a sibling + or descendant without an explicit upward dependency; +- define a negative test for missing or malformed upward export files; +- define custom-metamodel behavior when an imported Need type is absent from + the consumer schema; +- specify duplicate-ID and URL behavior when a bundle is mounted downward and + consumed upward at the same time; +- review generated target visibility and public naming across repositories; +- add release notes once the API is approved. + +## Architecture contract to preserve + +```text +child local Needs export + │ + └── depends upward on explicit parent exports + +parent public composed bundle + └── mounts child documentation downward +``` + +Bundle-local exports contain no consumer mount paths. Mount paths, backlinks, +global checks, metrics, and final rendered output are resolved only by the +composed host documentation build. + +`bundles` describes downward documentation composition. `upward_bundles` +describes the explicit Needs interface available to a bundle's own validation. diff --git a/bundle_needs_conf.py.tpl b/bundle_needs_conf.py.tpl new file mode 100644 index 000000000..0b2c71122 --- /dev/null +++ b/bundle_needs_conf.py.tpl @@ -0,0 +1,23 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# Default Sphinx configuration for a ``docs_bundle`` local Needs export. + +project = {PROJECT} +version = "0.0.0" +master_doc = {ENTRY_DOC} + +extensions = ["score_sphinx_bundle"] + +# A bundle-local export intentionally omits mounted descendants. The composed +# host Needs build resolves those outgoing links later. +suppress_warnings = ["needs.link_outgoing"] diff --git a/bzl/bundle_rules.bzl b/bzl/bundle_rules.bzl index 7145741f4..df621ea8e 100644 --- a/bzl/bundle_rules.bzl +++ b/bzl/bundle_rules.bzl @@ -70,6 +70,13 @@ DocsBundleInfo = provider( # these are resolved at this bundle's mount (for example, a generated # index.rst). "data": "Bundle-owned generated/supporting files resolved at the bundle's mount.", + "own_sources": "Depset of documentation sources owned directly by this bundle.", + "own_sourcelinks": "Depset of source-code-link JSON files declared directly by this bundle.", + "own_data": "Depset of non-source-tree files declared directly by this bundle.", + "entry_doc": "Bundle-relative entry document for the bundle's own sources.", + "metamodel": "The metamodel file selected for this bundle's Needs processing.", + "direct_upward_bundles": "The bundle targets declared directly in upward_bundles.", + "upward_bundles": "Depset containing this bundle's declared upward dependencies and their transitive upward closure.", }, ) @@ -238,6 +245,13 @@ def _docs_bundle_impl(ctx): own_source_files = [] own_external_runfiles = [] own_data = depset(direct = ctx.files.data) + upward_bundles = depset( + direct = ctx.attr.upward_bundles, + transitive = [ + upward_bundle[DocsBundleInfo].upward_bundles + for upward_bundle in ctx.attr.upward_bundles + ], + ) if ctx.files.srcs: runtime_path = _bundle_runtime_path(ctx) @@ -317,6 +331,13 @@ def _docs_bundle_impl(ctx): sourcelinks = sourcelinks, external_runfiles = external_runfiles, data = all_data, + own_sources = depset(ctx.files.srcs), + own_sourcelinks = depset(ctx.files.sourcelinks), + own_data = own_data, + entry_doc = ctx.attr.entry_doc, + metamodel = ctx.file.metamodel, + direct_upward_bundles = ctx.attr.upward_bundles, + upward_bundles = upward_bundles, ), ] @@ -327,7 +348,12 @@ _docs_bundle = rule( "sourcelinks": attr.label_list(allow_files = True), "strip_prefix": attr.string(default = ""), "entry_doc": attr.string(default = "index"), + "metamodel": attr.label( + allow_single_file = True, + default = Label("@score_docs_as_code//src/extensions/score_metamodel:metamodel_yaml"), + ), "bundles": attr.label_list(providers = [DocsBundleInfo]), + "upward_bundles": attr.label_list(providers = [DocsBundleInfo]), "bundle_mount_ats": attr.string_list(), "bundle_attach_tos": attr.string_list(), "data": attr.label_list(allow_files = True), @@ -335,15 +361,18 @@ _docs_bundle = rule( doc = "Internal rule that carries bundle files and their documentation-tree locations.", ) -def create_bundle(name, bundles, srcs = [], sourcelinks = [], strip_prefix = "", entry_doc = "index", data = [], visibility = None, **kwargs): +def create_bundle(name, bundles, srcs = [], sourcelinks = [], strip_prefix = "", entry_doc = "index", metamodel = None, upward_bundles = [], data = [], visibility = None, **kwargs): """Create a reusable documentation bundle from files and child declarations.""" parsed_bundles = [_parse_bundle_declaration(declaration) for declaration in bundles] + metamodel = metamodel or Label("@score_docs_as_code//src/extensions/score_metamodel:metamodel_yaml") _docs_bundle( name = name, srcs = srcs, sourcelinks = sourcelinks, strip_prefix = strip_prefix, entry_doc = entry_doc, + metamodel = metamodel, + upward_bundles = upward_bundles, bundles = [bundle.bundle for bundle in parsed_bundles], bundle_mount_ats = [bundle.mount_at for bundle in parsed_bundles], bundle_attach_tos = [bundle.attach_to for bundle in parsed_bundles], @@ -484,3 +513,28 @@ def generate_code_target_sourcelinks(name, code_targets, visibility = None): visibility = visibility, ) return ":" + name + +def _bundle_own_files_impl(ctx): + """Expose only the files owned directly by a documentation bundle.""" + bundle = ctx.attr.bundle[DocsBundleInfo] + return [DefaultInfo(files = depset(transitive = [ + bundle.own_sources, + bundle.own_data, + ]))] + +_bundle_own_files = rule( + implementation = _bundle_own_files_impl, + attrs = { + "bundle": attr.label(providers = [DocsBundleInfo]), + }, + doc = "Exposes a bundle's own documentation files without nested bundles.", +) + +def bundle_own_files(name, bundle, visibility = None): + """Create a target containing only a bundle's directly owned files.""" + _bundle_own_files( + name = name, + bundle = bundle, + visibility = visibility, + ) + return ":" + name diff --git a/docs.bzl b/docs.bzl index 56eaa297a..a8a680542 100644 --- a/docs.bzl +++ b/docs.bzl @@ -52,9 +52,10 @@ load( "@score_docs_as_code//:bzl/bundle_rules.bzl", "bundle_source_files", "create_bundle", + "bundle_own_files", + "merge_bundle_sourcelinks", "external_docs_runfiles", "generate_code_target_sourcelinks", - "merge_bundle_sourcelinks", ) load( "@score_docs_as_code//:bzl/mount_rules.bzl", @@ -83,6 +84,7 @@ def _generated_conf_impl(ctx): "{PROJECT}": repr(ctx.attr.project), "{PROJECT_URL}": repr(ctx.attr.project_url), "{REQUIRED_IN_ID}": repr([ctx.attr.required_in_id]) if ctx.attr.required_in_id else "[]", + "{ENTRY_DOC}": repr(ctx.attr.entry_doc), }, ) return [DefaultInfo(files = depset([output]))] @@ -93,6 +95,7 @@ _generated_conf = rule( "project": attr.string(mandatory = True), "project_url": attr.string(mandatory = True), "required_in_id": attr.string(mandatory = True), + "entry_doc": attr.string(default = "index"), "output_path": attr.string(mandatory = True), "template": attr.label( allow_single_file = True, @@ -101,7 +104,50 @@ _generated_conf = rule( }, ) -def docs_bundle(name, source_dir = None, data = [], entry_doc = "index", bundles = [], scan_code = [], code_targets = [], visibility = None, **kwargs): +def _create_metamodel_tool(name, metamodel): + """Expose a metamodel file as an executable Sphinx tool input. + + rules_sphinxdocs intentionally restricts ``tools`` to executable targets, + while ``$(location ...)`` in a Sphinx action still needs the metamodel's + sandboxed path. Copying the file through an executable genrule satisfies + both constraints without changing the file contents. + """ + native.genrule( + name = name, + srcs = [metamodel], + outs = [name + ".yaml"], + cmd = "cp $(location " + str(metamodel) + ") $@", + executable = True, + visibility = ["//visibility:private"], + ) + return ":" + name + +def _bundle_upward_needs_label(bundle): + """Return the generated upward-needs label for a docs_bundle label.""" + bundle_string = str(bundle) + if bundle_string.startswith(":"): + bundle_string = "//" + native.package_name() + bundle_string + elif not bundle_string.startswith("//") and not bundle_string.startswith("@"): + bundle_string = "//" + native.package_name() + ":" + bundle_string + return Label(bundle_string + "_needs_upward") + +def _external_needs_label(label): + """Format a label for score_metamodel's external-needs parser.""" + label_string = str(label) + if label_string.startswith("@@//"): + return label_string[2:] + if label_string.startswith("@@"): + canonical = label_string[2:] + repository, separator, package_and_target = canonical.partition("//") + # Bazel 8's canonical Bzlmod spelling uses ``repo+``. The runfiles + # layout and the external-needs parser use the user-facing module name + # and add the ``+`` themselves where needed. + if repository.endswith("+"): + repository = repository[:-1] + return "@" + repository + separator + package_and_target + return label_string + +def docs_bundle(name, source_dir = None, data = [], entry_doc = "index", metamodel = None, bundles = [], upward_bundles = [], scan_code = [], code_targets = [], deps = [], bundle_conf = None, visibility = None, **kwargs): """A docs bundle, optionally composed of others. Args: @@ -118,17 +164,29 @@ def docs_bundle(name, source_dir = None, data = [], entry_doc = "index", bundles only bundle data travels with a mounted bundle. entry_doc: bundle-relative docname attached when this bundle is mounted. Defaults to `index`. + metamodel: optional metamodel label used for this bundle's Needs + processing. If omitted, the built-in SCORE metamodel is used. bundles: nested bundles to compose, each a dict { "bundle": , "mount_at": , "attach_to": }. + upward_bundles: docs_bundle targets in the documentation hierarchy above + this bundle. These are explicit Bazel dependencies used by the + bundle-local Needs export; both direct declarations and the complete + transitive upward closure are propagated through DocsBundleInfo. A + source bundle imports the direct parent's merged export; an aggregator + may use the relationship as a named hierarchy group. scan_code: Deprecated. Explicit source files or filegroups to scan for source-code links. Use `code_targets` for implementation targets. code_targets: Implementation targets or filegroups to scan for source-code links. Implementation target source files and their dependencies are collected recursively; filegroups expand to their files. + deps: Additional Python dependencies for the bundle-local Needs export. + bundle_conf: Optional Sphinx conf.py label to reuse for the bundle-local + Needs export. This is used by docs() for its generated host + configuration. visibility: Target visibility. **kwargs: Additional attributes forwarded to the underlying rule. """ @@ -153,18 +211,160 @@ def docs_bundle(name, source_dir = None, data = [], entry_doc = "index", bundles strip_prefix = join_path(pkg, source_dir) if source_dir != None else "" # The helper validates child declarations and creates the internal target. + selected_metamodel = metamodel or Label("@score_docs_as_code//src/extensions/score_metamodel:metamodel_yaml") create_bundle( name = name, srcs = srcs, sourcelinks = sourcelinks, strip_prefix = strip_prefix, entry_doc = entry_doc, + metamodel = selected_metamodel, bundles = bundles, + upward_bundles = upward_bundles, data = data, visibility = visibility, **kwargs ) + upward_needs = [_bundle_upward_needs_label(bundle) for bundle in upward_bundles] + + # A bundle owns Needs only through its own documentation sources. Keep + # nested bundles out of this source set, while still making the generated + # target useful for source-less aggregators that only carry no local docs. + if srcs: + metamodel_tool = None + if metamodel: + metamodel_tool = _create_metamodel_tool( + name + "_needs_metamodel", + selected_metamodel, + ) + + own_files = bundle_own_files( + name = name + "_own_files", + bundle = ":" + name, + visibility = visibility, + ) + + config_file_path = join_path(source_dir, "conf.py") + if bundle_conf: + needs_config = bundle_conf + elif native.glob([config_file_path], allow_empty = True): + needs_config = ":" + config_file_path + else: + needs_config = ":" + name + "_needs_conf" + _generated_conf( + name = name + "_needs_conf", + project = name, + project_url = "", + required_in_id = "", + entry_doc = entry_doc, + output_path = config_file_path, + template = Label("@score_docs_as_code//:bundle_needs_conf.py.tpl"), + ) + + bundle_deps = deps + _missing_requirements(deps) + [ + Label("//src:plantuml_for_python"), + Label("//src/extensions/score_sphinx_bundle:score_sphinx_bundle"), + ] + bundle_data = [own_files] + upward_needs + if metamodel: + bundle_data.append(selected_metamodel) + + if len(sourcelinks) == 0: + needs_sourcelinks = ":" + name + "_needs_sourcelinks_json" + _sourcelinks_json( + name = name + "_needs_sourcelinks_json", + srcs = [], + ) + elif len(sourcelinks) == 1: + needs_sourcelinks = sourcelinks[0] + else: + needs_sourcelinks_name = name + "_needs_sourcelinks_json" + merge_bundle_sourcelinks( + name = needs_sourcelinks_name, + bundle = ":" + name, + visibility = visibility, + ) + needs_sourcelinks = ":" + needs_sourcelinks_name + bundle_data.append(needs_sourcelinks) + + sphinx_build_binary( + name = name + "_needs_sphinx_build", + data = bundle_data, + deps = bundle_deps, + visibility = visibility, + ) + + needs_local = ":" + name + "_needs_local" + needs_extra_opts = [ + "--keep-going", + "-T", + "--define=external_needs_source=" + str([ + _external_needs_label(label) + for label in upward_needs + ]), + ] + if metamodel: + needs_extra_opts.append( + "--define=score_metamodel_yaml=$(location " + metamodel_tool + ")" + ) + needs_extra_opts.append( + "--define=score_sourcelinks_json=$(location " + str(needs_sourcelinks) + ")" + ) + + needs_tools = list(upward_needs) + if metamodel: + needs_tools.append(metamodel_tool) + needs_tools.append(needs_sourcelinks) + + sphinx_docs( + name = name + "_needs_local", + srcs = [own_files], + config = needs_config, + # sphinxdocs removes this string literally from short_path. Keep + # the separator so a config at ``source_dir/conf.py`` becomes + # ``conf.py`` rather than ``/conf.py``. + strip_prefix = strip_prefix + "/" if strip_prefix else "", + extra_opts = needs_extra_opts, + formats = ["needs"], + sphinx = ":" + name + "_needs_sphinx_build", + tools = needs_tools, + visibility = visibility, + allow_persistent_workers = False, + ) + + needs_upward = name + "_needs_upward" + merge_inputs = [needs_local] + upward_needs + merge_command = "$(location //scripts_bazel:merge_needs_json) --output $@ $(location " + needs_local + ")/needs.json" + for input_label in upward_needs: + merge_command += " $(location " + str(input_label) + ")" + native.genrule( + name = needs_upward, + srcs = merge_inputs, + outs = [needs_upward + "/needs.json"], + cmd = merge_command, + tools = [Label("//scripts_bazel:merge_needs_json")], + visibility = visibility, + ) + elif upward_needs: + # A source-less hierarchy group owns no Needs of its own, but it can + # still expose the merged export of its declared ancestors. This lets + # a child depend on a named hierarchy group without knowing how the + # group's parent chain is assembled. + needs_upward = name + "_needs_upward" + merge_command = "$(location //scripts_bazel:merge_needs_json) --output $@ $(location " + str(upward_needs[0]) + ")" + for input_label in upward_needs[1:]: + merge_command += " $(location " + str(input_label) + ")" + native.genrule( + name = needs_upward, + srcs = upward_needs, + outs = [needs_upward + "/needs.json"], + cmd = merge_command, + tools = [Label("//scripts_bazel:merge_needs_json")], + visibility = visibility, + tags = ["manual"], + ) + def _missing_requirements(deps): """Add Python hub dependencies if they are missing.""" found = [] @@ -212,7 +412,9 @@ def docs( test_sources = [], known_good = None, metamodel = None, - bundles = []): + bundles = [], + upward_bundles = [], + ): """Creates all targets related to documentation. By using this function, you'll get any and all updates for documentation targets in one place. @@ -238,7 +440,11 @@ def docs( When empty (default), all testcases found in `bazel-testlogs` will be used. known_good: Optional label to a "known good" JSON file for source links. metamodel: Optional label to a metamodel.yaml file. When set, the extension loads this - file instead of the default metamodel shipped with score_metamodel. + file instead of the default metamodel shipped with score_metamodel. The same + metamodel is bound to the host source bundle created by this macro. + upward_bundles: docs_bundle targets in the documentation hierarchy above the + host's own source bundle. Their Needs are available while + processing the host sources and in the final composed Needs build. bundles: List of placement dicts describing documentation bundles to overlay into this documentation's source tree. Each entry is a dict { @@ -257,6 +463,14 @@ def docs( """ # HINT: keep documentation sync docs/reference/bazel_macros.rst + upward_needs = [_bundle_upward_needs_label(bundle) for bundle in upward_bundles] + all_external_needs = external_needs + upward_needs + all_external_needs_sources = [ + _external_needs_label(label) + for label in all_external_needs + ] + data_sources = [str(label) for label in data] + config_file_path = join_path(source_dir, "conf.py") sphinx_config = ":" + config_file_path config_is_generated = len(native.glob([config_file_path], allow_empty = True)) == 0 @@ -331,10 +545,62 @@ def docs( incremental_src = Label("//src:incremental.py") + # Keep the host's own source bundle separate from the composed bundle. + # Child bundle Needs exports use this source target as their upward + # interface; the public bundle remains the complete source tree consumed + # by Sphinx and docs_check. The public docs_source_bundle alias below is + # the stable cross-package name for this source-level target. + docs_bundle( + name = "_docs_source_bundle", + source_dir = source_dir, + entry_doc = "index", + metamodel = metamodel, + bundle_conf = sphinx_config, + scan_code = scan_code, + code_targets = code_targets, + upward_bundles = upward_bundles, + visibility = ["//visibility:public"], + ) + + # ``_docs_source_bundle`` was the original generated label and is kept for + # compatibility with existing consumers. Expose a stable public name for + # the host's own-source hierarchy anchor so cross-module users do not need + # to depend on a private-looking implementation label. + native.alias( + name = "docs_source_bundle", + actual = ":_docs_source_bundle", + visibility = ["//visibility:public"], + ) + if glob_doc_sources(source_dir): + # A source-bearing host always gets this generated export from + # docs_bundle(). Use a real output target rather than an alias: the + # external-needs loader resolves named bundle exports from their + # runfiles path, and Bazel aliases retain the implementation target's + # output directory. + native.genrule( + name = "docs_source_bundle_needs_upward", + srcs = [":_docs_source_bundle_needs_upward"], + outs = ["docs_source_bundle_needs_upward/needs.json"], + cmd = "cp $(location :_docs_source_bundle_needs_upward) $@", + visibility = ["//visibility:public"], + ) + + composed_bundles = [{ + "bundle": ":_docs_source_bundle", + "mount_at": "", + }] + bundles + + docs_bundle( + name = "docs_bundle", + metamodel = metamodel, + bundles = composed_bundles, + visibility = ["//visibility:public"], + ) + sphinx_build_binary( name = "sphinx_build", visibility = ["//visibility:private"], - data = data + external_needs + metamodel_label + [":docs_bundle"], + data = data + all_external_needs + metamodel_label + [":docs_bundle"], deps = deps, tags = ["manual"] ) @@ -342,20 +608,19 @@ def docs( known_good_label = [known_good] if known_good else [] # The public bundle carries both the complete source tree and the - # transitive source-code links of every nested bundle. - docs_bundle( - name = "docs_bundle", - source_dir = source_dir, - entry_doc = "index", - bundles = bundles, - scan_code = scan_code, - code_targets = code_targets, - visibility = ["//visibility:public"], - tags = ["manual"] - ) + # transitive source-code links of every nested bundle. The own-source + # bundle is composed above so it can also be used as a private hierarchy + # interface without depending on mounted children. + # transitive source-code links of every nested bundle. Sphinx itself only + # receives the host's direct sources; mounted children are staged by + # score_mounts so their Needs are not discovered a second time. sphinx_sources = bundle_source_files( name = "_docs_sphinx_sources", - bundle = ":docs_bundle", + # ``docs_bundle`` is the complete composed aggregator in the + # hierarchy-aware implementation. Its direct sources are therefore + # the host sources exposed by ``_docs_source_bundle``; mounted child + # sources must remain supplied through score_mounts. + bundle = ":_docs_source_bundle", visibility = ["//visibility:private"], ) merge_bundle_sourcelinks( @@ -374,7 +639,7 @@ def docs( # the complete bundle in runfiles would duplicate those sources. External # bundles do need runfiles, so keep only those sources. docs_data = ( - data + external_needs + metamodel_label + + data + all_external_needs + metamodel_label + [":sourcelinks_json", ":_external_docs_runfiles"] + mounts_manifest_label ) @@ -387,8 +652,8 @@ def docs( "SOURCE_DIRECTORY": source_dir, "PACKAGE_DIR": native.package_name(), "TEST_SOURCES": str(test_sources), - "DATA": str(data), - "EXTERNAL_NEEDS_FILES": str(external_needs), + "DATA": str(data_sources), + "EXTERNAL_NEEDS_FILES": str(all_external_needs_sources), # `bazel run` starts from a runfiles tree, so this logical path is # resolved by score_mounts through ``RUNFILES_DIR``. "MOUNTS_MANIFEST": "$(rlocationpath :_mounts_manifest)" if bundles else "", @@ -466,6 +731,12 @@ def docs( package_collisions = "warning", ) + metamodel_tool = [] + if metamodel: + metamodel_tool = [ + _create_metamodel_tool("_docs_metamodel", metamodel), + ] + sphinx_docs( name = "needs_json", # Nested bundle sources are mounted by score_mounts. Passing the @@ -480,17 +751,17 @@ def docs( "-T", # show more details in case of errors "--jobs", "auto", - "--define=external_needs_source=" + str(data + external_needs), + "--define=external_needs_source=" + str(data_sources + all_external_needs_sources), "--define=score_sourcelinks_json=$(location :sourcelinks_json)", "--define=score_source_code_linker_plain_links=1", ] + ( # ``sphinx_docs`` is a sandboxed build action, so it needs the # action-input path rather than the runfiles-relative spelling. ["--define=mounts_manifest=$(location :_mounts_manifest)"] if bundles else [] - ) + (["--define=score_metamodel_yaml=$(location " + str(metamodel) + ")"] if metamodel else []), + ) + (["--define=score_metamodel_yaml=$(location " + metamodel_tool[0] + ")"] if metamodel else []), formats = ["needs"], sphinx = ":sphinx_build", - tools = data + external_needs + metamodel_label + [":sourcelinks_json", ":docs_bundle"] + mounts_manifest_label, + tools = data + all_external_needs + metamodel_tool + [":sourcelinks_json", ":docs_bundle"] + mounts_manifest_label, visibility = ["//visibility:public"], # Persistent workers cause stale symlinks after dependency version # changes, corrupting the Bazel cache. diff --git a/docs/concepts/hierarchical_bundle_needs.md b/docs/concepts/hierarchical_bundle_needs.md new file mode 100644 index 000000000..28066fad4 --- /dev/null +++ b/docs/concepts/hierarchical_bundle_needs.md @@ -0,0 +1,473 @@ + + +# Hierarchically linked bundle Needs exports + +This document defines how reusable documentation bundles own Needs, how those +Needs may refer to a higher architectural level, and how the individual +bundle results are combined into the public `needs_json` result. + +The design separates two relationships that are easy to conflate: + +* `bundles` describes which documentation sources are composed into a source + tree and where they are mounted. +* `upward_bundles` describes which Needs from a higher level are available to + a bundle while its own Needs are processed. + +The first relationship is about documentation composition. The second is +about Needs ownership and validation. Neither relationship is inferred from +the other. + +## Problem and rationale + +The existing `needs_json` build runs Sphinx-Needs over the complete composed +documentation tree. A mounted bundle is therefore processed together with the +host documentation and all other mounted bundles. This has two important +consequences: + +1. A change in one bundle invalidates the complete Needs action. +2. A Need is evaluated only after all source trees have been rebased to their + final mount locations. + +The second property is useful for the final documentation result, but it is +too coarse for reusable bundles. A bundle must be able to validate and cache +the Needs it owns without taking ownership of all sources mounted below it. +At the same time, a lower-level Need must be able to refer to a Need owned by +an explicitly selected higher-level bundle. + +The solution is to process a bundle's own sources separately and to exchange +Needs through an explicit upward interface. A final composition step still +builds the complete graph, so the public result retains global backlinks, +constraints, metrics, and final document paths. + +## Bundle model + +A `docs_bundle` is a named documentation component. It can contain its own +source directory, data, source-code links, a metamodel, and nested bundles. +The presence of nested sources does not change who owns their Needs: each +source-bearing bundle owns the Needs created from its own sources. + +There are three relevant kinds of bundle: + +* A **source bundle** has a `source_dir` and owns the Needs extracted from + those sources. +* A **composed bundle** has nested `bundles`. It exposes one combined source + tree to its consumer, but it does not take ownership of the child bundles' + Needs. +* A **hierarchy group** may have no sources at all and may still declare + `upward_bundles`. It is a named Bazel node for reusing a hierarchy + relationship; because it has no sources, it owns no local Needs. + +The same `docs_bundle` target can be both a source bundle and a composed +bundle. These terms describe its two independent roles, not different rule +types. + +### Composition and hierarchy are different graphs + +| Concern | Bazel attribute | Direction | Meaning | +| --- | --- | --- | --- | +| Documentation composition | `bundles` | Downwards | Include a child source tree and rebase it below `mount_at`. | +| Needs hierarchy | `upward_bundles` | Upwards | Make Needs owned by an ancestor available as an input. | + +`bundles` controls files, document placement, attachments, data, and source +links. `upward_bundles` controls the permitted Needs interface. For example, +mounting `engine_bundle` below `powertrain` does not by itself allow engine +Needs to link to powertrain Needs. That permission is granted by the explicit +`upward_bundles = [":powertrain_bundle"]` declaration. + +This distinction is important for Bazel analysis. A source tree can be +composed downwards while a Needs export depends upwards, without making the +bundle responsible for all of the sources visible in the final tree. + +## User-facing declaration + +A normal reusable bundle does not need a hierarchy declaration. A bundle that +has Needs links to a higher level names the allowed ancestors explicitly: + +```starlark +load("//:docs.bzl", "docs", "docs_bundle") + +# A normal reusable bundle. It owns its own Needs and has no upward interface. +docs_bundle( + name = "guidance_bundle", + source_dir = "guidance/docs", +) + +# Needs owned by this bundle are available to lower-level bundles that name it. +docs_bundle( + name = "powertrain_bundle", + source_dir = "powertrain/docs", +) + +# This bundle may link its Needs to powertrain_bundle or to one of its +# declared ancestors. The dependency is explicit in the Bazel graph. +docs_bundle( + name = "engine_bundle", + source_dir = "engine/docs", + upward_bundles = [":powertrain_bundle"], +) + +# A source-less named hierarchy group is also valid. It owns no local Needs. +docs_bundle( + name = "vehicle_architecture_group", + upward_bundles = [":powertrain_bundle"], +) + +docs( + source_dir = "docs", + bundles = [ + { + "bundle": ":guidance_bundle", + "mount_at": "guidance", + }, + { + "bundle": ":engine_bundle", + "mount_at": "powertrain/engine", + }, + ], +) +``` + +An upward declaration grants access to the direct target and its transitive +upward closure. The bundle's ``*_needs_local`` export contains only Needs +created from the bundle's own sources. Its ``*_needs_upward`` export is the +reusable interface view: it merges that local export with the exports of the +declared ancestors. The imported ancestor records remain owned by their +original source bundles; aggregating them does not transfer ownership to the +declaring bundle. + +The dependency is deliberately declared in Bazel rather than inferred from +Need links. This makes the allowed interface visible during analysis, +provides the inputs needed for caching, and prevents accidental links to +siblings, descendants, or unrelated bundles. + +## The two bundle views created by `docs()` + +`docs()` has two different consumers: hierarchy-aware Needs processing and the +normal composed documentation build. They need different views of the host +documentation. + +Internally, the macro therefore creates: + +```text +host sources --------------------> :docs_source_bundle + | + | composed with mounted children + v +host source bundle + child bundles -> :docs_bundle + | + +-> Sphinx documentation build + +-> docs_check + +-> public needs_json +``` + +`:docs_source_bundle` contains only the host's own sources, data, source-code +links, and metamodel. It is the source-level node of the host in the bundle +graph and is available as an upward hierarchy anchor. The public +`:docs_bundle` is an aggregator containing that source node and all bundles +passed to `docs(bundles = ...)`. + +This separation prevents a dependency cycle. A lower-level bundle can depend +on the own-source export of an ancestor, while the ancestor's composed source +tree can include the lower-level bundle: + +```text +child Needs export ----depends upwards----> parent own-source export +parent composed tree <----mounts downwards---- child source bundle +``` + +The documentation tree can consequently express parent/child relationships +without making the Bazel dependency graph cyclic. `:docs_source_bundle` is the +stable public source-anchor label for cross-package consumers. The historical +`:_docs_source_bundle` label remains available for compatibility. Users +configure the host through `docs()` and reusable components through +`docs_bundle()`. + +## Needs ownership and bundle exports + +The unit of Needs ownership is a source bundle, not a mount point. Mounting the +same bundle at another location changes the document path at which its Needs +are rendered, but it does not create a second owner or a second Need ID. + +A bundle-local ``*_needs_local`` export contains: + +* Needs created from the bundle's own documentation sources; +* the bundle-relative `docname` and source location; +* the bundle's ownership/origin information; +* Need parts, link conditions, and other data needed for later validation; +* source-link information belonging to the bundle; and +* the selected metamodel context. + +The corresponding ``*_needs_upward`` export additionally contains the Needs +from the direct ``upward_bundles`` exports. Those exports already contain their +own upward closure, so the result exposes the complete declared interface to a +downstream consumer. Imported records remain distinguishable as ancestor +records and are not owned by the child bundle. + +Need IDs are the stable cross-bundle identity. A bundle-relative `docname` is +not stable across mounts: the final path is determined by the consumer's +`mount_at`. This is why links between bundles use Need identity while the +top-level composition is responsible for final document paths and generated +URLs. + +### Direct and transitive upward dependencies + +The direct declaration and its closure have distinct meanings: + +* `direct_upward_bundles` is the set written in the current rule's + `upward_bundles` attribute. +* `upward_bundles` is the transitive set consisting of those direct targets + and the upward closure exported by each target. + +The closure is the complete validation interface of the bundle. If `engine` +declares `powertrain`, and `powertrain` declares `vehicle`, engine can use +Needs from both `powertrain` and `vehicle`. It does not need to repeat the +transitive declaration, and a consumer can still inspect which edge was +declared directly. + +## Processing model + +## Execution traces + +The two entry points below deliberately show different Bazel paths. The public +module ``needs_json`` target consumes the complete composed source tree. The +bundle-local export targets are a separate, cacheable path used when a bundle +is consumed through ``upward_bundles``. + +### Trigger from the module's public ``needs_json`` target + +The module-level build traverses the downward ``bundles`` composition and runs +one final Sphinx-Needs build. It does not invoke the intermediate +``*_needs_local`` or ``*_needs_upward`` targets. If ``code_targets`` are +configured, each source bundle first produces its source-link cache; the +public ``sourcelinks_json`` target then merges those caches before Sphinx +consumes them. + +```mermaid +sequenceDiagram + autonumber + participant User as "bazel build //path/to/module:needs_json" + participant Needs as "//path/to/module:needs_json" + participant Bundle as "//path/to/module:docs_bundle" + participant Host as "//path/to/module:docs_source_bundle" + participant Component as "//path/to/module:component" + participant Subcomponent as "//path/to/module:subcomponent" + participant Mounts as "//path/to/module:_mounts_manifest" + participant SourceCache as "//path/to/module:sourcelinks_json" + participant LinkGen as "//scripts_bazel:generate_sourcelinks" + participant LinkMerge as "//scripts_bazel:merge_sourcelinks" + participant Sphinx as "//path/to/module:sphinx_build" + participant Linker as "score_source_code_linker extension" + + User->>Needs: request final Needs export + Needs->>Bundle: consume composed source tree + Bundle->>Host: include host sources + Bundle->>Component: include component at mount_at + Bundle->>Subcomponent: include subcomponent at mount_at + Bundle->>Mounts: provide mounted source manifest + + Component->>LinkGen: run component_code_targets_sourcelinks_json + LinkGen-->>Component: component source-link JSON + Subcomponent->>LinkGen: run subcomponent_code_targets_sourcelinks_json + LinkGen-->>Subcomponent: subcomponent source-link JSON + Host->>LinkGen: run _docs_source_bundle_code_targets_sourcelinks_json + LinkGen-->>Host: module source-link JSON + Needs->>SourceCache: collect propagated source-link caches + SourceCache->>LinkMerge: run merge_sourcelinks + LinkMerge-->>SourceCache: merged sourcelinks_json + Needs->>Sphinx: run composed Sphinx-Needs build + Sphinx->>Linker: read sourcelinks_json + Linker-->>Sphinx: attach source_code_link and testlink + Sphinx-->>Needs: write final needs.json and metrics + Needs-->>User: completed public result +``` + +The diagram shows the ``code_targets`` path: ``component_code_targets_sourcelinks_json`` +and its siblings invoke ``//scripts_bazel:generate_sourcelinks``. Without +``code_targets``, the corresponding bundle-local Needs action uses an empty +``*_needs_sourcelinks_json`` target instead. + +### Trigger from the subcomponent's upward export + +When the requested target is ``subcomponent_needs_upward``, Bazel follows the +Needs hierarchy in the opposite semantic direction: the subcomponent builds +its own local export and requests the component export; the component in turn +requests the module's private own-source export. Each level runs its own +Sphinx-Needs action and then ``merge_needs_json`` adds the local result to the +already merged parent export. + +```mermaid +sequenceDiagram + autonumber + participant User as "bazel build //path/to/module:subcomponent_needs_upward" + participant SubUp as ":subcomponent_needs_upward" + participant SubLocal as ":subcomponent_needs_local" + participant SubFiles as ":subcomponent_own_files" + participant CompUp as ":component_needs_upward" + participant CompLocal as ":component_needs_local" + participant CompFiles as ":component_own_files" + participant HostUp as ":docs_source_bundle_needs_upward" + participant HostLocal as ":_docs_source_bundle_needs_local" + participant HostFiles as ":_docs_source_bundle_own_files" + participant Sphinx as "*_needs_sphinx_build" + participant LinkGen as "//scripts_bazel:generate_sourcelinks" + participant NeedsMerge as "//scripts_bazel:merge_needs_json" + + User->>SubUp: request reusable subcomponent export + SubUp->>SubLocal: build :subcomponent_needs_local + SubLocal->>SubFiles: read :subcomponent_own_files + SubLocal->>LinkGen: run :subcomponent_code_targets_sourcelinks_json + LinkGen-->>SubLocal: source-link JSON + SubLocal->>Sphinx: run :subcomponent_needs_sphinx_build + Sphinx-->>SubLocal: local subcomponent needs.json + SubUp->>CompUp: request :component_needs_upward + CompUp->>CompLocal: build :component_needs_local + CompLocal->>CompFiles: read :component_own_files + CompLocal->>LinkGen: run :component_code_targets_sourcelinks_json + LinkGen-->>CompLocal: source-link JSON + CompLocal->>Sphinx: run :component_needs_sphinx_build + Sphinx-->>CompLocal: local component needs.json + CompUp->>HostUp: request :docs_source_bundle_needs_upward + HostUp->>HostLocal: build :_docs_source_bundle_needs_local + HostLocal->>HostFiles: read :_docs_source_bundle_own_files + HostLocal->>LinkGen: run :_docs_source_bundle_code_targets_sourcelinks_json + LinkGen-->>HostLocal: module source-link JSON + HostLocal->>Sphinx: run :_docs_source_bundle_needs_sphinx_build + Sphinx-->>HostLocal: local module needs.json + HostUp->>NeedsMerge: merge local module export + NeedsMerge-->>HostUp: :docs_source_bundle_needs_upward/needs.json + CompUp->>NeedsMerge: merge component local + module export + NeedsMerge-->>CompUp: :component_needs_upward/needs.json + SubUp->>NeedsMerge: merge subcomponent local + component export + NeedsMerge-->>SubUp: :subcomponent_needs_upward/needs.json + SubUp-->>User: completed hierarchical export +``` + +The sequence is shown in dependency order for readability. Bazel may build +independent prerequisites in parallel: the subcomponent's local export does +not depend on the component export. The component export itself waits for the +module's own-source export because ``upward_bundles`` is an explicit +dependency. The source-code linker path is shown at every level; if a level +does not configure ``code_targets``, its empty ``*_needs_sourcelinks_json`` +target replaces the generated source-link cache. + +### Bundle-local processing + +The bundle-local action receives the source bundle's own sources and its +metamodel. It also receives the exports of the bundle's upward closure. The +Sphinx-Needs run can therefore resolve: + +* Needs defined in the current bundle; and +* Needs defined by a declared upward bundle or one of its ancestors. + +The bundle-local action does not see sibling or descendant exports through +this interface. A link outside the local ownership set and the declared +upward closure is invalid because the bundle has no declared dependency on +the target's ownership context. + +The result is an internal artifact. Its records remain relative to the +bundle's own source tree, because no consumer has assigned a mount path yet. + +### Top-level composition + +The public `needs_json` action consumes the complete source tree exposed by the +composed documentation bundle and applies the placements from `bundles`. It +does not depend on the intermediate ``*_needs_local`` or ``*_needs_upward`` +artifacts. It then performs the operations whose meaning depends on the +complete graph: + +* rebase each bundle-relative `docname` to its final `mount_at` path; +* resolve links and calculate backlinks over all imported Needs; +* evaluate global constraints and dead-link state; +* calculate metrics and traceability gates; and +* write the existing public `needs_json` output. + +This step is the final graph view, not a second ownership layer. It evaluates +the records from all composed source owners in one Sphinx run; it does not +copy a parent's Needs into a child or recreate HTML from an intermediate +export. The ``*_needs_upward`` artifacts are the separate reusable interface +shown in the second sequence diagram. + +Rendering, navigation, `:doc:`/`:ref:` resolution, includes, images, and +other source-tree behavior continue to use the complete composed Sphinx source +tree. `docs_check` remains the document-level integration check for that tree. + +## Metamodel and ownership context + +The metamodel belongs to the bundle that owns the Needs. It is selected on +`docs_bundle(metamodel = ...)`, defaults to the built-in SCORE metamodel, and +is propagated in `DocsBundleInfo`. + +Selecting the metamodel per bundle has two reasons: + +1. A reusable bundle must be analyzable with the schema that defines its own + Need types and fields. +2. The metamodel is an input to the bundle-local action, so changing it must + invalidate that bundle's result rather than an unrelated bundle. + +The final composition retains the origin of each Need separately from its +rendered location. A bundle from another module may be mounted into the local +documentation tree while remaining foreign for ownership-sensitive metrics +and release gates. URL placement and ownership are therefore not represented +by a single boolean such as `is_external`. + +## Bazel representation + +`DocsBundleInfo` carries the metadata needed by both composition and +hierarchy-aware processing: + +* source entries with their eventual mount metadata; +* source-code links, data, and external runfiles; +* the selected metamodel; +* the directly declared upward bundle targets; and +* a deduplicated depset containing the direct targets and their transitive + upward closure. + +The closure is calculated while analyzing a bundle: + +```text +upward closure(current) = direct(current) + + union(upward closure(parent)) +``` + +Using a Bazel depset makes the interface transitive, deduplicated, and +available to downstream rules without requiring each rule to rediscover the +hierarchy. The same provider is present on source bundles, composed bundles, +and source-less hierarchy groups. A composed bundle aggregates source entries +for file placement, but its upward interface remains a separate piece of +metadata. + +## Invariants + +The architecture relies on the following invariants: + +* A Need has exactly one owning source bundle. +* `bundles` does not transfer Need ownership. +* `upward_bundles` is explicit and is the only cross-bundle validation + interface. +* A bundle's ``*_needs_local`` export contains its own Needs; its + ``*_needs_upward`` export may aggregate imported ancestor Needs without + transferring ownership. +* Need IDs remain stable when a bundle is mounted at another path. +* Final document paths, backlinks, and global checks are computed only after + composition has assigned mount locations. +* The Bazel dependency graph remains acyclic even though documentation + composition and Needs references describe opposite semantic directions. +* The public `needs_json` shape and existing consumer commands remain + compatible with the composed result. + +These invariants are the reason the implementation keeps the private host +source bundle separate from the public composed `docs_bundle` and propagates +both direct and transitive upward dependencies. diff --git a/docs/concepts/index.rst b/docs/concepts/index.rst index 62f1db4ed..0e9eb7029 100644 --- a/docs/concepts/index.rst +++ b/docs/concepts/index.rst @@ -24,4 +24,5 @@ Here you find explanations how and why docs-as-code works the way it does. bidirectional_traceability cross_module_compatibility docs_deps + hierarchical_bundle_needs mounts/index diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst index 8f78a58cf..f9621e14d 100644 --- a/docs/how-to/index.rst +++ b/docs/how-to/index.rst @@ -27,6 +27,8 @@ Here you find practical guides on how to use docs-as-code. write_docs other_modules cross_module_compatibility + upward_bundles + mounts/index bundles/index dashboards_and_quality_gates source_to_doc_links diff --git a/docs/how-to/upward_bundles.rst b/docs/how-to/upward_bundles.rst new file mode 100644 index 000000000..a2e581b28 --- /dev/null +++ b/docs/how-to/upward_bundles.rst @@ -0,0 +1,217 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +.. _howto_upward_bundles: + +Hierarchical Needs with ``upward_bundles`` +========================================== + +Use ``upward_bundles`` when a documentation bundle owns Needs at a lower +architectural level and those Needs refer to Needs owned by a higher level. +The declaration is the validation interface of the bundle; it is separate +from the ``bundles`` declarations that decide where documentation is mounted. + +This How-To uses the SCORE metamodel and the following ownership model: + +* the module's ``docs()`` source owns the external-to-the-components + ``feat_req`` and the module architecture Need; +* a component is a ``docs_bundle`` and owns a ``comp`` and a ``comp_req``; +* a subcomponent is another ``docs_bundle`` and owns its own ``comp`` and + ``comp_req``; +* both component requirements derive from the module-level feature + requirement. + +Here, *external* means external to the component bundle's ownership. It does +not necessarily mean a different Bazel module or repository. + +.. contents:: + :local: + :depth: 2 + +Declare the hierarchy +--------------------- + +The module and its component bundles can be declared in one Bazel package: + +.. code-block:: starlark + + load("//:docs.bzl", "docs", "docs_bundle") + + docs_bundle( + name = "component", + source_dir = "component", + metamodel = "//src/extensions/score_metamodel:metamodel_yaml", + # The module's own source bundle is generated by docs(). + upward_bundles = [":docs_source_bundle"], + ) + + docs_bundle( + name = "subcomponent", + source_dir = "subcomponent", + metamodel = "//src/extensions/score_metamodel:metamodel_yaml", + # The module is inherited transitively through component. + upward_bundles = [":component"], + ) + + docs( + source_dir = "module", + project = "Example module", + project_url = "https://example.invalid/module", + metamodel = "//src/extensions/score_metamodel:metamodel_yaml", + bundles = [ + {"bundle": ":component", "mount_at": "component"}, + { + "bundle": ":subcomponent", + "mount_at": "component/subcomponent", + }, + ], + ) + +``upward_bundles`` points from the child to its parent. The parent does not +mount the child through this attribute. Mounting remains a separate, +downward ``bundles`` relationship in ``docs()``. + +The same attribute is available on ``docs()`` when the module's own source +bundle is itself below another documentation bundle: + +.. code-block:: starlark + + docs( + source_dir = "docs", + upward_bundles = ["@platform_docs//:docs_bundle"], + bundles = [ + {"bundle": ":component", "mount_at": "component"}, + ], + ) + +This makes the ancestor export available both to the module's private source +bundle and to the final composed ``needs_json`` build. The generated +``:docs_source_bundle`` remains the upward parent for bundles declared in the +same module. The legacy ``:_docs_source_bundle`` label is still accepted, but +new consumers should use the stable public name. + +The generated ``:docs_source_bundle`` is the module's own source-level bundle +and is available as the host's upward hierarchy anchor. This allows +hierarchical bundles to be declared in another Bazel package. Its reusable +Needs export is available as ``:docs_source_bundle_needs_upward``. +The public ``:docs_bundle`` target is the composed documentation tree and +must not be used as the upward parent when it already mounts the child; that +would introduce a dependency cycle. + +If the module uses a custom ``conf.py`` and its feature requirement lives in +``module/index.rst``, add the module name to ``required_in_id`` so the +three-part SCORE requirement ID remains meaningful after the source directory +is stripped: + +.. code-block:: python + + extensions = ["score_sphinx_bundle"] + version = "0.0.0" + required_in_id = ["module"] + +Write the Needs +--------------- + +The module source can own the feature and module Needs: + +.. code-block:: rst + + .. feat:: Seat heating + :id: feat__seat_heating + :version: 1 + :security: NO + :safety: QM + :status: valid + + .. feat_req:: Seat heating availability + :id: feat_req__module__seat_heating + :version: 1 + :reqtype: Functional + :security: NO + :safety: QM + :status: valid + :valid_from: v1.0 + :satisfied_by: feat__seat_heating + + The seat heating feature is available to the vehicle user. + + .. mod:: Seat heating module + :id: mod__seat_heating_module + :version: 1 + :includes: comp__seat_heating_controller + +The component bundle can refer to the module-level ``feat_req`` while owning +its own requirement and architecture element: + +.. code-block:: rst + + .. comp:: Seat heating controller + :id: comp__seat_heating_controller + :version: 1 + :security: NO + :safety: QM + :status: valid + :belongs_to: feat__seat_heating + :consists_of: comp__seat_heating_sensor + + .. comp_req:: Controller temperature control + :id: comp_req__component__temperature_control + :version: 1 + :reqtype: Functional + :security: NO + :safety: QM + :status: valid + :derived_from: feat_req__module__seat_heating + :satisfied_by: comp__seat_heating_controller + + The controller regulates the requested heating level. + +The subcomponent follows the same SCORE pattern. Its ``derived_from`` link +tests the transitive closure: ``subcomponent`` declares only ``component``, +while ``component`` declares the module source bundle. + +Verify the composed graph +------------------------- + +Build the normal Needs target from the module package: + +.. code-block:: console + + $ bazel build //path/to/module:needs_json + +The resulting graph must contain the module-level ``feat_req``, the component +and subcomponent ``comp_req`` Needs, and their resolved ``derived_from`` links. +The same fixture should also be rendered with ``bazel run //path/to/module:docs`` +to verify the final mounted document paths. + +Bundle-local exports +-------------------- + +For every source-bearing ``docs_bundle`` the macro also creates +``_needs_local`` and ``_needs_upward``. The first parses only the +bundle's own sources. The second is the reusable export: it merges the local +Needs with the direct ``upward_bundles`` exports, whose files already contain +their own upward closure. This makes the target useful as an explicit input to +another bundle while keeping ownership local. + +The public ``needs_json`` target still evaluates the complete composed source +tree. It remains the right target for final backlinks, mounted document paths, +global checks, metrics, and rendered documentation. + +There is no separate ``_needs_downward`` target: downward composition is +owned by the host ``docs()`` target and is exposed as its public ``needs_json`` +target. + +For the separation of composition and Needs ownership, see +:doc:`the architecture concept <../concepts/hierarchical_bundle_needs>`. diff --git a/docs/reference/bazel_macros.rst b/docs/reference/bazel_macros.rst index d6f2ffc43..9a196188c 100644 --- a/docs/reference/bazel_macros.rst +++ b/docs/reference/bazel_macros.rst @@ -164,11 +164,13 @@ site). name = "docs_dir", source_dir = "docs", entry_doc = "index", + metamodel = "//:my_metamodel.yaml", bundles = [], + upward_bundles = [], visibility = ["//visibility:public"], ) -Signature: ``docs_bundle(name, source_dir = None, data = [], entry_doc = "index", bundles = [], scan_code = [], code_targets = [], visibility = None)``. +Signature: ``docs_bundle(name, source_dir = None, data = [], entry_doc = "index", metamodel = None, bundles = [], upward_bundles = [], scan_code = [], code_targets = [], deps = [], bundle_conf = None, visibility = None)``. - ``source_dir`` (string, optional) Directory holding the bundle's own doc sources. It is globbed the same way as @@ -194,6 +196,11 @@ Signature: ``docs_bundle(name, source_dir = None, data = [], entry_doc = "index" ``index``. Every mount attaches this entry to the parent ``index`` toctree by default; a placement's ``attach_to`` may override that host document. +- ``metamodel`` (Bazel label, optional) + Metamodel used when processing Needs owned by this bundle. If omitted, the + built-in SCORE metamodel is selected. The metamodel belongs to the bundle and + is an input to the bundle-local Needs export. + - ``bundles`` (list of composition dicts, optional) Nested bundles to compose into this one, so a bundle can aggregate other bundles transitively. Each entry is a dict: @@ -211,6 +218,29 @@ Signature: ``docs_bundle(name, source_dir = None, data = [], entry_doc = "index" error. See :ref:`howto_mount_external_sources` for a worked example and :ref:`docs_concept_mounts` for the composition and transitivity semantics. +- ``upward_bundles`` (list of labels, optional) + Explicit documentation-hierarchy dependencies above this bundle. A source + bundle may declare one or more ancestor bundles; both the direct declarations + and their transitive closure are propagated through ``DocsBundleInfo`` for the + per-bundle Needs export. A source-less aggregator may also declare them to + provide a named hierarchy group for consumers. + Dependencies are declared explicitly and are not inferred from Need links. + + If ``source_dir`` contains documentation sources, the macro creates + ``_needs_local`` for the bundle's own Needs and + ``_needs_upward`` for an importable JSON export. The latter merges + the bundle's own Needs with the exports of its direct ``upward_bundles``; + those exports already contain their own ancestors. The public file target + therefore exposes the complete declared upward interface without assigning + ownership of ancestor Needs to the child. + +The top-level ``docs()`` macro also accepts ``upward_bundles``. Those labels are +attached to the generated ``:docs_source_bundle`` target, so the module's own +Needs can refer to an ancestor bundle while mounted descendants continue to use +the module source bundle as their upward parent. The older +``:_docs_source_bundle`` label remains available for compatibility, but new +consumers should use the stable ``:docs_source_bundle`` name. + .. note:: A bundle is **placement-free**: its ``mount_at`` and ``attach_to`` are assigned @@ -225,6 +255,10 @@ Signature: ``docs_bundle(name, source_dir = None, data = [], entry_doc = "index" owns one cached scan result; Bazel only regenerates it when its collected source inputs change. +- ``deps`` (list of Bazel labels, optional) + Additional Python dependencies used by the bundle-local Sphinx/Needs + action. + - ``scan_code`` (list of Bazel labels, deprecated) Explicit source files or filegroups to scan. Prefer ``code_targets`` for implementation targets. diff --git a/scripts_bazel/docs_targets.sh b/scripts_bazel/docs_targets.sh new file mode 100755 index 000000000..de0410b9b --- /dev/null +++ b/scripts_bazel/docs_targets.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +set -euo pipefail + +query_scope="${1:-//...}" + +docs_targets="$(bazel query \ + 'attr(generator_function, "^docs$", '"${query_scope}"')' \ + --output=label)" +bundle_targets="$(bazel query \ + 'kind("_docs_bundle rule", attr(generator_function, "^docs_bundle$", '"${query_scope}"'))' \ + --output=label)" +bundle_needs_targets="$(bazel query \ + 'attr(generator_function, "^docs_bundle$", '"${query_scope}"')' \ + --output=label)" + +query_output="$( + { + while IFS= read -r target; do + [[ -n "$target" ]] || continue + case "$target" in + */tests/*|*.run|*.serve|*.venv|*.find_main) + continue + ;; + esac + name="${target##*:}" + case "$name" in + docs|docs_check|docs_link_check|live_preview|ide_support|\ + needs_json|needs_json_file|metrics_json|sourcelinks_json|traceability_gate) + printf 'docs\t%s\n' "$target" + ;; + esac + done <<< "$docs_targets" + + while IFS= read -r target; do + [[ -n "$target" ]] || continue + case "$target" in + */tests/*|*.run|*.serve|*.venv|*.find_main) + continue + ;; + esac + name="${target##*:}" + [[ "$name" != _* ]] && printf 'bundle\t%s\n' "$target" + done <<< "$bundle_targets" + + while IFS= read -r target; do + [[ -n "$target" ]] || continue + case "$target" in + */tests/*|*.run|*.serve|*.venv|*.find_main) + continue + ;; + esac + name="${target##*:}" + case "$name" in + _*_needs_local) + ;; + _*_needs_upward) + ;; + *_needs_local) + printf 'local\t%s\n' "$target" + ;; + *_needs_upward) + printf 'upward\t%s\n' "$target" + ;; + esac + done <<< "$bundle_needs_targets" + } | LC_ALL=C sort -t $'\t' -k2,2 -u +)" + +while IFS=$'\t' read -r role target; do + [[ -n "$target" ]] || continue + + name="${target##*:}" + case "$role" in + bundle) + description="mountable and composable documentation bundle" + ;; + local) + description="bundle-local Needs export (own sources only)" + ;; + upward) + description="bundle Needs export (own sources plus upward dependencies)" + ;; + docs) + case "$name" in + docs) + description="build the rendered documentation" + ;; + docs_check) + description="run the documentation validation checks" + ;; + docs_link_check) + description="check links in the documentation" + ;; + live_preview) + description="run the live documentation preview" + ;; + ide_support) + description="create the documentation development virtualenv" + ;; + needs_json) + description="build the complete composed Needs graph" + ;; + needs_json_file) + description="export the complete composed Needs graph as JSON" + ;; + metrics_json) + description="export traceability metrics from the Needs build" + ;; + sourcelinks_json) + description="export merged source-to-documentation links" + ;; + traceability_gate) + description="enforce configured traceability coverage thresholds" + ;; + esac + ;; + esac + + printf '%-90s %s\n' "$target" "$description" +done <<< "$query_output" diff --git a/scripts_bazel/tests/BUILD b/scripts_bazel/tests/BUILD index 53c64d92e..43a3ea6b9 100644 --- a/scripts_bazel/tests/BUILD +++ b/scripts_bazel/tests/BUILD @@ -33,6 +33,20 @@ score_pytest( pytest_config = "//:pyproject.toml", ) +score_pytest( + name = "merge_needs_json_app_tests", + srcs = ["merge_needs_json_app_tests.py"], + data = [ + "//src/tests/docs_bzl/scenarios/basic_docs:docs_bundle", + "//src/tests/docs_bzl/scenarios/external_needs/producer:docs_bundle", + "//src/tests/docs_bzl/scenarios/nested_bundles:docs_bundle", + ], + deps = [ + "//scripts_bazel:merge_needs_json", + ] + all_requirements, + pytest_config = "//:pyproject.toml", +) + score_pytest( name = "merge_needs_json_tests", srcs = ["merge_needs_json_tests.py"], diff --git a/scripts_bazel/tests/merge_needs_json_app_tests.py b/scripts_bazel/tests/merge_needs_json_app_tests.py new file mode 100644 index 000000000..481fe64e9 --- /dev/null +++ b/scripts_bazel/tests/merge_needs_json_app_tests.py @@ -0,0 +1,182 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""App-level tests for the merge tool using needs.json from Sphinx.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import cast + +import pytest +from sphinx.testing.util import SphinxTestApp + +from scripts_bazel.merge_needs_json import merge_needs_json + +REPO_ROOT = Path(__file__).absolute().parents[2] +SCENARIOS_DIR = REPO_ROOT / "src/tests/docs_bzl/scenarios" + + +def _single_version_needs(needs_json: Path) -> tuple[str, dict[str, object]]: + data = json.loads(needs_json.read_text(encoding="utf-8")) + assert isinstance(data, dict) + raw_versions = cast(object, data["versions"]) + assert isinstance(raw_versions, dict) + versions = cast(dict[str, object], raw_versions) + assert len(versions) == 1 + + version_key, raw_version_object = next(iter(versions.items())) + assert isinstance(raw_version_object, dict) + version_data = cast(dict[str, object], raw_version_object) + raw_needs = version_data.get("needs") + assert isinstance(raw_needs, dict) + return version_key, cast(dict[str, object], raw_needs) + + +def _write_conf( + source_dir: Path, + *, + project: str, + project_url: str, + version: str | None = None, + producer_type: bool = False, +) -> None: + config = [ + f"project = {project!r}", + f"project_url = {project_url!r}", + 'extensions = ["sphinx_needs"]', + "needs_id_regex = r'^[A-Za-z0-9_-]{6,}'", + ] + if version is not None: + config.append(f"version = {version!r}") + if producer_type: + config.append( + "needs_types = [{" + "'directive': 'test_req', " + "'title': 'Test Requirement', " + "'prefix': 'test_req__', " + "'parts': 3, " + "'mandatory_options': {" + "'id': r'^test_req__[0-9a-zA-Z_]*$', " + "'status': r'^(draft|valid)$'}, " + "'optional_options': {'tags': '.*', 'content': '.*', " + "'template': '.*'}, " + "'mandatory_links': {}, " + "'optional_links': {}}]" + ) + + source_dir.joinpath("conf.py").write_text( + "\n".join(config) + "\n", + encoding="utf-8", + ) + + +def _build_needs_json( + source_dir: Path, + build_dir: Path, + monkeypatch: pytest.MonkeyPatch, +) -> Path: + app = SphinxTestApp( + buildername="needs", + srcdir=source_dir, + builddir=build_dir, + freshenv=True, + warningiserror=True, + ) + monkeypatch.chdir(source_dir) + try: + app.build() + finally: + app.cleanup() + + needs_json = build_dir / "needs" / "needs.json" + assert needs_json.is_file(), f"Sphinx did not create {needs_json}" + return needs_json + + +def test_merges_needs_json_generated_by_sphinx( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + basic_source = tmp_path / "basic_docs" + shutil.copytree(SCENARIOS_DIR / "basic_docs" / "docs", basic_source) + _write_conf( + basic_source, + project="Basic Test", + project_url="https://github.com/eclipse-score/docs-as-code", + version="0.0.0", + ) + + producer_source = tmp_path / "producer" + shutil.copytree( + SCENARIOS_DIR / "external_needs" / "producer" / "docs", + producer_source, + ) + _write_conf( + producer_source, + project="External Needs Producer", + project_url="https://example.invalid/external-needs-producer", + version="main", + producer_type=True, + ) + + empty_source = tmp_path / "empty" + shutil.copytree( + SCENARIOS_DIR / "nested_bundles" / "host_docs", + empty_source, + ) + _write_conf( + empty_source, + project="Mount contract fixture", + project_url="https://example.invalid/mount-contract", + ) + + basic_needs_json = _build_needs_json( + basic_source, tmp_path / "basic_out", monkeypatch + ) + producer_needs_json = _build_needs_json( + producer_source, tmp_path / "producer_out", monkeypatch + ) + empty_needs_json = _build_needs_json( + empty_source, tmp_path / "empty_out", monkeypatch + ) + + basic_version, basic_needs = _single_version_needs(basic_needs_json) + producer_version, producer_needs = _single_version_needs(producer_needs_json) + empty_version, empty_needs = _single_version_needs(empty_needs_json) + + assert basic_version == "0.0.0" + assert producer_version == "main" + assert empty_version == "" + assert basic_needs == {} + assert empty_needs == {} + assert set(producer_needs) == {"test_req__producer__demo"} + + merged = merge_needs_json([basic_needs_json, producer_needs_json, empty_needs_json]) + raw_merged_versions = merged["versions"] + assert isinstance(raw_merged_versions, dict) + merged_versions = cast(dict[str, object], raw_merged_versions) + assert set(merged_versions) == {basic_version} + + raw_merged_version = merged_versions[basic_version] + assert isinstance(raw_merged_version, dict) + merged_version = cast(dict[str, object], raw_merged_version) + raw_merged_needs = merged_version["needs"] + assert isinstance(raw_merged_needs, dict) + merged_needs = cast(dict[str, object], raw_merged_needs) + assert set(merged_needs) == set(producer_needs) + assert merged_version["needs_amount"] == len(merged_needs) + + # Top-level metadata remains owned by the first input. + assert merged["project"] == "Basic Test" diff --git a/src/extensions/docs/BUILD b/src/extensions/docs/BUILD index a2ea53b9d..35f3b8e29 100644 --- a/src/extensions/docs/BUILD +++ b/src/extensions/docs/BUILD @@ -16,5 +16,6 @@ load("//:docs.bzl", "docs_bundle") docs_bundle( name = "extensions", source_dir = "", + upward_bundles = ["//:docs_source_bundle"], visibility = ["//visibility:public"], ) diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index 01c788293..bfbf560f5 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -243,6 +243,15 @@ def _clear_needs_defaults(app: Sphinx): def setup(app: Sphinx) -> dict[str, str | bool]: app.add_config_value("external_needs_source", "", rebuild="env") + # Register this before Sphinx reads conf.py. The external-needs exporter + # augments the needs JSON during ``config-inited`` and therefore cannot + # reliably add the value at that point anymore. + app.add_config_value( + "project_url", + "", + rebuild="env", + description="Canonical URL of the documentation project.", + ) app.add_config_value("score_metamodel_yaml", "", rebuild="env") app.add_config_value("required_in_id", [], rebuild="env") config_setdefault(app.config, "needs_id_required", True) diff --git a/src/extensions/score_metamodel/docs/BUILD b/src/extensions/score_metamodel/docs/BUILD index bb7621a9f..44403351c 100644 --- a/src/extensions/score_metamodel/docs/BUILD +++ b/src/extensions/score_metamodel/docs/BUILD @@ -41,5 +41,6 @@ docs_bundle( name = "metamodel", data = [":generate_metamodel_rst"], entry_doc = "index", + upward_bundles = ["//src/extensions/docs:extensions"], visibility = ["//visibility:public"], ) diff --git a/src/extensions/score_metamodel/external_needs.py b/src/extensions/score_metamodel/external_needs.py index 51f927a21..55ed07909 100644 --- a/src/extensions/score_metamodel/external_needs.py +++ b/src/extensions/score_metamodel/external_needs.py @@ -54,7 +54,9 @@ def _parse_bazel_external_need(s: str) -> ExternalNeedsSource | None: repo, path_to_target = repo_and_path.split("//", 1) repo = repo.lstrip("@") # empty for same-repo `//pkg:needs_json` - if target in ("needs_json", "needs_json_file", "docs_sources"): + if target in ("needs_json", "needs_json_file", "docs_sources") or target.endswith( + "_needs_upward" + ): return ExternalNeedsSource( bazel_module=repo, path_to_target=path_to_target, @@ -137,9 +139,11 @@ def extend_needs_json_exporter(config: Config, params: list[str]) -> None: """ for p in params: - # Note: we are currently addinig these values to config after config-inited. - # This is wrong. But good enough. - config.add(p, default="", rebuild="env", types=(), description="") + # Most callers register values during extension setup, before Sphinx + # reads conf.py. Keep this helper usable for callers that do not, but + # do not try to register an already-known value at config-inited. + if not hasattr(config, p): + config.add(p, default="", rebuild="env", types=(), description="") if not getattr(config, p): logger.error( @@ -160,6 +164,15 @@ def temp(self: NeedsList): NeedsList._finalise = temp # pyright: ignore[reportPrivateUsage] +def _external_needs_base_url(config: Config, needs_json_data: dict[str, object]) -> str: + """Return the consuming documentation project's base URL for imported Needs.""" + consumer_project_url = getattr(config, "project_url", "") + project_url = consumer_project_url or needs_json_data.get("project_url", "") + if not isinstance(project_url, str) or not project_url: + return "" + return project_url.rstrip("/") + "/main" + + def get_external_needs_source(external_needs_source: str) -> list[ExternalNeedsSource]: if external_needs_source: # Path taken for all invocations via `bazel` @@ -194,8 +207,7 @@ def add_external_needs_json(e: ExternalNeedsSource, config: Config): assert isinstance(config.needs_external_needs, list) # pyright: ignore[reportUnknownMemberType] config.needs_external_needs.append( # pyright: ignore[reportUnknownMemberType] { - "base_url": needs_json_data["project_url"] - + "/main", # for now always "main" + "base_url": _external_needs_base_url(config, needs_json_data), "json_path": json_file, } ) @@ -228,7 +240,10 @@ def add_external_docs_sources(e: ExternalNeedsSource, config: Config): def connect_external_needs(app: Sphinx, config: Config): - extend_needs_json_exporter(config, ["project_url"]) + # A host docs() config owns the canonical URL. Bundle-local Needs exports + # do not need to carry one because they can be imported by different hosts. + if getattr(config, "project_url", ""): + extend_needs_json_exporter(config, ["project_url"]) # Local external needs from DATA (e.g. :needs_json or :docs_sources) external_needs = get_external_needs_source(app.config.external_needs_source) @@ -240,7 +255,7 @@ def connect_external_needs(app: Sphinx, config: Config): for e in external_needs: if e.target == "needs_json": add_external_needs_json(e, app.config) - elif e.target == "needs_json_file": + elif e.target == "needs_json_file" or e.target.endswith("_needs_upward"): _add_needs_json_file(e, app.config) elif e.target == "docs_sources": add_external_docs_sources(e, app.config) @@ -251,16 +266,29 @@ def connect_external_needs(app: Sphinx, config: Config): def _add_needs_json_file(ext_needs: ExternalNeedsSource, config: Config) -> None: - """Resolve a needs_json_file target from runfiles and register it.""" - json_file_raw = ( - Path(_runfiles_module_dir(ext_needs)) / ext_needs.path_to_target / "needs.json" - ) + """Resolve a needs export target from runfiles and register it.""" r = get_runfiles_dir() - json_file = r / json_file_raw + module_dir = Path(_runfiles_module_dir(ext_needs)) + json_file_candidates = [ + r / module_dir / ext_needs.path_to_target / "needs.json", + ] + # Named docs_bundle exports use a target-specific output directory because one + # Bazel package may contain several bundles. Keep the historical flat path + # above for normal needs_json_file targets and try the namespaced layout + # used by bundle-local exports as a fallback. + if ext_needs.target != "needs_json_file": + json_file_candidates.append( + r / module_dir / ext_needs.path_to_target / ext_needs.target / "needs.json" + ) + + json_file = next( + (candidate for candidate in json_file_candidates if candidate.is_file()), + json_file_candidates[0], + ) logger.debug(f"External needs_json_file: {json_file}") try: needs_json_data = json.loads( - Path(json_file).read_text(encoding="utf-8") # pyright: ignore[reportAny] + json_file.read_text(encoding="utf-8") # pyright: ignore[reportAny] ) except FileNotFoundError: logger.error( @@ -274,7 +302,7 @@ def _add_needs_json_file(ext_needs: ExternalNeedsSource, config: Config) -> None return config.needs_external_needs.append( { # pyright: ignore[reportUnknownMemberType] - "base_url": needs_json_data.get("project_url", "") + "/main", + "base_url": _external_needs_base_url(config, needs_json_data), "json_path": json_file, } ) diff --git a/src/extensions/score_metamodel/tests/test_external_needs.py b/src/extensions/score_metamodel/tests/test_external_needs.py index 4c834590c..bdcb30913 100644 --- a/src/extensions/score_metamodel/tests/test_external_needs.py +++ b/src/extensions/score_metamodel/tests/test_external_needs.py @@ -101,6 +101,18 @@ def test_single_entry_json_no_path(): ] +def test_named_bundle_needs_upward_entry(): + result = parse_external_needs_sources_from_DATA('["//docs:component_needs_upward"]') + assert result == [ + ExternalNeedsSource( + bazel_module="", + path_to_target="docs", + target="component_needs_upward", + is_local=True, + ) + ] + + def test_multiple_entries(): result = parse_external_needs_sources_from_DATA( '["@repo1//:needs_json", "@repo2//:needs_json"]' @@ -203,6 +215,34 @@ def test_add_external_needs_json_appends_entry_local( assert Path(entry["json_path"]) == json_path +def test_add_external_needs_json_uses_consuming_project_url( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """External Needs links use the importing documentation project's URL.""" + e = ExternalNeedsSource( + bazel_module="ext_mod", target="needs_json", path_to_target="" + ) + config = Config() + config.needs_external_needs = [] + config.project_url = "https://example.test/consumer" + + rel_json = Path("ext_mod+/needs_json/_build/needs/needs.json") + json_path = tmp_path / rel_json + json_path.parent.mkdir(parents=True, exist_ok=True) + json_path.write_text( + json.dumps({"project_url": "https://example.test/provider"}), + encoding="utf-8", + ) + monkeypatch.setattr(ext_needs, "get_runfiles_dir", lambda: tmp_path) + + add_external_needs_json(e, config) + + assert config.needs_external_needs is not None + assert config.needs_external_needs[0]["base_url"] == ( + "https://example.test/consumer/main" + ) + + def test_add_needs_json_file_appends_entry( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -238,6 +278,36 @@ def test_add_needs_json_file_appends_entry( assert Path(entry["json_path"]) == json_path +def test_add_named_bundle_needs_upward_appends_entry( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Bundle-local exports resolve from their target-specific output directory.""" + rel_json = Path("_main/docs/component_needs_upward/needs.json") + json_path = tmp_path / rel_json + json_path.parent.mkdir(parents=True, exist_ok=True) + json_path.write_text( + json.dumps({"project_url": "https://example.test/bundle"}), + encoding="utf-8", + ) + + config = Config() + config.needs_external_needs = [] + monkeypatch.setattr(ext_needs, "get_runfiles_dir", lambda: tmp_path) + + _add_needs_json_file( + ExternalNeedsSource( + bazel_module="", + target="component_needs_upward", + path_to_target="docs", + is_local=True, + ), + config, + ) + + assert config.needs_external_needs is not None + assert Path(config.needs_external_needs[0]["json_path"]) == json_path + + def test_add_external_needs_json_missing_file_keeps_list_empty( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/src/extensions/score_sphinx_bundle/__init__.py b/src/extensions/score_sphinx_bundle/__init__.py index 523649be0..d3f3c6032 100644 --- a/src/extensions/score_sphinx_bundle/__init__.py +++ b/src/extensions/score_sphinx_bundle/__init__.py @@ -78,6 +78,12 @@ def setup(app: Sphinx) -> dict[str, object]: for e in score_extensions: app.setup_extension(e) + # Interpret GitHub-compatible Mermaid fences as Sphinx Mermaid directives. + # This keeps ``mermaid`` fences rendering in both GitHub Markdown and + # the generated Sphinx documentation. + if isinstance(app.config.myst_fence_as_directive, set): + app.config.myst_fence_as_directive.add("mermaid") + # Enable cross-reference resolution for [text](file.md#anchor) links up to h4. # Without this, MyST treats heading anchors as opaque and cannot resolve them. config_setdefault(app.config, "myst_heading_anchors", 4) diff --git a/src/tests/docs_bzl/README.md b/src/tests/docs_bzl/README.md index b270555a9..1b4a26966 100644 --- a/src/tests/docs_bzl/README.md +++ b/src/tests/docs_bzl/README.md @@ -22,6 +22,7 @@ docs_bzl/ │ ├── nested_bundles/ │ ├── subdirectory_bundle/ │ ├── external_bundle/ +│ ├── upward_bundles/ │ ├── local_version_mismatch/ │ └── invalid_bundle_placements/ └── test_.py diff --git a/src/tests/docs_bzl/cross_module_fixture/BUILD b/src/tests/docs_bzl/cross_module_fixture/BUILD index 59fd03761..65bd452e8 100644 --- a/src/tests/docs_bzl/cross_module_fixture/BUILD +++ b/src/tests/docs_bzl/cross_module_fixture/BUILD @@ -17,4 +17,5 @@ docs( source_dir = "docs", project = "Score Docs Compatibility Fixture", project_url = "https://eclipse-score.github.io/docs-as-code", + metamodel = ":docs/metamodel.yaml", ) diff --git a/src/tests/docs_bzl/cross_module_fixture/docs/conf.py b/src/tests/docs_bzl/cross_module_fixture/docs/conf.py new file mode 100644 index 000000000..8e8533bfe --- /dev/null +++ b/src/tests/docs_bzl/cross_module_fixture/docs/conf.py @@ -0,0 +1,18 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +project = "Score Docs Compatibility Fixture" +project_url = "https://eclipse-score.github.io/docs-as-code" +extensions = ["score_sphinx_bundle"] +required_in_id = ["fixture"] +suppress_warnings = ["needs.link_outgoing"] diff --git a/src/tests/docs_bzl/cross_module_fixture/docs/metamodel.yaml b/src/tests/docs_bzl/cross_module_fixture/docs/metamodel.yaml new file mode 100644 index 000000000..39767710d --- /dev/null +++ b/src/tests/docs_bzl/cross_module_fixture/docs/metamodel.yaml @@ -0,0 +1,28 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +needs_types: + test_req: + title: Test Requirement + prefix: test_req__ + parts: 3 + mandatory_options: + id: ^test_req__[0-9a-zA-Z_]*$ + version: ^[0-9]+$ + optional_options: + status: ^(draft|valid)$ + tags: .* + content: .* + template: .* + optional_links: + links: ANY +links: {} diff --git a/src/tests/docs_bzl/scenarios/data_files_runfiles/BUILD b/src/tests/docs_bzl/scenarios/data_files_runfiles/BUILD index 2743758e8..50e321557 100644 --- a/src/tests/docs_bzl/scenarios/data_files_runfiles/BUILD +++ b/src/tests/docs_bzl/scenarios/data_files_runfiles/BUILD @@ -46,6 +46,7 @@ docs_bundle( docs( source_dir = "docs", + test_sources = ["src/tests/docs_bzl/scenarios/data_files_runfiles"], bundles = [{ "bundle": ":data_bundle", "mount_at": "data_test", diff --git a/src/tests/docs_bzl/scenarios/invalid_bundle_placements/BUILD b/src/tests/docs_bzl/scenarios/invalid_bundle_placements/BUILD index 82dc36edf..b946d4dca 100644 --- a/src/tests/docs_bzl/scenarios/invalid_bundle_placements/BUILD +++ b/src/tests/docs_bzl/scenarios/invalid_bundle_placements/BUILD @@ -75,3 +75,18 @@ docs_bundle( ], tags = ["manual"], ) + +# Upward hierarchy edges are ordinary Bazel dependencies and must remain +# acyclic. This target is intentionally invalid and is built only by the +# corresponding negative test. +docs_bundle( + name = "cycle_a", + upward_bundles = [":cycle_b"], + tags = ["manual"], +) + +docs_bundle( + name = "cycle_b", + upward_bundles = [":cycle_a"], + tags = ["manual"], +) diff --git a/src/tests/docs_bzl/scenarios/upward_bundles/BUILD b/src/tests/docs_bzl/scenarios/upward_bundles/BUILD new file mode 100644 index 000000000..b9bf2ee23 --- /dev/null +++ b/src/tests/docs_bzl/scenarios/upward_bundles/BUILD @@ -0,0 +1,68 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("//:docs.bzl", "docs", "docs_bundle") + +docs_bundle( + name = "platform", + source_dir = "platform", +) + +docs_bundle( + name = "component", + source_dir = "component", + upward_bundles = [":docs_source_bundle"], +) + +# Keep the source-less hierarchy-group export contract covered as well. +docs_bundle( + name = "hierarchy_group", + upward_bundles = [":component"], + tags = ["manual"], +) + +# Multiple direct parents are merged in declaration order. The hierarchy group +# and platform together provide both the module/component closure and the +# platform-level Need. +docs_bundle( + name = "multiple_parent_group", + upward_bundles = [":hierarchy_group", ":platform"], + tags = ["manual"], +) + +# Both direct parents expose the component/module closure. The merger must +# accept the identical shared ancestor records without duplicating them. +docs_bundle( + name = "diamond_group", + upward_bundles = [":hierarchy_group", ":component"], + tags = ["manual"], +) + +docs_bundle( + name = "subcomponent", + source_dir = "subcomponent", + upward_bundles = [":hierarchy_group"], +) + +docs( + source_dir = "module", + project = "SCORE upward bundle example", + project_url = "https://example.invalid/upward-bundles", + upward_bundles = [":platform"], + test_sources = ["src/tests/docs_bzl/scenarios/upward_bundles"], + metamodel = "//src/extensions/score_metamodel:metamodel_yaml", + bundles = [ + {"bundle": ":component", "mount_at": "component"}, + {"bundle": ":subcomponent", "mount_at": "component/subcomponent"}, + ], +) diff --git a/src/tests/docs_bzl/scenarios/upward_bundles/component/index.rst b/src/tests/docs_bzl/scenarios/upward_bundles/component/index.rst new file mode 100644 index 000000000..b96da9655 --- /dev/null +++ b/src/tests/docs_bzl/scenarios/upward_bundles/component/index.rst @@ -0,0 +1,37 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Component +========= + +.. comp:: Seat heating controller + :id: comp__seat_heating_controller + :version: 1 + :security: NO + :safety: QM + :status: valid + :belongs_to: feat__seat_heating + :consists_of: comp__seat_heating_sensor + +.. comp_req:: Controller temperature control + :id: comp_req__component__temperature_control + :version: 1 + :reqtype: Functional + :security: NO + :safety: QM + :status: valid + :derived_from: feat_req__module__seat_heating + :satisfied_by: comp__seat_heating_controller + + The controller regulates the requested heating level. diff --git a/src/tests/docs_bzl/scenarios/upward_bundles/module/conf.py b/src/tests/docs_bzl/scenarios/upward_bundles/module/conf.py new file mode 100644 index 000000000..76939a9fa --- /dev/null +++ b/src/tests/docs_bzl/scenarios/upward_bundles/module/conf.py @@ -0,0 +1,22 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +project = "SCORE upward bundle example" +project_url = "https://example.invalid/upward-bundles" +version = "0.0.0" + +extensions = ["score_sphinx_bundle"] + +# The module's feature requirement lives in module/index.rst. Its meaningful +# ID part is `module`, not the rebased docname `index`. +required_in_id = ["module"] diff --git a/src/tests/docs_bzl/scenarios/upward_bundles/module/index.rst b/src/tests/docs_bzl/scenarios/upward_bundles/module/index.rst new file mode 100644 index 000000000..585dae6cf --- /dev/null +++ b/src/tests/docs_bzl/scenarios/upward_bundles/module/index.rst @@ -0,0 +1,43 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Module +====== + +The module-level feature requirement is reviewed against +:need:`feat_req__platform__seat_heating` from the platform bundle. + +.. feat:: Seat heating + :id: feat__seat_heating + :version: 1 + :security: NO + :safety: QM + :status: valid + +.. feat_req:: Seat heating availability + :id: feat_req__module__seat_heating + :version: 1 + :reqtype: Functional + :security: NO + :safety: QM + :status: valid + :valid_from: v1.0 + :satisfied_by: feat__seat_heating + + The seat heating feature is available to the vehicle user. + +.. mod:: Seat heating module + :id: mod__seat_heating_module + :version: 1 + :includes: comp__seat_heating_controller diff --git a/src/tests/docs_bzl/scenarios/upward_bundles/platform/index.rst b/src/tests/docs_bzl/scenarios/upward_bundles/platform/index.rst new file mode 100644 index 000000000..6e4b7e973 --- /dev/null +++ b/src/tests/docs_bzl/scenarios/upward_bundles/platform/index.rst @@ -0,0 +1,35 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License, Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Platform +======== + +.. feat:: Platform seat heating + :id: feat__platform__seat_heating + :version: 1 + :security: NO + :safety: QM + :status: valid + +.. feat_req:: Platform seat heating availability + :id: feat_req__platform__seat_heating + :version: 1 + :reqtype: Functional + :security: NO + :safety: QM + :status: valid + :valid_from: v1.0 + :satisfied_by: feat__platform__seat_heating + + The platform makes the seat heating capability available. diff --git a/src/tests/docs_bzl/scenarios/upward_bundles/subcomponent/index.rst b/src/tests/docs_bzl/scenarios/upward_bundles/subcomponent/index.rst new file mode 100644 index 000000000..ec5095b49 --- /dev/null +++ b/src/tests/docs_bzl/scenarios/upward_bundles/subcomponent/index.rst @@ -0,0 +1,36 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Subcomponent +============ + +.. comp:: Seat heating sensor + :id: comp__seat_heating_sensor + :version: 1 + :security: NO + :safety: QM + :status: valid + :belongs_to: feat__seat_heating + +.. comp_req:: Sensor temperature measurement + :id: comp_req__subcomponent__temp_measure + :version: 1 + :reqtype: Functional + :security: NO + :safety: QM + :status: valid + :derived_from: feat_req__module__seat_heating + :satisfied_by: comp__seat_heating_sensor + + The sensor reports the measured seat temperature. diff --git a/src/tests/docs_bzl/test_basic_docs.py b/src/tests/docs_bzl/test_basic_docs.py index ff92ffb1e..b2c5354ff 100644 --- a/src/tests/docs_bzl/test_basic_docs.py +++ b/src/tests/docs_bzl/test_basic_docs.py @@ -34,6 +34,13 @@ def test_basic_docs_builds_needs_without_conf_py(): data = load_needs_json(result.artifacts["needs.json"]) assert data["current_version"], "current_version must be non-empty" + +def test_basic_docs_exposes_stable_and_legacy_source_bundle_labels(): + run_scenario("build", "basic_docs", ":docs_source_bundle") + run_scenario("build", "basic_docs", ":_docs_source_bundle") + + +def test_basic_docs_generated_conf_uses_module_name_for_required_in_id(): generated_conf = built_output("scenarios/basic_docs", "docs/conf.py") assert 'required_in_id = ["docs_as_code"]' in generated_conf.read_text( encoding="utf-8" diff --git a/src/tests/docs_bzl/test_cross_module_compatibility.py b/src/tests/docs_bzl/test_cross_module_compatibility.py index c2d428bd8..5de18ab66 100644 --- a/src/tests/docs_bzl/test_cross_module_compatibility.py +++ b/src/tests/docs_bzl/test_cross_module_compatibility.py @@ -88,6 +88,53 @@ def _write_cross_module_consumer( ) +def _write_cross_module_upward_consumer(workspace: Path) -> None: + """Add a local bundle that consumes an upward export from the fixture.""" + _write_cross_module_consumer( + workspace, + "score_cross_module_compatibility_allow_missing_mandatory_attributes = True\n" + "score_cross_module_compatibility_allow_version_mismatches = True", + ) + workspace.joinpath("BUILD").write_text( + """load(\"@score_docs_as_code//:docs.bzl\", \"docs\", \"docs_bundle\") + +docs_bundle( + name = "consumer_child", + source_dir = "child", + metamodel = "//:docs/metamodel.yaml", + upward_bundles = ["@score_docs_compatibility_fixture//:docs_source_bundle"], +) + +docs( + source_dir = "docs", + metamodel = "//:docs/metamodel.yaml", + bundles = [ + { + "bundle": "@score_docs_compatibility_fixture//:docs_source_bundle", + "mount_at": "fixture", + }, + {"bundle": ":consumer_child", "mount_at": "child"}, + ], +) +""", + encoding="utf-8", + ) + child = workspace / "child" + child.mkdir() + child.joinpath("index.rst").write_text( + """Consumer child +=============== + +.. test_req:: Consumer child target + :id: test_req__child__target + :status: valid + :version: 2 + :links: test_req__fixture__source +""", + encoding="utf-8", + ) + + def test_cross_module_version_mismatch_is_reported_without_failing( tmp_path: Path, ) -> None: @@ -137,6 +184,23 @@ def test_external_findings_remain_fatal_by_default(tmp_path: Path) -> None: assert "is missing required attribute: `status`" in result.stderr +def test_cross_module_upward_bundle_is_consumed_by_a_local_bundle( + tmp_path: Path, +) -> None: + _write_cross_module_upward_consumer(tmp_path) + result = subprocess.run( + ["bazel", f"--bazelrc={repo_root() / '.bazelrc'}", "run", "//:docs"], + cwd=tmp_path, + text=True, + capture_output=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert (tmp_path / "_build/index.html").is_file() + assert (tmp_path / "_build/child/index.html").is_file() + assert (tmp_path / "_build/fixture/index.html").is_file() + + def test_local_version_mismatch_remains_fatal() -> None: result = run_scenario("run", "local_version_mismatch", ":docs", expect_error=True) assert "condition 'version==1' not satisfied" in result.stderr diff --git a/src/tests/docs_bzl/test_external_needs.py b/src/tests/docs_bzl/test_external_needs.py index 41a3b7b29..1809e528b 100644 --- a/src/tests/docs_bzl/test_external_needs.py +++ b/src/tests/docs_bzl/test_external_needs.py @@ -35,8 +35,8 @@ def test_producer_needs_json_contains_local_need(): def test_consumer_link_resolves(): - """A consumer renders the producer's need as an external HTML link.""" + """A consumer renders the producer's need under its canonical site URL.""" result = run_scenario("run", "external_needs/consumer", ":docs") html = (result.build_dir / "index.html").read_text(encoding="utf-8") - assert "external-needs-producer/main/index.html#test_req__producer__demo" in html + assert "external-needs-consumer/main/index.html#test_req__producer__demo" in html diff --git a/src/tests/docs_bzl/test_invalid_bundle_placements.py b/src/tests/docs_bzl/test_invalid_bundle_placements.py index 2c59249b9..9650e3a22 100644 --- a/src/tests/docs_bzl/test_invalid_bundle_placements.py +++ b/src/tests/docs_bzl/test_invalid_bundle_placements.py @@ -31,3 +31,4 @@ def test_invalid_bundle_placements_are_rejected_during_analysis(): run_scenario( "build", "invalid_bundle_placements", ":bad_attach_to", expect_error=True ) + run_scenario("build", "invalid_bundle_placements", ":cycle_a", expect_error=True) diff --git a/src/tests/docs_bzl/test_upward_bundles.py b/src/tests/docs_bzl/test_upward_bundles.py new file mode 100644 index 000000000..e3354423a --- /dev/null +++ b/src/tests/docs_bzl/test_upward_bundles.py @@ -0,0 +1,159 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Public docs_bundle() hierarchy-contract tests.""" + +import json + +from src.tests.docs_bzl.helpers import built_output, load_needs, run_scenario + + +def test_real_score_needs_cross_the_module_component_hierarchy(): + """Use the real SCORE metamodel across module, component and subcomponent.""" + result = run_scenario("build", "upward_bundles", ":needs_json") + + assert result.artifacts + needs = load_needs(result.artifacts["needs.json"]) + + expected_ids = { + "feat__seat_heating", + "feat_req__module__seat_heating", + "mod__seat_heating_module", + "comp__seat_heating_controller", + "comp_req__component__temperature_control", + "comp__seat_heating_sensor", + "comp_req__subcomponent__temp_measure", + } + assert expected_ids <= needs.keys() + + # Keep this assertion deliberately representation-independent: Sphinx-Needs + # serializes scalar and list links differently across supported versions. + needs_text = json.dumps(needs, sort_keys=True) + assert needs_text.count("feat_req__module__seat_heating") >= 3 + assert "comp__seat_heating_controller" in needs_text + assert "comp__seat_heating_sensor" in needs_text + + +def test_docs_can_consume_an_upward_bundle_for_its_own_sources(): + run_scenario( + "build", + "upward_bundles", + ":_docs_source_bundle_needs_upward", + ) + source_needs = load_needs( + built_output( + "scenarios/upward_bundles", + "_docs_source_bundle_needs_upward/needs.json", + ) + ) + assert "feat_req__platform__seat_heating" in source_needs + + result = run_scenario("run", "upward_bundles", ":docs") + html = (result.build_dir / "index.html").read_text(encoding="utf-8") + assert "feat_req__platform__seat_heating" in html + + +def test_docs_exposes_a_stable_public_source_bundle_alias(): + run_scenario("build", "upward_bundles", ":docs_source_bundle") + run_scenario("build", "upward_bundles", ":docs_source_bundle_needs_upward") + needs = load_needs( + built_output( + "scenarios/upward_bundles", + "docs_source_bundle_needs_upward/needs.json", + ) + ) + assert "feat_req__platform__seat_heating" in needs + + +def test_bundle_needs_exports_are_local_and_include_declared_ancestors(): + run_scenario( + "build", + "upward_bundles", + ":component_needs_upward", + ) + run_scenario( + "build", + "upward_bundles", + ":subcomponent_needs_upward", + ) + + component_needs = load_needs( + built_output( + "scenarios/upward_bundles", + "component_needs_upward/needs.json", + ) + ) + subcomponent_needs = load_needs( + built_output( + "scenarios/upward_bundles", + "subcomponent_needs_upward/needs.json", + ) + ) + + assert "comp_req__component__temperature_control" in component_needs + assert "feat_req__module__seat_heating" in component_needs + assert "comp_req__subcomponent__temp_measure" not in component_needs + assert "comp_req__subcomponent__temp_measure" in subcomponent_needs + + +def test_upward_bundle_documentation_renders_all_three_levels(): + result = run_scenario("run", "upward_bundles", ":docs") + + assert (result.build_dir / "index.html").is_file() + assert (result.build_dir / "component" / "index.html").is_file() + assert (result.build_dir / "component" / "subcomponent" / "index.html").is_file() + + +def test_aggregators_can_group_upward_bundle_dependencies(): + run_scenario("build", "upward_bundles", ":hierarchy_group") + run_scenario("build", "upward_bundles", ":subcomponent_needs_upward") + + hierarchy_needs = load_needs( + built_output( + "scenarios/upward_bundles", + "hierarchy_group_needs_upward/needs.json", + ) + ) + assert "feat_req__module__seat_heating" in hierarchy_needs + + +def test_multiple_direct_parents_merge_their_upward_exports(): + run_scenario("build", "upward_bundles", ":multiple_parent_group_needs_upward") + + needs = load_needs( + built_output( + "scenarios/upward_bundles", + "multiple_parent_group_needs_upward/needs.json", + ) + ) + assert { + "feat_req__platform__seat_heating", + "feat_req__module__seat_heating", + "comp_req__component__temperature_control", + } <= needs.keys() + assert len(needs) == 7 + + +def test_shared_ancestor_is_deduplicated_in_a_diamond(): + run_scenario("build", "upward_bundles", ":diamond_group_needs_upward") + + needs = load_needs( + built_output( + "scenarios/upward_bundles", + "diamond_group_needs_upward/needs.json", + ) + ) + assert { + "feat_req__module__seat_heating", + "comp_req__component__temperature_control", + } <= needs.keys() + assert len(needs) == 7 From 8ea6d22ab38750d010e0f18280f5f3804533ee33 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 28 Aug 2026 23:23:43 +0200 Subject: [PATCH 2/2] chore: remove temporary merge test and target tooling --- backlog.md | 44 +---- scripts_bazel/docs_targets.sh | 133 ------------- scripts_bazel/tests/BUILD | 14 -- .../tests/merge_needs_json_app_tests.py | 182 ------------------ 4 files changed, 7 insertions(+), 366 deletions(-) delete mode 100755 scripts_bazel/docs_targets.sh delete mode 100644 scripts_bazel/tests/merge_needs_json_app_tests.py diff --git a/backlog.md b/backlog.md index 707f1c09a..ed50ed8a6 100644 --- a/backlog.md +++ b/backlog.md @@ -19,17 +19,18 @@ Last updated: 2026-08-28 ## Current state -The local branch is based on `upstream/main` at `298732e4`. The previous merge -commit was intentionally removed with a soft reset; the resulting changes are -currently staged and deliberately have no commit yet. +The local `upward_bundle` branch is based on `upstream/main` at `298732e4`. +The WIP integration is secured in commits `9eba2304` and `1a16863c` and has +been force-updated to `upstream/upward_bundle`. The working tree is clean; no +staged changes remain. -The staged delta is currently 37 files with approximately 2,580 additions and -42 deletions. It is a working integration result, not yet a reviewable PR. +The final WIP delta is currently 33 files with approximately 1,947 additions +and 42 deletions. It is a working integration result, not yet a reviewable PR. Verification completed before the split: ```text -bazel test //... 23/23 passed +bazel test //... 21/21 passed .venv_docs/bin/python -m pytest -q src/tests/docs_bzl 28 passed ``` @@ -86,20 +87,6 @@ The Mermaid-fence handling in `score_sphinx_bundle` supports the diagrams in the Markdown documentation. It should remain only if the documentation PR needs it and has a rendering regression test. -### 5. Merger application tests - -`merge_needs_json.py` is already present in `upstream/main`. The staged -application-level test exercises merging actual Sphinx-generated inventories. -The staged `merge_needs_json_unit_tests.py`, however, is byte-for-byte -identical to the existing `merge_needs_json_tests.py` and must be removed -before splitting the PRs. - -### 6. Developer tooling - -`scripts_bazel/docs_targets.sh` lists documentation, bundle-local, and upward -Needs targets. It is useful convenience tooling but is not required by the -hierarchy implementation. - ## Proposed PR plan The PRs should be stacked on `upstream/main` in this order. Test fixtures and @@ -148,27 +135,10 @@ Include: - Mermaid-fence support if required by the new Markdown diagrams, together with a small rendering test. -### PR 4 — Sphinx application-level tests for `merge_needs_json` (optional) - -Approximate size: 190 LOC. - -Keep the application-level test if the stronger Sphinx-generated-inventory -contract is useful. Do not include the duplicate unit-test file or its BUILD -target; the existing upstream unit test already covers that behavior. - -### PR 5 — `docs_targets.sh` developer utility (optional) - -Approximate size: 133 LOC. - -Submit this independently if the target-discovery command is useful to users. -It has no production dependency on the hierarchy implementation. - ## Cleanup before creating the PRs Remove or split out the following from the current staged integration: -- the duplicate `merge_needs_json_unit_tests.py` and its BUILD entry; -- `scripts_bazel/docs_targets.sh` unless it is wanted as separate tooling; - Mermaid-fence support unless it is required by the documentation PR and is covered by a regression test. diff --git a/scripts_bazel/docs_targets.sh b/scripts_bazel/docs_targets.sh deleted file mode 100755 index de0410b9b..000000000 --- a/scripts_bazel/docs_targets.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -set -euo pipefail - -query_scope="${1:-//...}" - -docs_targets="$(bazel query \ - 'attr(generator_function, "^docs$", '"${query_scope}"')' \ - --output=label)" -bundle_targets="$(bazel query \ - 'kind("_docs_bundle rule", attr(generator_function, "^docs_bundle$", '"${query_scope}"'))' \ - --output=label)" -bundle_needs_targets="$(bazel query \ - 'attr(generator_function, "^docs_bundle$", '"${query_scope}"')' \ - --output=label)" - -query_output="$( - { - while IFS= read -r target; do - [[ -n "$target" ]] || continue - case "$target" in - */tests/*|*.run|*.serve|*.venv|*.find_main) - continue - ;; - esac - name="${target##*:}" - case "$name" in - docs|docs_check|docs_link_check|live_preview|ide_support|\ - needs_json|needs_json_file|metrics_json|sourcelinks_json|traceability_gate) - printf 'docs\t%s\n' "$target" - ;; - esac - done <<< "$docs_targets" - - while IFS= read -r target; do - [[ -n "$target" ]] || continue - case "$target" in - */tests/*|*.run|*.serve|*.venv|*.find_main) - continue - ;; - esac - name="${target##*:}" - [[ "$name" != _* ]] && printf 'bundle\t%s\n' "$target" - done <<< "$bundle_targets" - - while IFS= read -r target; do - [[ -n "$target" ]] || continue - case "$target" in - */tests/*|*.run|*.serve|*.venv|*.find_main) - continue - ;; - esac - name="${target##*:}" - case "$name" in - _*_needs_local) - ;; - _*_needs_upward) - ;; - *_needs_local) - printf 'local\t%s\n' "$target" - ;; - *_needs_upward) - printf 'upward\t%s\n' "$target" - ;; - esac - done <<< "$bundle_needs_targets" - } | LC_ALL=C sort -t $'\t' -k2,2 -u -)" - -while IFS=$'\t' read -r role target; do - [[ -n "$target" ]] || continue - - name="${target##*:}" - case "$role" in - bundle) - description="mountable and composable documentation bundle" - ;; - local) - description="bundle-local Needs export (own sources only)" - ;; - upward) - description="bundle Needs export (own sources plus upward dependencies)" - ;; - docs) - case "$name" in - docs) - description="build the rendered documentation" - ;; - docs_check) - description="run the documentation validation checks" - ;; - docs_link_check) - description="check links in the documentation" - ;; - live_preview) - description="run the live documentation preview" - ;; - ide_support) - description="create the documentation development virtualenv" - ;; - needs_json) - description="build the complete composed Needs graph" - ;; - needs_json_file) - description="export the complete composed Needs graph as JSON" - ;; - metrics_json) - description="export traceability metrics from the Needs build" - ;; - sourcelinks_json) - description="export merged source-to-documentation links" - ;; - traceability_gate) - description="enforce configured traceability coverage thresholds" - ;; - esac - ;; - esac - - printf '%-90s %s\n' "$target" "$description" -done <<< "$query_output" diff --git a/scripts_bazel/tests/BUILD b/scripts_bazel/tests/BUILD index 43a3ea6b9..53c64d92e 100644 --- a/scripts_bazel/tests/BUILD +++ b/scripts_bazel/tests/BUILD @@ -33,20 +33,6 @@ score_pytest( pytest_config = "//:pyproject.toml", ) -score_pytest( - name = "merge_needs_json_app_tests", - srcs = ["merge_needs_json_app_tests.py"], - data = [ - "//src/tests/docs_bzl/scenarios/basic_docs:docs_bundle", - "//src/tests/docs_bzl/scenarios/external_needs/producer:docs_bundle", - "//src/tests/docs_bzl/scenarios/nested_bundles:docs_bundle", - ], - deps = [ - "//scripts_bazel:merge_needs_json", - ] + all_requirements, - pytest_config = "//:pyproject.toml", -) - score_pytest( name = "merge_needs_json_tests", srcs = ["merge_needs_json_tests.py"], diff --git a/scripts_bazel/tests/merge_needs_json_app_tests.py b/scripts_bazel/tests/merge_needs_json_app_tests.py deleted file mode 100644 index 481fe64e9..000000000 --- a/scripts_bazel/tests/merge_needs_json_app_tests.py +++ /dev/null @@ -1,182 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -"""App-level tests for the merge tool using needs.json from Sphinx.""" - -from __future__ import annotations - -import json -import shutil -from pathlib import Path -from typing import cast - -import pytest -from sphinx.testing.util import SphinxTestApp - -from scripts_bazel.merge_needs_json import merge_needs_json - -REPO_ROOT = Path(__file__).absolute().parents[2] -SCENARIOS_DIR = REPO_ROOT / "src/tests/docs_bzl/scenarios" - - -def _single_version_needs(needs_json: Path) -> tuple[str, dict[str, object]]: - data = json.loads(needs_json.read_text(encoding="utf-8")) - assert isinstance(data, dict) - raw_versions = cast(object, data["versions"]) - assert isinstance(raw_versions, dict) - versions = cast(dict[str, object], raw_versions) - assert len(versions) == 1 - - version_key, raw_version_object = next(iter(versions.items())) - assert isinstance(raw_version_object, dict) - version_data = cast(dict[str, object], raw_version_object) - raw_needs = version_data.get("needs") - assert isinstance(raw_needs, dict) - return version_key, cast(dict[str, object], raw_needs) - - -def _write_conf( - source_dir: Path, - *, - project: str, - project_url: str, - version: str | None = None, - producer_type: bool = False, -) -> None: - config = [ - f"project = {project!r}", - f"project_url = {project_url!r}", - 'extensions = ["sphinx_needs"]', - "needs_id_regex = r'^[A-Za-z0-9_-]{6,}'", - ] - if version is not None: - config.append(f"version = {version!r}") - if producer_type: - config.append( - "needs_types = [{" - "'directive': 'test_req', " - "'title': 'Test Requirement', " - "'prefix': 'test_req__', " - "'parts': 3, " - "'mandatory_options': {" - "'id': r'^test_req__[0-9a-zA-Z_]*$', " - "'status': r'^(draft|valid)$'}, " - "'optional_options': {'tags': '.*', 'content': '.*', " - "'template': '.*'}, " - "'mandatory_links': {}, " - "'optional_links': {}}]" - ) - - source_dir.joinpath("conf.py").write_text( - "\n".join(config) + "\n", - encoding="utf-8", - ) - - -def _build_needs_json( - source_dir: Path, - build_dir: Path, - monkeypatch: pytest.MonkeyPatch, -) -> Path: - app = SphinxTestApp( - buildername="needs", - srcdir=source_dir, - builddir=build_dir, - freshenv=True, - warningiserror=True, - ) - monkeypatch.chdir(source_dir) - try: - app.build() - finally: - app.cleanup() - - needs_json = build_dir / "needs" / "needs.json" - assert needs_json.is_file(), f"Sphinx did not create {needs_json}" - return needs_json - - -def test_merges_needs_json_generated_by_sphinx( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - basic_source = tmp_path / "basic_docs" - shutil.copytree(SCENARIOS_DIR / "basic_docs" / "docs", basic_source) - _write_conf( - basic_source, - project="Basic Test", - project_url="https://github.com/eclipse-score/docs-as-code", - version="0.0.0", - ) - - producer_source = tmp_path / "producer" - shutil.copytree( - SCENARIOS_DIR / "external_needs" / "producer" / "docs", - producer_source, - ) - _write_conf( - producer_source, - project="External Needs Producer", - project_url="https://example.invalid/external-needs-producer", - version="main", - producer_type=True, - ) - - empty_source = tmp_path / "empty" - shutil.copytree( - SCENARIOS_DIR / "nested_bundles" / "host_docs", - empty_source, - ) - _write_conf( - empty_source, - project="Mount contract fixture", - project_url="https://example.invalid/mount-contract", - ) - - basic_needs_json = _build_needs_json( - basic_source, tmp_path / "basic_out", monkeypatch - ) - producer_needs_json = _build_needs_json( - producer_source, tmp_path / "producer_out", monkeypatch - ) - empty_needs_json = _build_needs_json( - empty_source, tmp_path / "empty_out", monkeypatch - ) - - basic_version, basic_needs = _single_version_needs(basic_needs_json) - producer_version, producer_needs = _single_version_needs(producer_needs_json) - empty_version, empty_needs = _single_version_needs(empty_needs_json) - - assert basic_version == "0.0.0" - assert producer_version == "main" - assert empty_version == "" - assert basic_needs == {} - assert empty_needs == {} - assert set(producer_needs) == {"test_req__producer__demo"} - - merged = merge_needs_json([basic_needs_json, producer_needs_json, empty_needs_json]) - raw_merged_versions = merged["versions"] - assert isinstance(raw_merged_versions, dict) - merged_versions = cast(dict[str, object], raw_merged_versions) - assert set(merged_versions) == {basic_version} - - raw_merged_version = merged_versions[basic_version] - assert isinstance(raw_merged_version, dict) - merged_version = cast(dict[str, object], raw_merged_version) - raw_merged_needs = merged_version["needs"] - assert isinstance(raw_merged_needs, dict) - merged_needs = cast(dict[str, object], raw_merged_needs) - assert set(merged_needs) == set(producer_needs) - assert merged_version["needs_amount"] == len(merged_needs) - - # Top-level metadata remains owned by the first input. - assert merged["project"] == "Basic Test"