From 406a43f898a4da21849b15da0fe4e0c0b6e5c995 Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Sun, 13 Sep 2026 13:52:16 +0300 Subject: [PATCH 1/3] fix(test): isolate benchmark harness daemon runtimes scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the product with no runtime or cache of their own: the benchmark repository was indexed into the operator's live store and every one-shot joined the operator's account daemon. Source scripts/test-runtime.sh in both, start the private daemon before timing, record setup-time.txt and total-time.txt beside index-time.txt, and index from a repository path instead of querying a project in the live store. Add tests/test_benchmark_runtime_isolation_contract.sh, which fails before this change, and wire it into scripts/test.sh. Part of #1696. Signed-off-by: Anton Standrik --- scripts/benchmark-index.sh | 20 ++++ scripts/benchmark-search-graph.sh | 38 +++++++- scripts/test.sh | 3 + ...st_benchmark_runtime_isolation_contract.sh | 96 +++++++++++++++++++ 4 files changed, 152 insertions(+), 5 deletions(-) create mode 100755 tests/test_benchmark_runtime_isolation_contract.sh diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 756bda06e..71e694c17 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -9,6 +9,14 @@ LANG="${2:?}" REPO="${3:?}" RESULTS_DIR="${4:?}" +# The index must run against a daemon rendezvous and cache this run owns: only +# CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the +# benchmark repository was indexed into the operator's live store (#1696). +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) @@ -33,6 +41,16 @@ LOC=$(find "$REPO" -type f \ echo "$FILE_COUNT" > "$OUT/file-count.txt" echo "$LOC" > "$OUT/loc.txt" +# Start the private daemon before timing so index-time.txt measures the index +# alone. setup-time.txt keeps the activation cost attributable and +# total-time.txt is their sum — the figure comparable with earlier runs, which +# paid activation inside the index timing whenever no daemon was already warm. +SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))") +if ! "$BINARY" daemon start >/dev/null 2>&1; then + echo " $LANG: private daemon did not start" >&2 + exit 1 +fi + # Index via CLI and capture timing START_MS=$(python3 -c "import time; print(int(time.time()*1000))") @@ -43,6 +61,8 @@ ELAPSED=$((END_MS - START_MS)) echo "$INDEX_JSON" > "$OUT/00-index.json" echo "$ELAPSED" > "$OUT/index-time.txt" +echo "$((START_MS - SETUP_START_MS))" > "$OUT/setup-time.txt" +echo "$((END_MS - SETUP_START_MS))" > "$OUT/total-time.txt" # Extract node/edge counts (CLI wraps in MCP content envelope) NODES=$(echo "$INDEX_JSON" | python3 -c " diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh index cc94147ec..1d67b9581 100755 --- a/scripts/benchmark-search-graph.sh +++ b/scripts/benchmark-search-graph.sh @@ -3,18 +3,46 @@ # codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance. # # Usage: -# scripts/benchmark-search-graph.sh +# scripts/benchmark-search-graph.sh # # Example: -# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project +# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp ~/src/my-project +# +# The repository is indexed (untimed) into a private runtime and cache first; +# the queries then run against that index through a daemon this run keeps warm, +# so a timing never includes daemon activation and never touches the operator's +# live store (#1696). set -euo pipefail -BINARY="${1:?Usage: $0 }" -PROJECT="${2:?Usage: $0 }" +BINARY="${1:?Usage: $0 }" +REPO="${2:?Usage: $0 }" +REPO=$(cd "$REPO" && pwd -P) + +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + +if ! "$BINARY" daemon start >/dev/null 2>&1; then + echo "private daemon did not start" >&2 + exit 1 +fi +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}') +PROJECT=$(echo "$INDEX_JSON" | python3 -c " +import json, sys +d = json.load(sys.stdin) +if 'content' in d: + d = json.loads(d['content'][0]['text']) +print(d.get('project', '')) +" 2>/dev/null || echo "") +if [ -z "$PROJECT" ]; then + echo "index of $REPO did not report a project" >&2 + exit 1 +fi echo "Binary: $BINARY" -echo "Project: $PROJECT" +echo "Project: $PROJECT (indexed from $REPO)" echo "" run_case() { diff --git a/scripts/test.sh b/scripts/test.sh index 50d65643f..77cbf42d4 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -239,6 +239,9 @@ bash "$ROOT/tests/test_smoke_fixture_contract.sh" echo "=== Step 0i: parallel suite scheduler contract ===" bash "$ROOT/tests/test_parallel_harness_contract.sh" +echo "=== Step 0i2: benchmark harness runtime isolation contract (#1696) ===" +bash "$ROOT/tests/test_benchmark_runtime_isolation_contract.sh" + echo "=== Step 0j: venue parity contract (one harness, every venue) ===" bash "$ROOT/tests/test_venue_parity_contract.sh" diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh new file mode 100755 index 000000000..b944df768 --- /dev/null +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runtime-isolation contract for the benchmark harnesses (#1696, follow-up to +# #1691). +# +# scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the +# product with no runtime or cache of their own: the index landed in the +# operator's live store, every one-shot joined the operator's account daemon, +# and the timings depended on whatever that daemon was doing. Drive both with +# an environment-probe fixture and require that no product process ever +# receives the caller's runtime or cache, and that the index benchmark records +# the setup cost it now pays explicitly. + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +normalize_path() { + local path=${1%$'\r'} + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$path" 2>/dev/null && return 0 + fi + printf '%s\n' "${path//\\//}" +} + +ENV_PROBE="$WORKDIR/environment-probe" +cat > "$ENV_PROBE" <<'EOF' +#!/usr/bin/env bash +printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE" +[[ "${1-} ${2-}" == "daemon status" ]] && exit 1 +exit 0 +EOF +chmod +x "$ENV_PROBE" + +CALLER_CACHE="$WORKDIR/caller-cache" +CALLER_RUNTIME="$WORKDIR/caller-runtime" +REPO="$WORKDIR/repo" +mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" "$REPO" +echo 'def bench(): return 1' > "$REPO/bench.py" +CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE") +CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME") + +# The fixture answers nothing, so the search benchmark stops once the index +# reports no project; only the environment handed to the product is under test. +assert_isolated() { + local harness="$1" env_log="$2" private_root="" + [[ -s "$env_log" ]] || fail "$harness did not execute the environment-probe fixture" + while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do + local child_cache child_runtime + child_cache=$(normalize_path "$child_cache_raw") + child_runtime=$(normalize_path "$child_runtime_raw") + if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then + fail "$harness exposed the caller CBM_RUNTIME_DIR to a product process" + fi + if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then + fail "$harness exposed the caller CBM_CACHE_DIR to a product process" + fi + if [[ "${child_runtime%/*}" != "${child_cache%/*}" || + "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then + fail "$harness runtime/cache were not isolated beneath one private root" + fi + if [[ -n "$private_root" && "$private_root" != "${child_runtime%/*}" ]]; then + fail "$harness switched private roots mid-run" + fi + private_root="${child_runtime%/*}" + done < "$env_log" + [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root" +} + +INDEX_LOG="$WORKDIR/index-environment.log" +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$INDEX_LOG" \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/index.out" 2>&1 || true +assert_isolated "benchmark-index" "$INDEX_LOG" +for metric in setup-time total-time index-time; do + [[ -s "$WORKDIR/results/probe/$metric.txt" ]] || + fail "benchmark-index did not record $metric.txt" +done + +SEARCH_LOG="$WORKDIR/search-environment.log" +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \ + "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$REPO" \ + > "$WORKDIR/search.out" 2>&1 || true +assert_isolated "benchmark-search-graph" "$SEARCH_LOG" + +echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller" From 8f0ff31e7de014d65cd3c33fc884607a112f2c6b Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Mon, 21 Sep 2026 12:00:10 +0300 Subject: [PATCH 2/3] fix(test): escape benchmark paths, keep stderr, use monotonic clock Review follow-up for the benchmark harness isolation. - Both benchmark scripts build the index request with a JSON-escaped repository path (python3 json.dumps, the spelling soak-test.sh already uses): a path containing a quote or a backslash produced a payload the server could not parse. - benchmark-index.sh reads its three timestamps from one helper backed by time.monotonic_ns(); an NTP step mid-run no longer skews or negates a figure whose purpose is comparison across runs. - benchmark-search-graph.sh keeps the stderr of the index call and of the response parse in a run-private temp file (removed by the EXIT trap) and prints it, plus the first 500 bytes of the response, under the existing "did not report a project" line. - The contract test now refuses the index from its fixture and requires the cause to reach the output, and (POSIX only) drives a repository path with a quote and a backslash and requires the recorded request to parse as JSON with the resolved path. Both assertions fail against the previous head of this branch. Signed-off-by: Anton Standrik --- scripts/benchmark-index.sh | 19 +++++-- scripts/benchmark-search-graph.sh | 25 ++++++++-- ...st_benchmark_runtime_isolation_contract.sh | 49 +++++++++++++++++++ 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 71e694c17..0bc24af2f 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -19,6 +19,17 @@ trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) +# One pre-escaped spelling of the path for every request below, the way the +# soak harness builds its own: a repository path may legitimately contain a +# quote or a backslash, and hand-built JSON turns that into a parse error. +REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO") + +# Elapsed time is read from a monotonic clock, never the wall clock: an NTP +# step mid-run would otherwise skew — or negate — a figure whose whole purpose +# is comparison across runs. Its reference point is fixed per boot on every +# platform CPython supports here, so the three readings below are comparable +# even though each comes from its own process. +bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"; } OUT="$RESULTS_DIR/$LANG" mkdir -p "$OUT" @@ -45,18 +56,18 @@ echo "$LOC" > "$OUT/loc.txt" # alone. setup-time.txt keeps the activation cost attributable and # total-time.txt is their sum — the figure comparable with earlier runs, which # paid activation inside the index timing whenever no daemon was already warm. -SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))") +SETUP_START_MS=$(bench_now_ms) if ! "$BINARY" daemon start >/dev/null 2>&1; then echo " $LANG: private daemon did not start" >&2 exit 1 fi # Index via CLI and capture timing -START_MS=$(python3 -c "import time; print(int(time.time()*1000))") +START_MS=$(bench_now_ms) -INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}') +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}') -END_MS=$(python3 -c "import time; print(int(time.time()*1000))") +END_MS=$(bench_now_ms) ELAPSED=$((END_MS - START_MS)) echo "$INDEX_JSON" > "$OUT/00-index.json" diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh index 1d67b9581..6296d31fb 100755 --- a/scripts/benchmark-search-graph.sh +++ b/scripts/benchmark-search-graph.sh @@ -22,22 +22,41 @@ REPO=$(cd "$REPO" && pwd -P) # shellcheck source=test-runtime.sh source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" cbm_test_runtime_init -trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT +BENCH_TMP="" +trap 'cbm_test_runtime_cleanup "$BINARY"; [ -z "$BENCH_TMP" ] || rm -rf -- "$BENCH_TMP"' EXIT +BENCH_TMP=$(mktemp -d) +INDEX_ERR="$BENCH_TMP/index-stderr.log" if ! "$BINARY" daemon start >/dev/null 2>&1; then echo "private daemon did not start" >&2 exit 1 fi -INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}') +# One pre-escaped spelling of the path, the way the soak harness builds its +# own: a repository path may legitimately contain a quote or a backslash, and +# hand-built JSON turns that into a parse error. +REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO") +# Index and parse keep their stderr instead of discarding it: without it the +# failure below names only its symptom, and the cause — an unreadable +# repository, a refused daemon, a malformed envelope — is unrecoverable. +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" \ + 2>"$INDEX_ERR" || echo '{}') PROJECT=$(echo "$INDEX_JSON" | python3 -c " import json, sys d = json.load(sys.stdin) if 'content' in d: d = json.loads(d['content'][0]['text']) print(d.get('project', '')) -" 2>/dev/null || echo "") +" 2>>"$INDEX_ERR" || echo "") if [ -z "$PROJECT" ]; then echo "index of $REPO did not report a project" >&2 + if [ -s "$INDEX_ERR" ]; then + echo "--- index/parse stderr ---" >&2 + cat "$INDEX_ERR" >&2 + fi + if [ -n "$INDEX_JSON" ]; then + echo "--- index response (first 500 bytes) ---" >&2 + printf '%.500s\n' "$INDEX_JSON" >&2 + fi exit 1 fi diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh index b944df768..1dc618ae6 100755 --- a/tests/test_benchmark_runtime_isolation_contract.sh +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -33,6 +33,11 @@ ENV_PROBE="$WORKDIR/environment-probe" cat > "$ENV_PROBE" <<'EOF' #!/usr/bin/env bash printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE" +if [[ "${1-} ${2-}" == "cli index_repository" ]]; then + [[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG" + echo "probe: index refused" >&2 + exit 1 +fi [[ "${1-} ${2-}" == "daemon status" ]] && exit 1 exit 0 EOF @@ -93,4 +98,48 @@ CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \ > "$WORKDIR/search.out" 2>&1 || true assert_isolated "benchmark-search-graph" "$SEARCH_LOG" +# A refused index must name its cause. The fixture writes one line to stderr and +# exits non-zero; discarding it leaves "did not report a project" as the only +# thing an operator sees. +grep -q -- '--- index/parse stderr ---' "$WORKDIR/search.out" || + fail "benchmark-search-graph hid the index stderr behind its own message" +grep -q 'probe: index refused' "$WORKDIR/search.out" || + fail "benchmark-search-graph did not surface the cause of the index failure" + +# The request carrying the repository path has to be built as JSON. A path may +# legitimately contain a quote or a backslash — hand-built JSON turns that into +# a payload the server cannot parse, or one that means something else. NTFS +# rejects both characters in a path component, so this case is POSIX-only. +case "$(uname -s)" in +MINGW* | MSYS* | CYGWIN*) ;; +*) + QUIRKY_REPO="$WORKDIR/re\"po\\dir" + mkdir -p "$QUIRKY_REPO" + echo 'def bench(): return 1' > "$QUIRKY_REPO/bench.py" + QUIRKY_RESOLVED=$(cd "$QUIRKY_REPO" && pwd -P) + REQUEST_LOG="$WORKDIR/requests.log" + CBM_CACHE_DIR="$CALLER_CACHE" \ + CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ + CBM_BENCH_ENV_PROBE="$WORKDIR/quirky-environment.log" \ + CBM_BENCH_REQUEST_LOG="$REQUEST_LOG" \ + "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$QUIRKY_REPO" \ + > "$WORKDIR/quirky.out" 2>&1 || true + python3 - "$REQUEST_LOG" "$QUIRKY_RESOLVED" <<'PY' || fail "benchmark-search-graph built an index request that is not valid JSON for a quoted path" +import json +import sys + +request_log, expected = sys.argv[1], sys.argv[2] +try: + lines = [line for line in open(request_log).read().splitlines() if line.strip()] +except OSError: + sys.exit("the search benchmark sent no index request") +if not lines: + sys.exit("the search benchmark sent no index request") +payload = json.loads(lines[0]) +if payload.get("repo_path") != expected: + sys.exit(f"repo_path is {payload.get('repo_path')!r}, expected {expected!r}") +PY + ;; +esac + echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller" From 9574c316e43527d6ca90eb007a9c8fc24fcfd88b Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Mon, 21 Sep 2026 12:25:28 +0300 Subject: [PATCH 3/3] fix(test): hand the benchmark index to the caller on request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language with benchmark-index.sh and then answers graph questions against that index from its own MCP session. With a run-private runtime (#1696) the index was gone before that session could start, so the handoff is now explicit and opt-in. With CBM_BENCH_KEEP_RUNTIME set, a SUCCESSFUL run stops its daemon, leaves the private root in place and records the paths that reach it in //runtime-root.txt (sourceable: CBM_BENCH_RUNTIME_ROOT, CBM_RUNTIME_DIR, CBM_CACHE_DIR). Ownership of that root, including its removal, passes to the caller. A failed run cleans up regardless: there is no index worth keeping and nothing may leak. Nothing changes in the default path or in the three timing files. §7's skeleton sets the flag, sources the file before the graph session and removes the root in step 8 instead of the live-store *.db files. The contract test drives a kept run and requires the root to survive, the handoff file to be sourceable, and its paths to be the ones the product processes actually used; the unflagged run still asserts the root is gone. The kept-run assertions fail against the previous head. Signed-off-by: Anton Standrik --- docs/EVALUATION_PLAN.md | 7 +++++- scripts/benchmark-index.sh | 23 ++++++++++++++++- ...st_benchmark_runtime_isolation_contract.sh | 25 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md index 5f8bb532c..0b546d187 100644 --- a/docs/EVALUATION_PLAN.md +++ b/docs/EVALUATION_PLAN.md @@ -283,8 +283,13 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list # --- step 2: cold index in the main channel, TIMED (key metric) --- t0=$(now_ms) + CBM_BENCH_KEEP_RUNTIME=1 \ scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results index_ms=$(( $(now_ms) - t0 )) # clone+index wall-clock → manifest + report (§5) + # The harness indexes into a run-private root (#1696). CBM_BENCH_KEEP_RUNTIME leaves that + # root behind on success and records its paths; the graph session in steps 4-7 must be + # started with these CBM_RUNTIME_DIR / CBM_CACHE_DIR, and step 8 removes the root. + set -a; . /tmp/eval-results/"$lang"/runtime-root.txt; set +a # --- step 3: record per-type histograms (zeros back-filled) --- # node-types.json, edge-types.json (every label + all 32 edge types, zeros kept, §7 below) @@ -293,7 +298,7 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list # + per-language report + (deferred, blind) judge --- # --- step 8: delete THIS language's index so the next is cold, then mark done --- - rm -f ~/.cache/codebase-memory-mcp/*.db + rm -rf -- "$CBM_BENCH_RUNTIME_ROOT" manifest_mark_done "$lang" "$index_ms" done ``` diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 0bc24af2f..11d79ee1f 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -15,7 +15,28 @@ RESULTS_DIR="${4:?}" # shellcheck source=test-runtime.sh source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" cbm_test_runtime_init -trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + +# The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language here and +# then answers graph questions against that index from its own MCP session. +# A run-private root would be gone before that session starts, so the caller +# may ask for it to be kept: after a SUCCESSFUL run the harness stops its +# daemon, leaves the root in place, records the paths that reach it in +# //runtime-root.txt (sourceable), and ownership of the root — +# including its removal — passes to the caller. A failed run cleans up +# regardless: there is no index worth keeping, and nothing must leak. Never on +# by default, or an unattended run accumulates one root per language. +bench_finish() { + local rc=$? + if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then + "$BINARY" daemon stop >/dev/null 2>&1 || true + printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \ + "$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt" + echo " $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2 + return 0 + fi + cbm_test_runtime_cleanup "$BINARY" +} +trap bench_finish EXIT # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh index 1dc618ae6..73c1758f2 100755 --- a/tests/test_benchmark_runtime_isolation_contract.sh +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -90,6 +90,31 @@ for metric in setup-time total-time index-time; do fail "benchmark-index did not record $metric.txt" done +# The evaluation plan indexes a language and then reads that index from its own +# MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a +# successful run must leave its root behind and record, sourceably, the paths +# that reach it — and they must be the paths the product processes actually +# used. Unasked, the root is gone (asserted above). +KEEP_LOG="$WORKDIR/keep-environment.log" +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$KEEP_LOG" \ +CBM_BENCH_KEEP_RUNTIME=1 \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/keep.out" 2>&1 || true +HANDOFF="$WORKDIR/results/keep/runtime-root.txt" +[[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff" +KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF") +KEPT_RUNTIME=$(bash -c '. "$1" && printf "%s" "${CBM_RUNTIME_DIR-}"' _ "$HANDOFF") +KEPT_CACHE=$(bash -c '. "$1" && printf "%s" "${CBM_CACHE_DIR-}"' _ "$HANDOFF") +[[ -n "$KEPT_ROOT" && -d "$KEPT_ROOT" && ! -L "$KEPT_ROOT" ]] || + fail "benchmark-index did not keep its runtime root: ${KEPT_ROOT:-}" +[[ -d "$KEPT_ROOT/cache" && -d "$KEPT_ROOT/runtime" ]] || + fail "the kept root $KEPT_ROOT lost its cache or runtime directory" +grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" || + fail "runtime-root.txt does not name the runtime and cache the product processes used" +rm -rf -- "$KEPT_ROOT" + SEARCH_LOG="$WORKDIR/search-environment.log" CBM_CACHE_DIR="$CALLER_CACHE" \ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \