From 9c086202c0e7f743773a47ff543d4a36d6f86b9e Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:03:11 +0000 Subject: [PATCH] ci: Shard e2e tests --- .github/workflows/checks.yaml | 39 ++++++++++++++++++++-- e2e/helpers/pr-e2e-links.test.ts | 6 ++-- e2e/scripts/build-pr-e2e-links-comment.mjs | 20 ++++++++--- e2e/scripts/run-e2e-tests.mjs | 23 ++++++++++++- 4 files changed, 77 insertions(+), 11 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index c0596a42a..acb20ac6a 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -146,8 +146,13 @@ jobs: retention-days: 1 e2e: + name: e2e (${{ matrix.shard }}/4) runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] env: BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BRAINTRUST_E2E_PROJECT_NAME: ${{ vars.BRAINTRUST_E2E_PROJECT_NAME }} @@ -166,18 +171,44 @@ jobs: shell: bash run: | RUN_CONTEXT_DIR="$(mktemp -d)" + touch "$RUN_CONTEXT_DIR/initialized" echo "dir=$RUN_CONTEXT_DIR" >> "$GITHUB_OUTPUT" - name: Install dependencies run: pnpm install --frozen-lockfile - name: Run e2e tests env: BRAINTRUST_E2E_RUN_CONTEXT_DIR: ${{ steps.run_context.outputs.dir }} - run: pnpm run test:e2e - - name: Build e2e Braintrust links summary + run: pnpm run test:e2e -- --shard=${{ matrix.shard }}/4 + - name: Upload e2e run context if: ${{ always() && steps.run_context.outputs.dir != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-run-context-${{ matrix.shard }} + path: ${{ steps.run_context.outputs.dir }} + retention-days: 1 + + e2e-summary: + needs: e2e + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + BRAINTRUST_E2E_PROJECT_NAME: ${{ vars.BRAINTRUST_E2E_PROJECT_NAME }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .tool-versions + - name: Download e2e run contexts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: e2e-run-context-* + path: ${{ runner.temp }}/e2e-run-context + - name: Build e2e Braintrust links summary shell: bash env: - BRAINTRUST_E2E_RUN_CONTEXT_DIR: ${{ steps.run_context.outputs.dir }} + BRAINTRUST_E2E_RUN_CONTEXT_DIR: ${{ runner.temp }}/e2e-run-context BRAINTRUST_ORG_NAME: ${{ vars.BRAINTRUST_ORG_NAME }} GITHUB_HEAD_REF: ${{ github.head_ref }} GITHUB_REF_NAME: ${{ github.ref_name }} @@ -495,6 +526,7 @@ jobs: - js-test - js-build - e2e + - e2e-summary - js-smoke-discover - js-smoke-test - temporal-js @@ -527,6 +559,7 @@ jobs: check_result "js-test" "${{ needs.js-test.result }}" check_result "js-build" "${{ needs.js-build.result }}" check_result "e2e" "${{ needs.e2e.result }}" + check_result "e2e-summary" "${{ needs.e2e-summary.result }}" check_result "js-smoke-discover" "${{ needs.js-smoke-discover.result }}" check_result "js-smoke-test" "${{ needs.js-smoke-test.result }}" check_result "temporal-js" "${{ needs.temporal-js.result }}" diff --git a/e2e/helpers/pr-e2e-links.test.ts b/e2e/helpers/pr-e2e-links.test.ts index dea3090e9..9f787de43 100644 --- a/e2e/helpers/pr-e2e-links.test.ts +++ b/e2e/helpers/pr-e2e-links.test.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { promisify } from "node:util"; @@ -9,7 +9,9 @@ import { expect, it } from "vitest"; it("links published runs while preserving legacy records and excluding local-only runs", async () => { const dir = await mkdtemp(path.join(tmpdir(), "braintrust-e2e-links-")); const configPath = path.join(dir, "config.json"); + const shardDir = path.join(dir, "shard-1"); try { + await mkdir(shardDir); await writeFile( configPath, JSON.stringify([ @@ -21,7 +23,7 @@ it("links published runs while preserving legacy records and excluding local-onl ]), ); await writeFile( - path.join(dir, "runs.ndjson"), + path.join(shardDir, "runs.ndjson"), [ { testRunId: "e2e-published", forwardToProduction: true }, { testRunId: "e2e-local-only", forwardToProduction: false }, diff --git a/e2e/scripts/build-pr-e2e-links-comment.mjs b/e2e/scripts/build-pr-e2e-links-comment.mjs index b7144d754..7c2aaaf41 100644 --- a/e2e/scripts/build-pr-e2e-links-comment.mjs +++ b/e2e/scripts/build-pr-e2e-links-comment.mjs @@ -173,11 +173,21 @@ async function readRunContextRecords(runContextDir) { return { records, runIdsByScenarioAndVariant }; } - const entries = await readdir(runContextDir, { withFileTypes: true }); - const ndjsonFiles = entries - .filter((entry) => entry.isFile() && entry.name.endsWith(".ndjson")) - .map((entry) => path.join(runContextDir, entry.name)) - .sort(); + const pendingDirs = [runContextDir]; + const ndjsonFiles = []; + while (pendingDirs.length > 0) { + const currentDir = pendingDirs.pop(); + const entries = await readdir(currentDir, { withFileTypes: true }); + for (const entry of entries) { + const entryPath = path.join(currentDir, entry.name); + if (entry.isDirectory()) { + pendingDirs.push(entryPath); + } else if (entry.isFile() && entry.name.endsWith(".ndjson")) { + ndjsonFiles.push(entryPath); + } + } + } + ndjsonFiles.sort(); for (const filePath of ndjsonFiles) { const raw = await readFile(filePath, "utf8"); diff --git a/e2e/scripts/run-e2e-tests.mjs b/e2e/scripts/run-e2e-tests.mjs index b4254e6b7..12d27d602 100644 --- a/e2e/scripts/run-e2e-tests.mjs +++ b/e2e/scripts/run-e2e-tests.mjs @@ -18,7 +18,27 @@ const DEFAULT_OPENAI_CODEX_E2E_MODEL = "gpt-5.3-codex"; const rawArgs = process.argv.slice(2).filter((arg) => arg !== "--"); const updateSnapshots = rawArgs.includes("--update"); -const scenarioArgs = rawArgs.filter((arg) => arg !== "--update"); +const shardArgIndex = rawArgs.findIndex( + (arg) => arg === "--shard" || arg.startsWith("--shard="), +); +const shardArg = rawArgs[shardArgIndex]; +const shardValue = + shardArgIndex === -1 + ? null + : shardArg === "--shard" + ? rawArgs[shardArgIndex + 1] + : shardArg.slice("--shard=".length); +if (shardArgIndex !== -1 && !shardValue) { + console.error("[e2e] --shard requires a value in the form /."); + process.exit(1); +} +const shardArgs = shardValue ? [`--shard=${shardValue}`] : []; +const scenarioArgs = rawArgs.filter( + (arg, index) => + arg !== "--update" && + index !== shardArgIndex && + !(shardArg === "--shard" && index === shardArgIndex + 1), +); const testTargets = scenarioArgs.length > 0 ? scenarioArgs.map((arg) => scenarioPathArg(arg)) @@ -27,6 +47,7 @@ const vitestArgs = [ "run", "--run", ...testTargets, + ...shardArgs, ...(updateSnapshots ? ["--update"] : []), ]; const result = await runProcess(VITEST_COMMAND, vitestArgs, {