ci: Shard e2e tests

.github/workflows/checks.yaml

@@ -146,8 +146,13 @@ jobs:
           retention-days: 1
 
   e2e:
+    name: e2e (${{ matrix.shard }}/4)
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
     env:
       BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
       BRAINTRUST_E2E_PROJECT_NAME: ${{ vars.BRAINTRUST_E2E_PROJECT_NAME }}
@@ -166,18 +171,44 @@ jobs:
         shell: bash
         run: |
           RUN_CONTEXT_DIR="$(mktemp -d)"
+          touch "$RUN_CONTEXT_DIR/initialized"
           echo "dir=$RUN_CONTEXT_DIR" >> "$GITHUB_OUTPUT"
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
       - name: Run e2e tests
         env:
           BRAINTRUST_E2E_RUN_CONTEXT_DIR: ${{ steps.run_context.outputs.dir }}
-        run: pnpm run test:e2e
-      - name: Build e2e Braintrust links summary
+        run: pnpm run test:e2e -- --shard=${{ matrix.shard }}/4
+      - name: Upload e2e run context
         if: ${{ always() && steps.run_context.outputs.dir != '' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-run-context-${{ matrix.shard }}
+          path: ${{ steps.run_context.outputs.dir }}
+          retention-days: 1
+
+  e2e-summary:
+    needs: e2e
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    env:
+      BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
+      BRAINTRUST_E2E_PROJECT_NAME: ${{ vars.BRAINTRUST_E2E_PROJECT_NAME }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version-file: .tool-versions
+      - name: Download e2e run contexts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: e2e-run-context-*
+          path: ${{ runner.temp }}/e2e-run-context
+      - name: Build e2e Braintrust links summary
         shell: bash
         env:
-          BRAINTRUST_E2E_RUN_CONTEXT_DIR: ${{ steps.run_context.outputs.dir }}
+          BRAINTRUST_E2E_RUN_CONTEXT_DIR: ${{ runner.temp }}/e2e-run-context
           BRAINTRUST_ORG_NAME: ${{ vars.BRAINTRUST_ORG_NAME }}
           GITHUB_HEAD_REF: ${{ github.head_ref }}
           GITHUB_REF_NAME: ${{ github.ref_name }}
@@ -495,6 +526,7 @@ jobs:
       - js-test
       - js-build
       - e2e
+      - e2e-summary
       - js-smoke-discover
       - js-smoke-test
       - temporal-js
@@ -527,6 +559,7 @@ jobs:
           check_result "js-test" "${{ needs.js-test.result }}"
           check_result "js-build" "${{ needs.js-build.result }}"
           check_result "e2e" "${{ needs.e2e.result }}"
+          check_result "e2e-summary" "${{ needs.e2e-summary.result }}"
           check_result "js-smoke-discover" "${{ needs.js-smoke-discover.result }}"
           check_result "js-smoke-test" "${{ needs.js-smoke-test.result }}"
           check_result "temporal-js" "${{ needs.temporal-js.result }}"

e2e/helpers/pr-e2e-links.test.ts

@@ -1,5 +1,5 @@
 import { execFile } from "node:child_process";
-import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import path from "node:path";
 import { promisify } from "node:util";
@@ -9,7 +9,9 @@ import { expect, it } from "vitest";
 it("links published runs while preserving legacy records and excluding local-only runs", async () => {
   const dir = await mkdtemp(path.join(tmpdir(), "braintrust-e2e-links-"));
   const configPath = path.join(dir, "config.json");
+  const shardDir = path.join(dir, "shard-1");
   try {
+    await mkdir(shardDir);
     await writeFile(
       configPath,
       JSON.stringify([
@@ -21,7 +23,7 @@ it("links published runs while preserving legacy records and excluding local-onl
       ]),
     );
     await writeFile(
-      path.join(dir, "runs.ndjson"),
+      path.join(shardDir, "runs.ndjson"),
       [
         { testRunId: "e2e-published", forwardToProduction: true },
         { testRunId: "e2e-local-only", forwardToProduction: false },

e2e/scripts/build-pr-e2e-links-comment.mjs

@@ -173,11 +173,21 @@ async function readRunContextRecords(runContextDir) {
     return { records, runIdsByScenarioAndVariant };
   }
 
-  const entries = await readdir(runContextDir, { withFileTypes: true });
-  const ndjsonFiles = entries
-    .filter((entry) => entry.isFile() && entry.name.endsWith(".ndjson"))
-    .map((entry) => path.join(runContextDir, entry.name))
-    .sort();
+  const pendingDirs = [runContextDir];
+  const ndjsonFiles = [];
+  while (pendingDirs.length > 0) {
+    const currentDir = pendingDirs.pop();
+    const entries = await readdir(currentDir, { withFileTypes: true });
+    for (const entry of entries) {
+      const entryPath = path.join(currentDir, entry.name);
+      if (entry.isDirectory()) {
+        pendingDirs.push(entryPath);
+      } else if (entry.isFile() && entry.name.endsWith(".ndjson")) {
+        ndjsonFiles.push(entryPath);
+      }
+    }
+  }
+  ndjsonFiles.sort();
 
   for (const filePath of ndjsonFiles) {
     const raw = await readFile(filePath, "utf8");

e2e/scripts/run-e2e-tests.mjs

@@ -18,7 +18,27 @@ const DEFAULT_OPENAI_CODEX_E2E_MODEL = "gpt-5.3-codex";
 
 const rawArgs = process.argv.slice(2).filter((arg) => arg !== "--");
 const updateSnapshots = rawArgs.includes("--update");
-const scenarioArgs = rawArgs.filter((arg) => arg !== "--update");
+const shardArgIndex = rawArgs.findIndex(
+  (arg) => arg === "--shard" || arg.startsWith("--shard="),
+);
+const shardArg = rawArgs[shardArgIndex];
+const shardValue =
+  shardArgIndex === -1
+    ? null
+    : shardArg === "--shard"
+      ? rawArgs[shardArgIndex + 1]
+      : shardArg.slice("--shard=".length);
+if (shardArgIndex !== -1 && !shardValue) {
+  console.error("[e2e] --shard requires a value in the form <index>/<count>.");
+  process.exit(1);
+}
+const shardArgs = shardValue ? [`--shard=${shardValue}`] : [];
+const scenarioArgs = rawArgs.filter(
+  (arg, index) =>
+    arg !== "--update" &&
+    index !== shardArgIndex &&
+    !(shardArg === "--shard" && index === shardArgIndex + 1),
+);
 const testTargets =
   scenarioArgs.length > 0
     ? scenarioArgs.map((arg) => scenarioPathArg(arg))
@@ -27,6 +47,7 @@ const vitestArgs = [
   "run",
   "--run",
   ...testTargets,
+  ...shardArgs,
   ...(updateSnapshots ? ["--update"] : []),
 ];
 const result = await runProcess(VITEST_COMMAND, vitestArgs, {