refactor(claim-evidence): consume agent-eval runBoundedProcess; delete the private runner

CHANGELOG.md

@@ -1,5 +1,34 @@
 # Changelog
 
+## 13.0.0 — 2026-09-01
+
+### Changed
+
+- `verifyGradeableEvidence` runs a claim's check through agent-eval's `runBoundedProcess` instead of this package's own `execFile` wrapper.
+The private wrapper spawned the check in the GRADER's process group, so a deadline killed the shell alone.
+A check that starts its real work in the background — `solver & wait` — left that work running, holding the stdout and stderr pipes open, and the grader observed no result either.
+Three solver processes outlived their grading parent by five days, and a grading loop hung twice on the same cause, for 8 hours and for 1.9 hours.
+The shared runner gives the check its own process group and kills the group, so a deadline reaches the whole tree.
+- The check body reaches bash as an argument vector and never as shell text, so no quoting stands between an author's check and the interpreter that reads it.
+This needed `args` on `runBoundedProcess`, added in agent-eval 0.172.1.
+- Grading is unchanged for a check that runs and finishes.
+A deadline now reports exit status 124, this package's own `DEADLINE_EXIT_CODE`, where the old wrapper reported 127.
+
+### Fixed
+
+- A `bash -n` parse that the executor stopped is no longer reported as a check that does not parse.
+`verifyGradeableEvidence` raised `UncheckableClaimError` on any non-zero status from the parse pass, so a parse killed at its deadline, or by the caller's own signal, blamed the author for a command bash never finished reading.
+Measured on the pinned runner: a signal already aborted at call time produced "the recorded check does not parse under bash".
+A stopped parse now grades `unrunnable` and raises nothing; a parse that really ran and failed still raises, because a command that cannot run anywhere is a record-time defect the author must fix.
+
+### Added
+
+- `CheckExecution` gains `killedBySignal` and `outputTruncated`, and `gradeFor` grades both `unrunnable` before it reads the exit status.
+**This is why the release is a major.** Reading a `CheckExecution` needs no change, because both fields are optional. Producing one does: an executor that kills a check on a caller's signal, or that stops keeping the check's output, must now say so, or `gradeFor` will grade a fragment as if it were the whole reading. Every executor in this package is updated; a consumer with its own executor sets the two fields.
+Neither is an observation of the claim: the first says the caller withdrew the run, the second says the executor stopped keeping the output before the check stopped printing, so the comparison would answer about a fragment.
+`UNRUNNABLE_SIGNATURES` stays what it was calibrated for — reading a failure out of what the CHECK printed — and no longer has to recognise the executor's own conditions from the words an error object happened to use.
+The old `execFile` wrapper reported both as `Error` text appended to stderr, which is why the signature list carried `AbortError` and `ERR_CHILD_PROCESS_STDIO_MAXBUFFER`; an executor that caused a condition now states it.
+
 ## 12.0.2 — 2026-09-01
 
 ### Changed

api-surface.json

@@ -78,7 +78,7 @@
       "BuildRetrievalEvalDispatchOptions": "value 2a1fe2b2da73",
       "CHECKABLE_RUNG_THRESHOLD": "value 534bcde62c80",
       "CITES_INVALIDATED_FIELD": "value 3fff28ee6d1f",
-      "CheckExecution": "value f8a48e381773",
+      "CheckExecution": "value 884df3b223af",
       "ChunkingOptions": "value 00fb66d7d155",
       "ClaimEvidence": "value f780da49a3ef",
       "ClaimGrade": "value a9e6edbc7877",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-knowledge",
-  "version": "12.0.2",
+  "version": "13.0.0",
   "description": "Build, search, evaluate, and improve source-backed knowledge bases.",
   "homepage": "https://github.com/tangle-network/agent-knowledge#readme",
   "repository": {

src/claim-evidence.test.ts

@@ -1,3 +1,4 @@
+import { execFile } from 'node:child_process'
 import { tmpdir } from 'node:os'
 import { describe, expect, it } from 'vitest'
 import {
@@ -603,3 +604,134 @@ describe('verifyGradeableEvidence — the executor reports its own deadline', ()
     expect(verified.grade).toEqual({ verdict: 'verified' })
   })
 })
+
+/** Pids still in a process group, read through `pgrep`, whose "no match" exit is 1. */
+async function pidsInGroup(pgid: number): Promise<string[]> {
+  return await new Promise((resolve, reject) => {
+    execFile('pgrep', ['-g', String(pgid)], (error, stdout) => {
+      const found = stdout.split('\n').filter((line) => line.trim().length > 0)
+      if (error && (error as { code?: unknown }).code !== 1) {
+        reject(error)
+        return
+      }
+      resolve(found)
+    })
+  })
+}
+
+describe('verifyGradeableEvidence — a deadline reaches the descendants, not just the shell', () => {
+  it.skipIf(process.platform === 'win32')(
+    'leaves nothing running when the check backgrounds its real work',
+    async () => {
+      // The defect this file's runner was moved to close. A check that starts a solver in the
+      // background puts the real work in a GRANDCHILD of the grader. A deadline aimed at the
+      // shell alone leaves that grandchild running, holding the stdout and stderr pipes open, so
+      // the grader never observes a result either. Three solver processes outlived their grading
+      // parent by five days on this exact shape, and a grading loop hung twice on it, for 8 hours
+      // and for 1.9 hours. Both writes below are shell builtins, so the two pids reach the pipe
+      // with no fork and no PATH lookup.
+      const started = Date.now()
+      const verified = await verifyGradeableEvidence(
+        { rung: 4, check: 'echo $$; sleep 60 & echo $!; wait', expect: 'never printed' },
+        { cwd: tmpdir(), env: { PATH: process.env.PATH ?? '' }, timeoutMs: 200 },
+      )
+
+      // The check asks for 60s of work. Returning near that means the kill missed the descendant.
+      expect(Date.now() - started).toBeLessThan(15_000)
+      expect(verified.execution.timedOut).toBe(true)
+      expect(verified.grade.verdict).toBe('unrunnable')
+
+      const [leader, descendant] = verified.execution.stdout
+        .split('\n')
+        .map((line) => Number.parseInt(line.trim(), 10))
+      expect(Number.isInteger(leader) && (leader as number) > 1).toBe(true)
+      expect(Number.isInteger(descendant) && (descendant as number) > 1).toBe(true)
+
+      const deadline = Date.now() + 5000
+      let survivors = await pidsInGroup(leader as number)
+      while (survivors.length > 0 && Date.now() < deadline) {
+        await new Promise((resolve) => setTimeout(resolve, 50))
+        survivors = await pidsInGroup(leader as number)
+      }
+      expect(survivors).toEqual([])
+      // Asserted directly, because a surviving `sleep` answers `kill(pid, 0)` whatever pgrep says.
+      let alive = true
+      try {
+        process.kill(descendant as number, 0)
+      } catch {
+        alive = false
+      }
+      expect(alive).toBe(false)
+    },
+    40_000,
+  )
+
+  it.skipIf(process.platform === 'win32')(
+    'refuses to grade a check whose output the runner could not read in full',
+    async () => {
+      // A capture that overflowed is not a reading of the check. Grading the truncated text would
+      // report `contradicted` for an expectation that was printed in the bytes nobody kept, which
+      // is a verdict on the grader and not on the claim.
+      const verified = await verifyGradeableEvidence(
+        {
+          rung: 4,
+          check: 'head -c 200000 /dev/zero | tr "\\0" "x"; echo "done=$((0 + 1))"',
+          expect: 'done=1',
+        },
+        {
+          cwd: tmpdir(),
+          env: { PATH: process.env.PATH ?? '' },
+          timeoutMs: 20_000,
+          maxBufferBytes: 4096,
+        },
+      )
+      // The check itself exits 0 here. The refusal comes from the executor's own report that
+      // it stopped keeping the output, not from anything the check printed.
+      expect(verified.execution.outputTruncated).toBe(true)
+      expect(verified.grade.verdict).toBe('unrunnable')
+      expect(verified.grade.note).toContain('printed more than the executor kept')
+    },
+    40_000,
+  )
+
+  it('refuses to grade a check whose signal had already fired', async () => {
+    // No timer, so the landing is not a race: the signal is aborted before the call. The abort
+    // reaches the `bash -n` parse rather than the execution, and a parse that never ran is not a
+    // verdict on the check's grammar — it must not surface as "does not parse under bash".
+    const controller = new AbortController()
+    controller.abort()
+    const verified = await verifyGradeableEvidence(
+      { rung: 4, check: 'sleep 30; echo "done=$((0 + 1))"', expect: 'done=1' },
+      { cwd: tmpdir(), env: { PATH: process.env.PATH ?? '' }, signal: controller.signal },
+    )
+    expect(verified.execution.killedBySignal).toBe(true)
+    expect(verified.grade.verdict).toBe('unrunnable')
+    expect(verified.grade.note).toContain('stopped by the caller')
+  }, 20_000)
+
+  it.skipIf(process.platform === 'win32')(
+    'refuses to grade a check the caller aborted',
+    async () => {
+      // The abort may land on either phase — the parse or the execution — because the first
+      // dynamic import of the runner is measured at 175-186ms cold. Both landings must reach the
+      // same verdict, which is what the assertions below check; a timer that had to beat the
+      // import would be the flake, not the test.
+      const controller = new AbortController()
+      setTimeout(() => controller.abort(), 100)
+      const verified = await verifyGradeableEvidence(
+        { rung: 4, check: 'sleep 30; echo "done=$((0 + 1))"', expect: 'done=1' },
+        {
+          cwd: tmpdir(),
+          env: { PATH: process.env.PATH ?? '' },
+          timeoutMs: 20_000,
+          signal: controller.signal,
+        },
+      )
+      expect(verified.execution.killedBySignal).toBe(true)
+      expect(verified.execution.timedOut).toBeUndefined()
+      expect(verified.grade.verdict).toBe('unrunnable')
+      expect(verified.grade.note).toContain('stopped by the caller')
+    },
+    40_000,
+  )
+})

src/claim-evidence.ts

@@ -24,6 +24,7 @@
  * Reporting a low rung in the vocabulary of a high one is the most expensive error available to a
  * knowledge system: it is indistinguishable from success and propagates as settled provenance.
  */
+
 export type EvidenceRung = 1 | 2 | 3 | 4 | 5
 
 /** The rung at and above which a claim must be machine-checkable to be recorded at that rung. */
@@ -214,6 +215,14 @@ const DEADLINE_NOTE =
   'the check was killed at its deadline, so it never tested the claim — a deadline is a budget, ' +
   'not a verdict; raise the budget or record a check that decides within it'
 
+const ABORTED_NOTE =
+  'the check was stopped by the caller before it finished, so it never tested the claim — ' +
+  'this is a decision about the run and not a verdict on the claim'
+
+const TRUNCATED_OUTPUT_NOTE =
+  'the check printed more than the executor kept, so the claim was graded against a fragment — ' +
+  'raise the output ceiling or record a check that prints only the decisive value'
+
 const UNREACHED_INPUT_NOTE =
   'the check never reached what the claim is about, so this is a verdict on the environment ' +
   'and not on the claim'
@@ -227,6 +236,18 @@ export interface CheckExecution {
    * verdict: a check that ran out of time never tested the claim.
    */
   timedOut?: boolean
+  /**
+   * True when the caller withdrew the run before it finished. Like a deadline, this is a
+   * decision about the budget and not an observation of the claim.
+   */
+  killedBySignal?: boolean
+  /**
+   * True when the executor stopped keeping the check's output before the check stopped printing.
+   * The kept text is a fragment, so a comparison against it answers about the fragment: an
+   * expectation printed in the discarded bytes reads as absent, which would refute a claim the
+   * check may well have established.
+   */
+  outputTruncated?: boolean
 }
 
 /** A verdict with the reason a grader may report to the claim's author. */
@@ -270,6 +291,12 @@ export function gradeFor(
   if (execution.timedOut || execution.exitCode === DEADLINE_EXIT_CODE) {
     return { verdict: 'unrunnable', note: DEADLINE_NOTE }
   }
+  // Both are reports from the EXECUTOR about the run, not observations of the claim, and both
+  // are read before the exit status because a killed or half-captured run has no status worth
+  // reading. They are fields rather than text in `stderr` on purpose: an executor that reports
+  // the condition it caused must not have to spell it in words a regex happens to know.
+  if (execution.killedBySignal) return { verdict: 'unrunnable', note: ABORTED_NOTE }
+  if (execution.outputTruncated) return { verdict: 'unrunnable', note: TRUNCATED_OUTPUT_NOTE }
   if (execution.exitCode !== 0) return refutation(output, evidence.expect)
   if (mustBeCheckable) {
     const note = expectationRefusalNote(evidence.expect)
@@ -392,24 +419,36 @@ export interface VerifiedGradeableEvidence {
  * Parse, execute, and grade evidence at intake using the same cwd and environment the blind grader
  * will use later.
  *
+ * A check that bash cannot parse raises `UncheckableClaimError`, because a command that cannot run
+ * anywhere is a record-time defect the author must fix. A parse the EXECUTOR stopped — its
+ * deadline, or the caller's signal — raises nothing and is graded `unrunnable`: nothing was
+ * learned about the check's grammar, so there is nothing to refuse it for.
+ *
  * This function executes `evidence.check` verbatim. It is intentionally opt-in, Node-only at call
- * time, and dynamically imports `node:child_process` so edge consumers that never invoke it remain
- * importable. Callers must apply their own sandbox and capability policy before passing untrusted
- * commands here.
+ * time, and dynamically imports `@tangle-network/agent-eval` so edge consumers that never invoke
+ * it remain importable. Callers must apply their own sandbox and capability policy before passing
+ * untrusted commands here.
  */
 export async function verifyGradeableEvidence(
   evidence: ClaimEvidence,
   options: VerifyGradeableEvidenceOptions,
 ): Promise<VerifiedGradeableEvidence> {
   const accepted = assertGradeableEvidence(evidence)
   const check = accepted.check as string
-  const syntax = await runBash(['-n', '-c', check], options)
+  const syntax = await runCheckProcess(['-n', '-c', check], options)
+  // A parse that the executor stopped is not a reading of the check's grammar. Reporting it as a
+  // parse failure would blame the author for a deadline or for the caller's own teardown, which
+  // is the shape every other rule in this file exists to refuse. The run is graded instead, and
+  // `gradeFor` reports it `unrunnable` from the field that says what stopped it.
+  if (syntax.timedOut || syntax.killedBySignal) {
+    return { evidence: accepted, execution: syntax, grade: gradeFor(accepted, syntax) }
+  }
   if (syntax.exitCode !== 0) {
     const detail = `${syntax.stdout}\n${syntax.stderr}`.trim()
     const note = detail ? `${INVALID_SHELL_SYNTAX_NOTE}: ${detail}` : INVALID_SHELL_SYNTAX_NOTE
     throw new UncheckableClaimError(accepted.rung, note)
   }
-  const execution = await runBash(['-c', check], options)
+  const execution = await runCheckProcess(['-c', check], options)
   return {
     evidence: accepted,
     execution,
@@ -463,35 +502,33 @@ function expectationRefusalNote(expect: string | undefined): string | undefined
   return undefined
 }
 
-async function runBash(
+async function runCheckProcess(
   args: string[],
   options: VerifyGradeableEvidenceOptions,
 ): Promise<CheckExecution> {
-  const { execFile } = await import('node:child_process')
-  return new Promise((resolve) => {
-    execFile(
-      options.bashPath ?? 'bash',
-      args,
-      {
-        cwd: options.cwd,
-        env: options.env ?? {},
-        timeout: options.timeoutMs ?? 30_000,
-        maxBuffer: options.maxBufferBytes ?? 1024 * 1024,
-        signal: options.signal,
-        encoding: 'utf8',
-      },
-      (error, stdout, stderr) => {
-        const diagnostic = error && typeof error.code !== 'number' ? `${stderr}\n${error}` : stderr
-        // A process killed at the deadline reports no exit status of its own, and its name
-        // separates the deadline from a caller's abort, which is a different verdict.
-        const timedOut = error?.killed === true && error.name !== 'AbortError'
-        resolve({
-          exitCode: typeof error?.code === 'number' ? error.code : error ? 127 : 0,
-          stdout: String(stdout ?? ''),
-          stderr: String(diagnostic ?? ''),
-          ...(timedOut ? { timedOut: true } : {}),
-        })
-      },
-    )
+  const { runBoundedProcess } = await import('@tangle-network/agent-eval')
+  const result = await runBoundedProcess({
+    command: options.bashPath ?? 'bash',
+    args,
+    cwd: options.cwd,
+    // The exact environment the caller named, and nothing else: a check is graded on what it could
+    // read, so an inherited variable would make the grade depend on this process.
+    env: options.env ?? {},
+    envMode: 'replace',
+    timeoutMs: options.timeoutMs ?? 30_000,
+    maxOutputBytes: options.maxBufferBytes ?? 1024 * 1024,
+    signal: options.signal,
   })
+  return {
+    exitCode: result.exitCode,
+    stdout: result.stdout,
+    // `runnerError` is the runner's own text for a run that never started — a missing
+    // interpreter, a signal that had already fired. It is appended rather than replacing
+    // stderr, because a caller reading a failed check wants both what the check said and why
+    // the runner could not run it.
+    stderr: result.runnerError ? `${result.stderr}\n${result.runnerError}`.trim() : result.stderr,
+    ...(result.killedByTimeout ? { timedOut: true } : {}),
+    ...(result.killedBySignal ? { killedBySignal: true } : {}),
+    ...(result.outputTruncated ? { outputTruncated: true } : {}),
+  }
 }